6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/Utils/AddDiscriminators.h"
35 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/Bitcode/BitcodeWriterPass.h"
38 #include "llvm-c/Transforms/PassManagerBuilder.h"
40 #include "llvm/Transforms/Instrumentation.h"
41 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
42 #include "llvm/Support/TimeProfiler.h"
43 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
44 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
45 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
46 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
47 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
48 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
49 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
50 #include "llvm/Transforms/Utils.h"
54 typedef struct LLVMOpaquePass *LLVMPassRef;
55 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
57 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
58 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
60 extern "C" void LLVMInitializePasses() {
61 PassRegistry &Registry = *PassRegistry::getPassRegistry();
62 initializeCore(Registry);
63 initializeCodeGen(Registry);
64 initializeScalarOpts(Registry);
65 initializeVectorization(Registry);
66 initializeIPO(Registry);
67 initializeAnalysis(Registry);
68 initializeTransformUtils(Registry);
69 initializeInstCombine(Registry);
70 initializeInstrumentation(Registry);
71 initializeTarget(Registry);
74 extern "C" void LLVMTimeTraceProfilerInitialize() {
75 timeTraceProfilerInitialize(
76 /* TimeTraceGranularity */ 0,
77 /* ProcName */ "rustc");
80 extern "C" void LLVMTimeTraceProfilerFinishThread() {
81 timeTraceProfilerFinishThread();
84 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
85 StringRef FN(FileName);
87 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
89 timeTraceProfilerWrite(OS);
90 timeTraceProfilerCleanup();
93 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
94 #if LLVM_VERSION_LT(15, 0)
95 StringRef SR(PassName);
96 PassRegistry *PR = PassRegistry::getPassRegistry();
98 const PassInfo *PI = PR->getPassInfo(SR);
100 return wrap(PI->createPass());
104 report_fatal_error("Legacy PM not supported with LLVM 15");
108 extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
109 #if LLVM_VERSION_LT(15, 0)
110 const bool CompileKernel = false;
111 const bool UseAfterScope = true;
113 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
115 report_fatal_error("Legacy PM not supported with LLVM 15");
119 extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
120 #if LLVM_VERSION_LT(15, 0)
121 const bool CompileKernel = false;
123 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
125 report_fatal_error("Legacy PM not supported with LLVM 15");
129 extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
130 #if LLVM_VERSION_LT(15, 0)
131 const bool CompileKernel = false;
133 return wrap(createMemorySanitizerLegacyPassPass(
134 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
136 report_fatal_error("Legacy PM not supported with LLVM 15");
140 extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
141 #if LLVM_VERSION_LT(15, 0)
142 return wrap(createThreadSanitizerLegacyPassPass());
144 report_fatal_error("Legacy PM not supported with LLVM 15");
148 extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) {
149 #if LLVM_VERSION_LT(15, 0)
150 const bool CompileKernel = false;
152 return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover));
154 report_fatal_error("Legacy PM not supported with LLVM 15");
158 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
159 #if LLVM_VERSION_LT(15, 0)
161 Pass *Pass = unwrap(RustPass);
162 PassManagerBase *PMB = unwrap(PMR);
165 report_fatal_error("Legacy PM not supported with LLVM 15");
169 extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() {
170 #if LLVM_VERSION_LT(15, 0)
171 return LLVMPassManagerBuilderCreate();
173 report_fatal_error("Legacy PM not supported with LLVM 15");
177 extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) {
178 #if LLVM_VERSION_LT(15, 0)
179 LLVMPassManagerBuilderDispose(PMB);
181 report_fatal_error("Legacy PM not supported with LLVM 15");
185 extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager(
186 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
187 #if LLVM_VERSION_LT(15, 0)
188 LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM);
190 report_fatal_error("Legacy PM not supported with LLVM 15");
194 extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager(
195 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) {
196 #if LLVM_VERSION_LT(15, 0)
197 LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM);
199 report_fatal_error("Legacy PM not supported with LLVM 15");
203 extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager(
204 LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) {
205 #if LLVM_VERSION_LT(15, 0)
206 LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner);
208 report_fatal_error("Legacy PM not supported with LLVM 15");
213 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
214 LLVMPassManagerBuilderRef PMBR,
215 LLVMPassManagerRef PMR
217 #if LLVM_VERSION_LT(15, 0)
218 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
220 report_fatal_error("Legacy PM not supported with LLVM 15");
224 extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold(
225 LLVMPassManagerBuilderRef PMB, unsigned Threshold) {
226 #if LLVM_VERSION_LT(15, 0)
227 LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold);
229 report_fatal_error("Legacy PM not supported with LLVM 15");
234 void LLVMRustAddLastExtensionPasses(
235 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
236 #if LLVM_VERSION_LT(15, 0)
237 auto AddExtensionPasses = [Passes, NumPasses](
238 const PassManagerBuilder &Builder, PassManagerBase &PM) {
239 for (size_t I = 0; I < NumPasses; I++) {
240 PM.add(unwrap(Passes[I]));
243 // Add the passes to both of the pre-finalization extension points,
244 // so they are run for optimized and non-optimized builds.
245 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
247 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
250 report_fatal_error("Legacy PM not supported with LLVM 15");
254 #ifdef LLVM_COMPONENT_X86
255 #define SUBTARGET_X86 SUBTARGET(X86)
257 #define SUBTARGET_X86
260 #ifdef LLVM_COMPONENT_ARM
261 #define SUBTARGET_ARM SUBTARGET(ARM)
263 #define SUBTARGET_ARM
266 #ifdef LLVM_COMPONENT_AARCH64
267 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
269 #define SUBTARGET_AARCH64
272 #ifdef LLVM_COMPONENT_AVR
273 #define SUBTARGET_AVR SUBTARGET(AVR)
275 #define SUBTARGET_AVR
278 #ifdef LLVM_COMPONENT_M68k
279 #define SUBTARGET_M68K SUBTARGET(M68k)
281 #define SUBTARGET_M68K
284 #ifdef LLVM_COMPONENT_MIPS
285 #define SUBTARGET_MIPS SUBTARGET(Mips)
287 #define SUBTARGET_MIPS
290 #ifdef LLVM_COMPONENT_POWERPC
291 #define SUBTARGET_PPC SUBTARGET(PPC)
293 #define SUBTARGET_PPC
296 #ifdef LLVM_COMPONENT_SYSTEMZ
297 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
299 #define SUBTARGET_SYSTEMZ
302 #ifdef LLVM_COMPONENT_MSP430
303 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
305 #define SUBTARGET_MSP430
308 #ifdef LLVM_COMPONENT_RISCV
309 #define SUBTARGET_RISCV SUBTARGET(RISCV)
311 #define SUBTARGET_RISCV
314 #ifdef LLVM_COMPONENT_SPARC
315 #define SUBTARGET_SPARC SUBTARGET(Sparc)
317 #define SUBTARGET_SPARC
320 #ifdef LLVM_COMPONENT_HEXAGON
321 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
323 #define SUBTARGET_HEXAGON
326 #define GEN_SUBTARGETS \
340 #define SUBTARGET(x) \
342 extern const SubtargetFeatureKV x##FeatureKV[]; \
343 extern const SubtargetFeatureKV x##SubTypeKV[]; \
349 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
350 const char *Feature) {
351 TargetMachine *Target = unwrap(TM);
352 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
353 return MCInfo->checkFeatures(std::string("+") + Feature);
356 enum class LLVMRustCodeModel {
365 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
367 case LLVMRustCodeModel::Tiny:
368 return CodeModel::Tiny;
369 case LLVMRustCodeModel::Small:
370 return CodeModel::Small;
371 case LLVMRustCodeModel::Kernel:
372 return CodeModel::Kernel;
373 case LLVMRustCodeModel::Medium:
374 return CodeModel::Medium;
375 case LLVMRustCodeModel::Large:
376 return CodeModel::Large;
377 case LLVMRustCodeModel::None:
380 report_fatal_error("Bad CodeModel.");
384 enum class LLVMRustCodeGenOptLevel {
391 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
393 case LLVMRustCodeGenOptLevel::None:
394 return CodeGenOpt::None;
395 case LLVMRustCodeGenOptLevel::Less:
396 return CodeGenOpt::Less;
397 case LLVMRustCodeGenOptLevel::Default:
398 return CodeGenOpt::Default;
399 case LLVMRustCodeGenOptLevel::Aggressive:
400 return CodeGenOpt::Aggressive;
402 report_fatal_error("Bad CodeGenOptLevel.");
406 enum class LLVMRustPassBuilderOptLevel {
415 #if LLVM_VERSION_LT(14,0)
416 using OptimizationLevel = PassBuilder::OptimizationLevel;
419 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
421 case LLVMRustPassBuilderOptLevel::O0:
422 return OptimizationLevel::O0;
423 case LLVMRustPassBuilderOptLevel::O1:
424 return OptimizationLevel::O1;
425 case LLVMRustPassBuilderOptLevel::O2:
426 return OptimizationLevel::O2;
427 case LLVMRustPassBuilderOptLevel::O3:
428 return OptimizationLevel::O3;
429 case LLVMRustPassBuilderOptLevel::Os:
430 return OptimizationLevel::Os;
431 case LLVMRustPassBuilderOptLevel::Oz:
432 return OptimizationLevel::Oz;
434 report_fatal_error("Bad PassBuilderOptLevel.");
438 enum class LLVMRustRelocModel {
447 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
449 case LLVMRustRelocModel::Static:
450 return Reloc::Static;
451 case LLVMRustRelocModel::PIC:
453 case LLVMRustRelocModel::DynamicNoPic:
454 return Reloc::DynamicNoPIC;
455 case LLVMRustRelocModel::ROPI:
457 case LLVMRustRelocModel::RWPI:
459 case LLVMRustRelocModel::ROPIRWPI:
460 return Reloc::ROPI_RWPI;
462 report_fatal_error("Bad RelocModel.");
466 /// getLongestEntryLength - Return the length of the longest entry in the table.
467 template<typename KV>
468 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
470 for (auto &I : Table)
471 MaxLen = std::max(MaxLen, std::strlen(I.Key));
475 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
476 const TargetMachine *Target = unwrap(TM);
477 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
478 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
479 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
480 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
481 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
483 printf("Available CPUs for this target:\n");
484 if (HostArch == TargetArch) {
485 const StringRef HostCPU = sys::getHostCPUName();
486 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
487 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
489 for (auto &CPU : CPUTable)
490 printf(" %-*s\n", MaxCPULen, CPU.Key);
494 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
495 const TargetMachine *Target = unwrap(TM);
496 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
497 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
498 return FeatTable.size();
501 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
502 const char** Feature, const char** Desc) {
503 const TargetMachine *Target = unwrap(TM);
504 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
505 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
506 const SubtargetFeatureKV Feat = FeatTable[Index];
513 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
514 printf("Target CPU help is not supported by this LLVM version.\n\n");
517 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
521 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
524 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
525 StringRef Name = sys::getHostCPUName();
530 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
531 const char *TripleStr, const char *CPU, const char *Feature,
532 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
533 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
534 bool FunctionSections,
536 bool UniqueSectionNames,
537 bool TrapUnreachable,
540 bool EmitStackSizeSection,
541 bool RelaxELFRelocations,
543 const char *SplitDwarfFile) {
545 auto OptLevel = fromRust(RustOptLevel);
546 auto RM = fromRust(RustReloc);
547 auto CM = fromRust(RustCM);
550 Triple Trip(Triple::normalize(TripleStr));
551 const llvm::Target *TheTarget =
552 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
553 if (TheTarget == nullptr) {
554 LLVMRustSetLastError(Error.c_str());
558 TargetOptions Options;
560 Options.FloatABIType = FloatABI::Default;
562 Options.FloatABIType = FloatABI::Soft;
564 Options.DataSections = DataSections;
565 Options.FunctionSections = FunctionSections;
566 Options.UniqueSectionNames = UniqueSectionNames;
567 Options.MCOptions.AsmVerbose = AsmComments;
568 Options.MCOptions.PreserveAsmComments = AsmComments;
569 Options.MCOptions.ABIName = ABIStr;
570 if (SplitDwarfFile) {
571 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
573 Options.RelaxELFRelocations = RelaxELFRelocations;
574 Options.UseInitArray = UseInitArray;
576 if (TrapUnreachable) {
577 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
578 // This limits the extent of possible undefined behavior in some cases, as
579 // it prevents control flow from "falling through" into whatever code
580 // happens to be laid out next in memory.
581 Options.TrapUnreachable = true;
585 Options.ThreadModel = ThreadModel::Single;
588 Options.EmitStackSizeSection = EmitStackSizeSection;
590 TargetMachine *TM = TheTarget->createTargetMachine(
591 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
595 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
599 extern "C" void LLVMRustConfigurePassManagerBuilder(
600 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
601 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
602 const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath,
604 #if LLVM_VERSION_LT(15, 0)
605 unwrap(PMBR)->MergeFunctions = MergeFunctions;
606 unwrap(PMBR)->SLPVectorize = SLPVectorize;
607 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
608 unwrap(PMBR)->LoopVectorize = LoopVectorize;
609 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
610 unwrap(PMBR)->SizeLevel = SizeLevel;
611 unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0;
614 assert(!PGOUsePath && !PGOSampleUsePath);
615 unwrap(PMBR)->EnablePGOInstrGen = true;
616 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
617 } else if (PGOUsePath) {
618 assert(!PGOSampleUsePath);
619 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
620 } else if (PGOSampleUsePath) {
621 unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath;
624 report_fatal_error("Legacy PM not supported with LLVM 15");
628 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
629 // field of a PassManagerBuilder, we expose our own method of doing so.
630 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
632 bool DisableSimplifyLibCalls) {
633 Triple TargetTriple(unwrap(M)->getTargetTriple());
634 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
635 if (DisableSimplifyLibCalls)
636 TLI->disableAllFunctions();
637 unwrap(PMBR)->LibraryInfo = TLI;
640 // Unfortunately, the LLVM C API doesn't provide a way to create the
641 // TargetLibraryInfo pass, so we use this method to do so.
642 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
643 bool DisableSimplifyLibCalls) {
644 Triple TargetTriple(unwrap(M)->getTargetTriple());
645 TargetLibraryInfoImpl TLII(TargetTriple);
646 if (DisableSimplifyLibCalls)
647 TLII.disableAllFunctions();
648 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
651 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
652 // all the functions in a module, so we do that manually here. You'll find
653 // similar code in clang's BackendUtil.cpp file.
654 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
656 llvm::legacy::FunctionPassManager *P =
657 unwrap<llvm::legacy::FunctionPassManager>(PMR);
658 P->doInitialization();
660 // Upgrade all calls to old intrinsics first.
661 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
662 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
664 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
666 if (!I->isDeclaration())
672 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
673 // Initializing the command-line options more than once is not allowed. So,
674 // check if they've already been initialized. (This could happen if we're
675 // being called from rustpkg, for example). If the arguments change, then
676 // that's just kinda unfortunate.
677 static bool Initialized = false;
681 cl::ParseCommandLineOptions(Argc, Argv);
684 enum class LLVMRustFileType {
689 static CodeGenFileType fromRust(LLVMRustFileType Type) {
691 case LLVMRustFileType::AssemblyFile:
692 return CGFT_AssemblyFile;
693 case LLVMRustFileType::ObjectFile:
694 return CGFT_ObjectFile;
696 report_fatal_error("Bad FileType.");
700 extern "C" LLVMRustResult
701 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
702 LLVMModuleRef M, const char *Path, const char *DwoPath,
703 LLVMRustFileType RustFileType) {
704 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
705 auto FileType = fromRust(RustFileType);
707 std::string ErrorInfo;
709 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
711 ErrorInfo = EC.message();
712 if (ErrorInfo != "") {
713 LLVMRustSetLastError(ErrorInfo.c_str());
714 return LLVMRustResult::Failure;
717 buffer_ostream BOS(OS);
719 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
722 ErrorInfo = EC.message();
723 if (ErrorInfo != "") {
724 LLVMRustSetLastError(ErrorInfo.c_str());
725 return LLVMRustResult::Failure;
727 buffer_ostream DBOS(DOS);
728 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
731 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
735 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
736 // stream (OS), so the only real safe place to delete this is here? Don't we
737 // wish this was written in Rust?
738 LLVMDisposePassManager(PMR);
739 return LLVMRustResult::Success;
742 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
743 const char*, // pass name
744 const char*); // IR name
745 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
747 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
748 if (any_isa<const Module *>(WrappedIr))
749 return any_cast<const Module *>(WrappedIr)->getName().str();
750 if (any_isa<const Function *>(WrappedIr))
751 return any_cast<const Function *>(WrappedIr)->getName().str();
752 if (any_isa<const Loop *>(WrappedIr))
753 return any_cast<const Loop *>(WrappedIr)->getName().str();
754 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
755 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
760 void LLVMSelfProfileInitializeCallbacks(
761 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
762 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
763 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
764 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
765 StringRef Pass, llvm::Any Ir) {
766 std::string PassName = Pass.str();
767 std::string IrName = LLVMRustwrappedIrGetName(Ir);
768 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
771 PIC.registerAfterPassCallback(
772 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
773 const PreservedAnalyses &Preserved) {
774 AfterPassCallback(LlvmSelfProfiler);
777 PIC.registerAfterPassInvalidatedCallback(
778 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
779 AfterPassCallback(LlvmSelfProfiler);
782 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
783 StringRef Pass, llvm::Any Ir) {
784 std::string PassName = Pass.str();
785 std::string IrName = LLVMRustwrappedIrGetName(Ir);
786 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
789 PIC.registerAfterAnalysisCallback(
790 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
791 AfterPassCallback(LlvmSelfProfiler);
795 enum class LLVMRustOptStage {
803 struct LLVMRustSanitizerOptions {
804 bool SanitizeAddress;
805 bool SanitizeAddressRecover;
807 bool SanitizeMemoryRecover;
808 int SanitizeMemoryTrackOrigins;
810 bool SanitizeHWAddress;
811 bool SanitizeHWAddressRecover;
814 extern "C" LLVMRustResult
815 LLVMRustOptimizeWithNewPassManager(
816 LLVMModuleRef ModuleRef,
817 LLVMTargetMachineRef TMRef,
818 LLVMRustPassBuilderOptLevel OptLevelRust,
819 LLVMRustOptStage OptStage,
820 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
821 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
822 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
823 LLVMRustSanitizerOptions *SanitizerOptions,
824 const char *PGOGenPath, const char *PGOUsePath,
825 bool InstrumentCoverage, const char *InstrProfileOutput,
827 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
828 void* LlvmSelfProfiler,
829 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
830 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
831 const char *ExtraPasses, size_t ExtraPassesLen,
832 const char *LLVMPlugins, size_t LLVMPluginsLen) {
833 Module *TheModule = unwrap(ModuleRef);
834 TargetMachine *TM = unwrap(TMRef);
835 OptimizationLevel OptLevel = fromRust(OptLevelRust);
838 PipelineTuningOptions PTO;
839 PTO.LoopUnrolling = UnrollLoops;
840 PTO.LoopInterleaving = UnrollLoops;
841 PTO.LoopVectorization = LoopVectorize;
842 PTO.SLPVectorization = SLPVectorize;
843 PTO.MergeFunctions = MergeFunctions;
845 // FIXME: We may want to expose this as an option.
846 bool DebugPassManager = false;
848 PassInstrumentationCallbacks PIC;
849 StandardInstrumentations SI(DebugPassManager);
850 SI.registerCallbacks(PIC);
852 if (LlvmSelfProfiler){
853 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
856 Optional<PGOOptions> PGOOpt;
858 assert(!PGOUsePath && !PGOSampleUsePath);
859 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
860 PGOOptions::NoCSAction, DebugInfoForProfiling);
861 } else if (PGOUsePath) {
862 assert(!PGOSampleUsePath);
863 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
864 PGOOptions::NoCSAction, DebugInfoForProfiling);
865 } else if (PGOSampleUsePath) {
866 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
867 PGOOptions::NoCSAction, DebugInfoForProfiling);
868 } else if (DebugInfoForProfiling) {
869 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
870 PGOOptions::NoCSAction, DebugInfoForProfiling);
873 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
874 LoopAnalysisManager LAM;
875 FunctionAnalysisManager FAM;
876 CGSCCAnalysisManager CGAM;
877 ModuleAnalysisManager MAM;
879 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
881 Triple TargetTriple(TheModule->getTargetTriple());
882 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
883 if (DisableSimplifyLibCalls)
884 TLII->disableAllFunctions();
885 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
887 PB.registerModuleAnalyses(MAM);
888 PB.registerCGSCCAnalyses(CGAM);
889 PB.registerFunctionAnalyses(FAM);
890 PB.registerLoopAnalyses(LAM);
891 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
893 // We manually collect pipeline callbacks so we can apply them at O0, where the
894 // PassBuilder does not create a pipeline.
895 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
896 PipelineStartEPCallbacks;
897 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
898 OptimizerLastEPCallbacks;
901 PipelineStartEPCallbacks.push_back(
902 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
903 MPM.addPass(VerifierPass());
908 if (InstrumentGCOV) {
909 PipelineStartEPCallbacks.push_back(
910 [](ModulePassManager &MPM, OptimizationLevel Level) {
911 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
916 if (InstrumentCoverage) {
917 PipelineStartEPCallbacks.push_back(
918 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
919 InstrProfOptions Options;
920 if (InstrProfileOutput) {
921 Options.InstrProfileOutput = InstrProfileOutput;
923 MPM.addPass(InstrProfiling(Options, false));
928 if (SanitizerOptions) {
929 if (SanitizerOptions->SanitizeMemory) {
930 MemorySanitizerOptions Options(
931 SanitizerOptions->SanitizeMemoryTrackOrigins,
932 SanitizerOptions->SanitizeMemoryRecover,
933 /*CompileKernel=*/false);
934 OptimizerLastEPCallbacks.push_back(
935 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
936 #if LLVM_VERSION_GE(14, 0)
937 MPM.addPass(ModuleMemorySanitizerPass(Options));
939 MPM.addPass(MemorySanitizerPass(Options));
941 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
946 if (SanitizerOptions->SanitizeThread) {
947 OptimizerLastEPCallbacks.push_back(
948 [](ModulePassManager &MPM, OptimizationLevel Level) {
949 #if LLVM_VERSION_GE(14, 0)
950 MPM.addPass(ModuleThreadSanitizerPass());
952 MPM.addPass(ThreadSanitizerPass());
954 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
959 if (SanitizerOptions->SanitizeAddress) {
960 OptimizerLastEPCallbacks.push_back(
961 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
962 #if LLVM_VERSION_LT(15, 0)
963 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
965 #if LLVM_VERSION_GE(14, 0)
966 AddressSanitizerOptions opts = AddressSanitizerOptions{
967 /*CompileKernel=*/false,
968 SanitizerOptions->SanitizeAddressRecover,
969 /*UseAfterScope=*/true,
970 AsanDetectStackUseAfterReturnMode::Runtime,
972 MPM.addPass(ModuleAddressSanitizerPass(opts));
974 MPM.addPass(ModuleAddressSanitizerPass(
975 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
976 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
977 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
978 /*UseAfterScope=*/true)));
983 if (SanitizerOptions->SanitizeHWAddress) {
984 OptimizerLastEPCallbacks.push_back(
985 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
986 #if LLVM_VERSION_GE(14, 0)
987 HWAddressSanitizerOptions opts(
988 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
989 /*DisableOptimization=*/false);
990 MPM.addPass(HWAddressSanitizerPass(opts));
992 MPM.addPass(HWAddressSanitizerPass(
993 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
1000 if (LLVMPluginsLen) {
1001 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
1002 SmallVector<StringRef> Plugins;
1003 PluginsStr.split(Plugins, ',', -1, false);
1004 for (auto PluginPath: Plugins) {
1005 auto Plugin = PassPlugin::Load(PluginPath.str());
1007 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
1010 Plugin->registerPassBuilderCallbacks(PB);
1014 ModulePassManager MPM;
1015 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
1016 if (!NoPrepopulatePasses) {
1017 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
1018 // At the same time, the LTO pipelines do support O0 and using them is required.
1019 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
1020 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
1021 for (const auto &C : PipelineStartEPCallbacks)
1022 PB.registerPipelineStartEPCallback(C);
1023 for (const auto &C : OptimizerLastEPCallbacks)
1024 PB.registerOptimizerLastEPCallback(C);
1026 // Pass false as we manually schedule ThinLTOBufferPasses below.
1027 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
1029 for (const auto &C : PipelineStartEPCallbacks)
1030 PB.registerPipelineStartEPCallback(C);
1031 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
1032 for (const auto &C : OptimizerLastEPCallbacks)
1033 PB.registerOptimizerLastEPCallback(C);
1037 case LLVMRustOptStage::PreLinkNoLTO:
1038 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
1040 case LLVMRustOptStage::PreLinkThinLTO:
1041 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
1042 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
1043 // passes may still run afterwards. This means we need to run the buffer passes again.
1044 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
1045 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
1046 if (OptimizerLastEPCallbacks.empty())
1047 NeedThinLTOBufferPasses = false;
1048 for (const auto &C : OptimizerLastEPCallbacks)
1051 case LLVMRustOptStage::PreLinkFatLTO:
1052 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
1053 NeedThinLTOBufferPasses = false;
1055 case LLVMRustOptStage::ThinLTO:
1056 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
1057 // It only seems to be needed for C++ specific optimizations.
1058 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
1060 case LLVMRustOptStage::FatLTO:
1061 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
1066 // We're not building any of the default pipelines but we still want to
1067 // add the verifier, instrumentation, etc passes if they were requested
1068 for (const auto &C : PipelineStartEPCallbacks)
1070 for (const auto &C : OptimizerLastEPCallbacks)
1074 if (ExtraPassesLen) {
1075 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
1076 std::string ErrMsg = toString(std::move(Err));
1077 LLVMRustSetLastError(ErrMsg.c_str());
1078 return LLVMRustResult::Failure;
1082 if (NeedThinLTOBufferPasses) {
1083 MPM.addPass(CanonicalizeAliasesPass());
1084 MPM.addPass(NameAnonGlobalPass());
1087 // Upgrade all calls to old intrinsics first.
1088 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1089 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1091 MPM.run(*TheModule, MAM);
1092 return LLVMRustResult::Success;
1095 // Callback to demangle function name
1097 // * name to be demangled
1100 // * output buffer len
1101 // Returns len of demangled string, or 0 if demangle failed.
1102 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1107 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1108 DemangleFn Demangle;
1109 std::vector<char> Buf;
1112 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1114 // Return empty string if demangle failed
1115 // or if name does not need to be demangled
1116 StringRef CallDemangle(StringRef name) {
1121 if (Buf.size() < name.size() * 2) {
1122 // Semangled name usually shorter than mangled,
1123 // but allocate twice as much memory just in case
1124 Buf.resize(name.size() * 2);
1127 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1133 auto Demangled = StringRef(Buf.data(), R);
1134 if (Demangled == name) {
1135 // Do not print anything if demangled name is equal to mangled.
1142 void emitFunctionAnnot(const Function *F,
1143 formatted_raw_ostream &OS) override {
1144 StringRef Demangled = CallDemangle(F->getName());
1145 if (Demangled.empty()) {
1149 OS << "; " << Demangled << "\n";
1152 void emitInstructionAnnot(const Instruction *I,
1153 formatted_raw_ostream &OS) override {
1156 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1158 Value = CI->getCalledOperand();
1159 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1161 Value = II->getCalledOperand();
1163 // Could demangle more operations, e. g.
1164 // `store %place, @function`.
1168 if (!Value->hasName()) {
1172 StringRef Demangled = CallDemangle(Value->getName());
1173 if (Demangled.empty()) {
1177 OS << "; " << Name << " " << Demangled << "\n";
1183 extern "C" LLVMRustResult
1184 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1185 std::string ErrorInfo;
1187 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1189 ErrorInfo = EC.message();
1190 if (ErrorInfo != "") {
1191 LLVMRustSetLastError(ErrorInfo.c_str());
1192 return LLVMRustResult::Failure;
1195 RustAssemblyAnnotationWriter AAW(Demangle);
1196 formatted_raw_ostream FOS(OS);
1197 unwrap(M)->print(FOS, &AAW);
1199 return LLVMRustResult::Success;
1202 extern "C" void LLVMRustPrintPasses() {
1203 LLVMInitializePasses();
1204 struct MyListener : PassRegistrationListener {
1205 void passEnumerate(const PassInfo *Info) {
1206 StringRef PassArg = Info->getPassArgument();
1207 StringRef PassName = Info->getPassName();
1208 if (!PassArg.empty()) {
1209 // These unsigned->signed casts could theoretically overflow, but
1210 // realistically never will (and even if, the result is implementation
1211 // defined rather plain UB).
1212 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1213 (int)PassName.size(), PassName.data());
1218 PassRegistry *PR = PassRegistry::getPassRegistry();
1219 PR->enumerateWith(&Listener);
1222 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1223 bool AddLifetimes) {
1224 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1227 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1229 llvm::legacy::PassManager passes;
1231 auto PreserveFunctions = [=](const GlobalValue &GV) {
1232 for (size_t I = 0; I < Len; I++) {
1233 if (GV.getName() == Symbols[I]) {
1240 passes.add(llvm::createInternalizePass(PreserveFunctions));
1242 passes.run(*unwrap(M));
1246 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1247 LLVMTargetMachineRef TMR) {
1248 TargetMachine *Target = unwrap(TMR);
1249 unwrap(Module)->setDataLayout(Target->createDataLayout());
1252 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1253 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1256 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1257 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1260 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1261 LLVMRustCodeModel Model) {
1262 auto CM = fromRust(Model);
1265 unwrap(M)->setCodeModel(*CM);
1268 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1269 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1270 // LLVM, and otherwise it's just blanket rejected from other compilers.
1272 // Most of this implementation is straight copied from LLVM. At the time of
1273 // this writing it wasn't *quite* suitable to reuse more code from upstream
1274 // for our purposes, but we should strive to upstream this support once it's
1275 // ready to go! I figure we may want a bit of testing locally first before
1276 // sending this upstream to LLVM. I hear though they're quite eager to receive
1277 // feedback like this!
1279 // If you're reading this code and wondering "what in the world" or you're
1280 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1281 // then fear not! (ok maybe fear a little). All code here is mostly based
1282 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1284 // You'll find that the general layout here roughly corresponds to the `run`
1285 // method in that file as well as `ProcessThinLTOModule`. Functions are
1286 // specifically commented below as well, but if you're updating this code
1287 // or otherwise trying to understand it, the LLVM source will be useful in
1288 // interpreting the mysteries within.
1290 // Otherwise I'll apologize in advance, it probably requires a relatively
1291 // significant investment on your part to "truly understand" what's going on
1292 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1293 // and various online resources about ThinLTO to make heads or tails of all
1296 // This is a shared data structure which *must* be threadsafe to share
1297 // read-only amongst threads. This also corresponds basically to the arguments
1298 // of the `ProcessThinLTOModule` function in the LLVM source.
1299 struct LLVMRustThinLTOData {
1300 // The combined index that is the global analysis over all modules we're
1301 // performing ThinLTO for. This is mostly managed by LLVM.
1302 ModuleSummaryIndex Index;
1304 // All modules we may look at, stored as in-memory serialized versions. This
1305 // is later used when inlining to ensure we can extract any module to inline
1307 StringMap<MemoryBufferRef> ModuleMap;
1309 // A set that we manage of everything we *don't* want internalized. Note that
1310 // this includes all transitive references right now as well, but it may not
1312 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1314 // Not 100% sure what these are, but they impact what's internalized and
1315 // what's inlined across modules, I believe.
1316 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1317 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1318 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1319 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1321 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1324 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1325 struct LLVMRustThinLTOModule {
1326 const char *identifier;
1331 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1333 static const GlobalValueSummary *
1334 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1335 auto StrongDefForLinker = llvm::find_if(
1336 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1337 auto Linkage = Summary->linkage();
1338 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1339 !GlobalValue::isWeakForLinker(Linkage);
1341 if (StrongDefForLinker != GVSummaryList.end())
1342 return StrongDefForLinker->get();
1344 auto FirstDefForLinker = llvm::find_if(
1345 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1346 auto Linkage = Summary->linkage();
1347 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1349 if (FirstDefForLinker == GVSummaryList.end())
1351 return FirstDefForLinker->get();
1354 // The main entry point for creating the global ThinLTO analysis. The structure
1355 // here is basically the same as before threads are spawned in the `run`
1356 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1357 extern "C" LLVMRustThinLTOData*
1358 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1360 const char **preserved_symbols,
1362 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1364 // Load each module's summary and merge it into one combined index
1365 for (int i = 0; i < num_modules; i++) {
1366 auto module = &modules[i];
1367 StringRef buffer(module->data, module->len);
1368 MemoryBufferRef mem_buffer(buffer, module->identifier);
1370 Ret->ModuleMap[module->identifier] = mem_buffer;
1372 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1373 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1378 // Collect for each module the list of function it defines (GUID -> Summary)
1379 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1381 // Convert the preserved symbols set from string to GUID, this is then needed
1382 // for internalization.
1383 for (int i = 0; i < num_symbols; i++) {
1384 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1385 Ret->GUIDPreservedSymbols.insert(GUID);
1388 // Collect the import/export lists for all modules from the call-graph in the
1391 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1392 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1393 return PrevailingType::Unknown;
1395 // We don't have a complete picture in our use of ThinLTO, just our immediate
1396 // crate, so we need `ImportEnabled = false` to limit internalization.
1397 // Otherwise, we sometimes lose `static` values -- see #60184.
1398 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1399 deadIsPrevailing, /* ImportEnabled = */ false);
1400 ComputeCrossModuleImport(
1402 Ret->ModuleToDefinedGVSummaries,
1407 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1408 // impacts the caching.
1410 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1411 // being lifted from `lib/LTO/LTO.cpp` as well
1412 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1413 for (auto &I : Ret->Index) {
1414 if (I.second.SummaryList.size() > 1)
1415 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1417 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1418 const auto &Prevailing = PrevailingCopy.find(GUID);
1419 if (Prevailing == PrevailingCopy.end())
1421 return Prevailing->second == S;
1423 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1424 GlobalValue::GUID GUID,
1425 GlobalValue::LinkageTypes NewLinkage) {
1426 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1429 // Uses FromPrevailing visibility scheme which works for many binary
1430 // formats. We probably could and should use ELF visibility scheme for many of
1431 // our targets, however.
1433 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1434 Ret->GUIDPreservedSymbols);
1436 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1437 // callback below. This callback below will dictate the linkage for all
1438 // summaries in the index, and we basically just only want to ensure that dead
1439 // symbols are internalized. Otherwise everything that's already external
1440 // linkage will stay as external, and internal will stay as internal.
1441 std::set<GlobalValue::GUID> ExportedGUIDs;
1442 for (auto &List : Ret->Index) {
1443 for (auto &GVS: List.second.SummaryList) {
1444 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1446 auto GUID = GVS->getOriginalName();
1447 if (GVS->flags().Live)
1448 ExportedGUIDs.insert(GUID);
1451 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1452 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1453 return (ExportList != Ret->ExportLists.end() &&
1454 ExportList->second.count(VI)) ||
1455 ExportedGUIDs.count(VI.getGUID());
1457 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1459 return Ret.release();
1463 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1467 // Below are the various passes that happen *per module* when doing ThinLTO.
1469 // In other words, these are the functions that are all run concurrently
1470 // with one another, one per module. The passes here correspond to the analysis
1471 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1472 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1473 // so rustc can save off the intermediate bytecode between each step.
1476 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1477 // When linking an ELF shared object, dso_local should be dropped. We
1478 // conservatively do this for -fpic.
1479 bool ClearDSOLocalOnDeclarations =
1480 TM.getTargetTriple().isOSBinFormatELF() &&
1481 TM.getRelocationModel() != Reloc::Static &&
1482 Mod.getPIELevel() == PIELevel::Default;
1483 return ClearDSOLocalOnDeclarations;
1487 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1488 LLVMTargetMachineRef TM) {
1489 Module &Mod = *unwrap(M);
1490 TargetMachine &Target = *unwrap(TM);
1492 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1493 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1496 LLVMRustSetLastError("renameModuleForThinLTO failed");
1503 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1504 Module &Mod = *unwrap(M);
1505 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1506 #if LLVM_VERSION_GE(14, 0)
1507 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1509 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1515 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1516 Module &Mod = *unwrap(M);
1517 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1518 thinLTOInternalizeModule(Mod, DefinedGlobals);
1523 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1524 LLVMTargetMachineRef TM) {
1525 Module &Mod = *unwrap(M);
1526 TargetMachine &Target = *unwrap(TM);
1528 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1529 auto Loader = [&](StringRef Identifier) {
1530 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1531 auto &Context = Mod.getContext();
1532 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1537 // The rest of this closure is a workaround for
1538 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1539 // we accidentally import wasm custom sections into different modules,
1540 // duplicating them by in the final output artifact.
1542 // The issue is worked around here by manually removing the
1543 // `wasm.custom_sections` named metadata node from any imported module. This
1544 // we know isn't used by any optimization pass so there's no need for it to
1547 // Note that the metadata is currently lazily loaded, so we materialize it
1548 // here before looking up if there's metadata inside. The `FunctionImporter`
1549 // will immediately materialize metadata anyway after an import, so this
1550 // shouldn't be a perf hit.
1551 if (Error Err = (*MOrErr)->materializeMetadata()) {
1552 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1556 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1557 if (WasmCustomSections)
1558 WasmCustomSections->eraseFromParent();
1562 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1563 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1564 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1566 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1572 // This struct and various functions are sort of a hack right now, but the
1573 // problem is that we've got in-memory LLVM modules after we generate and
1574 // optimize all codegen-units for one compilation in rustc. To be compatible
1575 // with the LTO support above we need to serialize the modules plus their
1576 // ThinLTO summary into memory.
1578 // This structure is basically an owned version of a serialize module, with
1579 // a ThinLTO summary attached.
1580 struct LLVMRustThinLTOBuffer {
1584 extern "C" LLVMRustThinLTOBuffer*
1585 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1586 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1588 raw_string_ostream OS(Ret->data);
1590 legacy::PassManager PM;
1592 PM.add(createWriteThinLTOBitcodePass(OS));
1594 PM.add(createBitcodeWriterPass(OS));
1599 return Ret.release();
1603 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1607 extern "C" const void*
1608 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1609 return Buffer->data.data();
1613 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1614 return Buffer->data.length();
1617 // This is what we used to parse upstream bitcode for actual ThinLTO
1618 // processing. We'll call this once per module optimized through ThinLTO, and
1619 // it'll be called concurrently on many threads.
1620 extern "C" LLVMModuleRef
1621 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1624 const char *identifier) {
1625 StringRef Data(data, len);
1626 MemoryBufferRef Buffer(Data, identifier);
1627 unwrap(Context)->enableDebugTypeODRUniquing();
1628 Expected<std::unique_ptr<Module>> SrcOrError =
1629 parseBitcodeFile(Buffer, *unwrap(Context));
1631 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1634 return wrap(std::move(*SrcOrError).release());
1637 // Find the bitcode section in the object file data and return it as a slice.
1638 // Fail if the bitcode section is present but empty.
1640 // On success, the return value is the pointer to the start of the slice and
1641 // `out_len` is filled with the (non-zero) length. On failure, the return value
1642 // is `nullptr` and `out_len` is set to zero.
1643 extern "C" const char*
1644 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1649 StringRef Data(data, len);
1650 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1652 Expected<MemoryBufferRef> BitcodeOrError =
1653 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1654 if (!BitcodeOrError) {
1655 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1659 *out_len = BitcodeOrError->getBufferSize();
1660 return BitcodeOrError->getBufferStart();
1663 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1664 // the comment in `back/lto.rs` for why this exists.
1666 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1668 DICompileUnit **B) {
1669 Module *M = unwrap(Mod);
1670 DICompileUnit **Cur = A;
1671 DICompileUnit **Next = B;
1672 for (DICompileUnit *CU : M->debug_compile_units()) {
1681 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1682 // the comment in `back/lto.rs` for why this exists.
1684 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1685 Module *M = unwrap(Mod);
1687 // If the original source module didn't have a `DICompileUnit` then try to
1688 // merge all the existing compile units. If there aren't actually any though
1689 // then there's not much for us to do so return.
1690 if (Unit == nullptr) {
1691 for (DICompileUnit *CU : M->debug_compile_units()) {
1695 if (Unit == nullptr)
1699 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1700 // process it recursively. Note that we used to specifically iterate over
1701 // instructions to ensure we feed everything into it, but `processModule`
1702 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1703 DebugInfoFinder Finder;
1704 Finder.processModule(*M);
1706 // After we've found all our debuginfo, rewrite all subprograms to point to
1707 // the same `DICompileUnit`.
1708 for (auto &F : Finder.subprograms()) {
1709 F->replaceUnit(Unit);
1712 // Erase any other references to other `DICompileUnit` instances, the verifier
1713 // will later ensure that we don't actually have any other stale references to
1715 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1716 MD->clearOperands();
1717 MD->addOperand(Unit);
1720 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1721 // storing the result in 'KeyOut'.
1722 // Currently, this cache key is a SHA-1 hash of anything that could affect
1723 // the result of optimizing this module (e.g. module imports, exports, liveness
1724 // of access globals, etc).
1725 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1726 // used during the normal linker-plugin incremental thin-LTO process.
1728 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1729 SmallString<40> Key;
1730 llvm::lto::Config conf;
1731 const auto &ImportList = Data->ImportLists.lookup(ModId);
1732 const auto &ExportList = Data->ExportLists.lookup(ModId);
1733 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1734 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1735 std::set<GlobalValue::GUID> CfiFunctionDefs;
1736 std::set<GlobalValue::GUID> CfiFunctionDecls;
1738 // Based on the 'InProcessThinBackend' constructor in LLVM
1739 for (auto &Name : Data->Index.cfiFunctionDefs())
1740 CfiFunctionDefs.insert(
1741 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1742 for (auto &Name : Data->Index.cfiFunctionDecls())
1743 CfiFunctionDecls.insert(
1744 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1746 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1747 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1750 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());