6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/IPO/Internalize.h"
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36 #include "llvm/Transforms/Utils/AddDiscriminators.h"
37 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
38 #include "llvm/LTO/LTO.h"
39 #include "llvm/Bitcode/BitcodeWriter.h"
40 #include "llvm-c/Transforms/PassManagerBuilder.h"
42 #include "llvm/Transforms/Instrumentation.h"
43 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
44 #include "llvm/Support/TimeProfiler.h"
45 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
46 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
47 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
48 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
49 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
50 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
51 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
52 #include "llvm/Transforms/Utils.h"
56 typedef struct LLVMOpaquePass *LLVMPassRef;
57 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
59 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
62 extern "C" void LLVMInitializePasses() {
63 PassRegistry &Registry = *PassRegistry::getPassRegistry();
64 initializeCore(Registry);
65 initializeCodeGen(Registry);
66 initializeScalarOpts(Registry);
67 initializeVectorization(Registry);
68 initializeIPO(Registry);
69 initializeAnalysis(Registry);
70 initializeTransformUtils(Registry);
71 initializeInstCombine(Registry);
72 initializeInstrumentation(Registry);
73 initializeTarget(Registry);
76 extern "C" void LLVMTimeTraceProfilerInitialize() {
77 timeTraceProfilerInitialize(
78 /* TimeTraceGranularity */ 0,
79 /* ProcName */ "rustc");
82 extern "C" void LLVMTimeTraceProfilerFinishThread() {
83 timeTraceProfilerFinishThread();
86 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
87 StringRef FN(FileName);
89 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
91 timeTraceProfilerWrite(OS);
92 timeTraceProfilerCleanup();
95 #ifdef LLVM_COMPONENT_X86
96 #define SUBTARGET_X86 SUBTARGET(X86)
101 #ifdef LLVM_COMPONENT_ARM
102 #define SUBTARGET_ARM SUBTARGET(ARM)
104 #define SUBTARGET_ARM
107 #ifdef LLVM_COMPONENT_AARCH64
108 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
110 #define SUBTARGET_AARCH64
113 #ifdef LLVM_COMPONENT_AVR
114 #define SUBTARGET_AVR SUBTARGET(AVR)
116 #define SUBTARGET_AVR
119 #ifdef LLVM_COMPONENT_M68k
120 #define SUBTARGET_M68K SUBTARGET(M68k)
122 #define SUBTARGET_M68K
125 #ifdef LLVM_COMPONENT_MIPS
126 #define SUBTARGET_MIPS SUBTARGET(Mips)
128 #define SUBTARGET_MIPS
131 #ifdef LLVM_COMPONENT_POWERPC
132 #define SUBTARGET_PPC SUBTARGET(PPC)
134 #define SUBTARGET_PPC
137 #ifdef LLVM_COMPONENT_SYSTEMZ
138 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
140 #define SUBTARGET_SYSTEMZ
143 #ifdef LLVM_COMPONENT_MSP430
144 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
146 #define SUBTARGET_MSP430
149 #ifdef LLVM_COMPONENT_RISCV
150 #define SUBTARGET_RISCV SUBTARGET(RISCV)
152 #define SUBTARGET_RISCV
155 #ifdef LLVM_COMPONENT_SPARC
156 #define SUBTARGET_SPARC SUBTARGET(Sparc)
158 #define SUBTARGET_SPARC
161 #ifdef LLVM_COMPONENT_HEXAGON
162 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
164 #define SUBTARGET_HEXAGON
167 #define GEN_SUBTARGETS \
181 #define SUBTARGET(x) \
183 extern const SubtargetFeatureKV x##FeatureKV[]; \
184 extern const SubtargetFeatureKV x##SubTypeKV[]; \
190 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
191 const char *Feature) {
192 TargetMachine *Target = unwrap(TM);
193 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
194 return MCInfo->checkFeatures(std::string("+") + Feature);
197 enum class LLVMRustCodeModel {
206 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
208 case LLVMRustCodeModel::Tiny:
209 return CodeModel::Tiny;
210 case LLVMRustCodeModel::Small:
211 return CodeModel::Small;
212 case LLVMRustCodeModel::Kernel:
213 return CodeModel::Kernel;
214 case LLVMRustCodeModel::Medium:
215 return CodeModel::Medium;
216 case LLVMRustCodeModel::Large:
217 return CodeModel::Large;
218 case LLVMRustCodeModel::None:
221 report_fatal_error("Bad CodeModel.");
225 enum class LLVMRustCodeGenOptLevel {
232 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
234 case LLVMRustCodeGenOptLevel::None:
235 return CodeGenOpt::None;
236 case LLVMRustCodeGenOptLevel::Less:
237 return CodeGenOpt::Less;
238 case LLVMRustCodeGenOptLevel::Default:
239 return CodeGenOpt::Default;
240 case LLVMRustCodeGenOptLevel::Aggressive:
241 return CodeGenOpt::Aggressive;
243 report_fatal_error("Bad CodeGenOptLevel.");
247 enum class LLVMRustPassBuilderOptLevel {
256 #if LLVM_VERSION_LT(14,0)
257 using OptimizationLevel = PassBuilder::OptimizationLevel;
260 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
262 case LLVMRustPassBuilderOptLevel::O0:
263 return OptimizationLevel::O0;
264 case LLVMRustPassBuilderOptLevel::O1:
265 return OptimizationLevel::O1;
266 case LLVMRustPassBuilderOptLevel::O2:
267 return OptimizationLevel::O2;
268 case LLVMRustPassBuilderOptLevel::O3:
269 return OptimizationLevel::O3;
270 case LLVMRustPassBuilderOptLevel::Os:
271 return OptimizationLevel::Os;
272 case LLVMRustPassBuilderOptLevel::Oz:
273 return OptimizationLevel::Oz;
275 report_fatal_error("Bad PassBuilderOptLevel.");
279 enum class LLVMRustRelocModel {
288 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
290 case LLVMRustRelocModel::Static:
291 return Reloc::Static;
292 case LLVMRustRelocModel::PIC:
294 case LLVMRustRelocModel::DynamicNoPic:
295 return Reloc::DynamicNoPIC;
296 case LLVMRustRelocModel::ROPI:
298 case LLVMRustRelocModel::RWPI:
300 case LLVMRustRelocModel::ROPIRWPI:
301 return Reloc::ROPI_RWPI;
303 report_fatal_error("Bad RelocModel.");
307 /// getLongestEntryLength - Return the length of the longest entry in the table.
308 template<typename KV>
309 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
311 for (auto &I : Table)
312 MaxLen = std::max(MaxLen, std::strlen(I.Key));
316 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
317 const TargetMachine *Target = unwrap(TM);
318 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
319 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
320 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
321 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
322 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
324 printf("Available CPUs for this target:\n");
325 if (HostArch == TargetArch) {
326 const StringRef HostCPU = sys::getHostCPUName();
327 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
328 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
330 for (auto &CPU : CPUTable)
331 printf(" %-*s\n", MaxCPULen, CPU.Key);
335 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
336 const TargetMachine *Target = unwrap(TM);
337 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
338 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
339 return FeatTable.size();
342 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
343 const char** Feature, const char** Desc) {
344 const TargetMachine *Target = unwrap(TM);
345 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
346 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
347 const SubtargetFeatureKV Feat = FeatTable[Index];
354 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
355 printf("Target CPU help is not supported by this LLVM version.\n\n");
358 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
362 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
365 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
366 StringRef Name = sys::getHostCPUName();
371 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
372 const char *TripleStr, const char *CPU, const char *Feature,
373 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
374 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
375 bool FunctionSections,
377 bool UniqueSectionNames,
378 bool TrapUnreachable,
381 bool EmitStackSizeSection,
382 bool RelaxELFRelocations,
384 const char *SplitDwarfFile) {
386 auto OptLevel = fromRust(RustOptLevel);
387 auto RM = fromRust(RustReloc);
388 auto CM = fromRust(RustCM);
391 Triple Trip(Triple::normalize(TripleStr));
392 const llvm::Target *TheTarget =
393 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
394 if (TheTarget == nullptr) {
395 LLVMRustSetLastError(Error.c_str());
399 TargetOptions Options;
401 Options.FloatABIType = FloatABI::Default;
403 Options.FloatABIType = FloatABI::Soft;
405 Options.DataSections = DataSections;
406 Options.FunctionSections = FunctionSections;
407 Options.UniqueSectionNames = UniqueSectionNames;
408 Options.MCOptions.AsmVerbose = AsmComments;
409 Options.MCOptions.PreserveAsmComments = AsmComments;
410 Options.MCOptions.ABIName = ABIStr;
411 if (SplitDwarfFile) {
412 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
414 Options.RelaxELFRelocations = RelaxELFRelocations;
415 Options.UseInitArray = UseInitArray;
417 if (TrapUnreachable) {
418 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
419 // This limits the extent of possible undefined behavior in some cases, as
420 // it prevents control flow from "falling through" into whatever code
421 // happens to be laid out next in memory.
422 Options.TrapUnreachable = true;
426 Options.ThreadModel = ThreadModel::Single;
429 Options.EmitStackSizeSection = EmitStackSizeSection;
431 TargetMachine *TM = TheTarget->createTargetMachine(
432 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
436 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
440 // Unfortunately, the LLVM C API doesn't provide a way to create the
441 // TargetLibraryInfo pass, so we use this method to do so.
442 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
443 bool DisableSimplifyLibCalls) {
444 Triple TargetTriple(unwrap(M)->getTargetTriple());
445 TargetLibraryInfoImpl TLII(TargetTriple);
446 if (DisableSimplifyLibCalls)
447 TLII.disableAllFunctions();
448 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
451 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
452 // Initializing the command-line options more than once is not allowed. So,
453 // check if they've already been initialized. (This could happen if we're
454 // being called from rustpkg, for example). If the arguments change, then
455 // that's just kinda unfortunate.
456 static bool Initialized = false;
460 cl::ParseCommandLineOptions(Argc, Argv);
463 enum class LLVMRustFileType {
468 static CodeGenFileType fromRust(LLVMRustFileType Type) {
470 case LLVMRustFileType::AssemblyFile:
471 return CGFT_AssemblyFile;
472 case LLVMRustFileType::ObjectFile:
473 return CGFT_ObjectFile;
475 report_fatal_error("Bad FileType.");
479 extern "C" LLVMRustResult
480 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
481 LLVMModuleRef M, const char *Path, const char *DwoPath,
482 LLVMRustFileType RustFileType) {
483 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
484 auto FileType = fromRust(RustFileType);
486 std::string ErrorInfo;
488 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
490 ErrorInfo = EC.message();
491 if (ErrorInfo != "") {
492 LLVMRustSetLastError(ErrorInfo.c_str());
493 return LLVMRustResult::Failure;
496 buffer_ostream BOS(OS);
498 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
501 ErrorInfo = EC.message();
502 if (ErrorInfo != "") {
503 LLVMRustSetLastError(ErrorInfo.c_str());
504 return LLVMRustResult::Failure;
506 buffer_ostream DBOS(DOS);
507 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
510 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
514 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
515 // stream (OS), so the only real safe place to delete this is here? Don't we
516 // wish this was written in Rust?
517 LLVMDisposePassManager(PMR);
518 return LLVMRustResult::Success;
521 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
522 const char*, // pass name
523 const char*); // IR name
524 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
526 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
527 if (any_isa<const Module *>(WrappedIr))
528 return any_cast<const Module *>(WrappedIr)->getName().str();
529 if (any_isa<const Function *>(WrappedIr))
530 return any_cast<const Function *>(WrappedIr)->getName().str();
531 if (any_isa<const Loop *>(WrappedIr))
532 return any_cast<const Loop *>(WrappedIr)->getName().str();
533 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
534 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
539 void LLVMSelfProfileInitializeCallbacks(
540 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
541 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
542 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
543 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
544 StringRef Pass, llvm::Any Ir) {
545 std::string PassName = Pass.str();
546 std::string IrName = LLVMRustwrappedIrGetName(Ir);
547 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
550 PIC.registerAfterPassCallback(
551 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
552 const PreservedAnalyses &Preserved) {
553 AfterPassCallback(LlvmSelfProfiler);
556 PIC.registerAfterPassInvalidatedCallback(
557 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
558 AfterPassCallback(LlvmSelfProfiler);
561 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
562 StringRef Pass, llvm::Any Ir) {
563 std::string PassName = Pass.str();
564 std::string IrName = LLVMRustwrappedIrGetName(Ir);
565 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
568 PIC.registerAfterAnalysisCallback(
569 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
570 AfterPassCallback(LlvmSelfProfiler);
574 enum class LLVMRustOptStage {
582 struct LLVMRustSanitizerOptions {
583 bool SanitizeAddress;
584 bool SanitizeAddressRecover;
586 bool SanitizeMemoryRecover;
587 int SanitizeMemoryTrackOrigins;
589 bool SanitizeHWAddress;
590 bool SanitizeHWAddressRecover;
593 extern "C" LLVMRustResult
595 LLVMModuleRef ModuleRef,
596 LLVMTargetMachineRef TMRef,
597 LLVMRustPassBuilderOptLevel OptLevelRust,
598 LLVMRustOptStage OptStage,
599 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
600 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
601 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
602 LLVMRustSanitizerOptions *SanitizerOptions,
603 const char *PGOGenPath, const char *PGOUsePath,
604 bool InstrumentCoverage, const char *InstrProfileOutput,
606 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
607 void* LlvmSelfProfiler,
608 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
609 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
610 const char *ExtraPasses, size_t ExtraPassesLen,
611 const char *LLVMPlugins, size_t LLVMPluginsLen) {
612 Module *TheModule = unwrap(ModuleRef);
613 TargetMachine *TM = unwrap(TMRef);
614 OptimizationLevel OptLevel = fromRust(OptLevelRust);
617 PipelineTuningOptions PTO;
618 PTO.LoopUnrolling = UnrollLoops;
619 PTO.LoopInterleaving = UnrollLoops;
620 PTO.LoopVectorization = LoopVectorize;
621 PTO.SLPVectorization = SLPVectorize;
622 PTO.MergeFunctions = MergeFunctions;
624 // FIXME: We may want to expose this as an option.
625 bool DebugPassManager = false;
627 PassInstrumentationCallbacks PIC;
628 StandardInstrumentations SI(DebugPassManager);
629 SI.registerCallbacks(PIC);
631 if (LlvmSelfProfiler){
632 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
635 Optional<PGOOptions> PGOOpt;
637 assert(!PGOUsePath && !PGOSampleUsePath);
638 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
639 PGOOptions::NoCSAction, DebugInfoForProfiling);
640 } else if (PGOUsePath) {
641 assert(!PGOSampleUsePath);
642 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
643 PGOOptions::NoCSAction, DebugInfoForProfiling);
644 } else if (PGOSampleUsePath) {
645 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
646 PGOOptions::NoCSAction, DebugInfoForProfiling);
647 } else if (DebugInfoForProfiling) {
648 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
649 PGOOptions::NoCSAction, DebugInfoForProfiling);
652 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
653 LoopAnalysisManager LAM;
654 FunctionAnalysisManager FAM;
655 CGSCCAnalysisManager CGAM;
656 ModuleAnalysisManager MAM;
658 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
660 Triple TargetTriple(TheModule->getTargetTriple());
661 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
662 if (DisableSimplifyLibCalls)
663 TLII->disableAllFunctions();
664 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
666 PB.registerModuleAnalyses(MAM);
667 PB.registerCGSCCAnalyses(CGAM);
668 PB.registerFunctionAnalyses(FAM);
669 PB.registerLoopAnalyses(LAM);
670 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
672 // We manually collect pipeline callbacks so we can apply them at O0, where the
673 // PassBuilder does not create a pipeline.
674 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
675 PipelineStartEPCallbacks;
676 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
677 OptimizerLastEPCallbacks;
680 PipelineStartEPCallbacks.push_back(
681 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
682 MPM.addPass(VerifierPass());
687 if (InstrumentGCOV) {
688 PipelineStartEPCallbacks.push_back(
689 [](ModulePassManager &MPM, OptimizationLevel Level) {
690 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
695 if (InstrumentCoverage) {
696 PipelineStartEPCallbacks.push_back(
697 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
698 InstrProfOptions Options;
699 if (InstrProfileOutput) {
700 Options.InstrProfileOutput = InstrProfileOutput;
702 MPM.addPass(InstrProfiling(Options, false));
707 if (SanitizerOptions) {
708 if (SanitizerOptions->SanitizeMemory) {
709 #if LLVM_VERSION_GE(14, 0)
710 MemorySanitizerOptions Options(
711 SanitizerOptions->SanitizeMemoryTrackOrigins,
712 SanitizerOptions->SanitizeMemoryRecover,
713 /*CompileKernel=*/false,
714 /*EagerChecks=*/true);
716 MemorySanitizerOptions Options(
717 SanitizerOptions->SanitizeMemoryTrackOrigins,
718 SanitizerOptions->SanitizeMemoryRecover,
719 /*CompileKernel=*/false);
721 OptimizerLastEPCallbacks.push_back(
722 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
723 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
724 MPM.addPass(ModuleMemorySanitizerPass(Options));
726 MPM.addPass(MemorySanitizerPass(Options));
728 #if LLVM_VERSION_LT(16, 0)
729 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
735 if (SanitizerOptions->SanitizeThread) {
736 OptimizerLastEPCallbacks.push_back(
737 [](ModulePassManager &MPM, OptimizationLevel Level) {
738 #if LLVM_VERSION_GE(14, 0)
739 MPM.addPass(ModuleThreadSanitizerPass());
741 MPM.addPass(ThreadSanitizerPass());
743 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
748 if (SanitizerOptions->SanitizeAddress) {
749 OptimizerLastEPCallbacks.push_back(
750 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
751 #if LLVM_VERSION_LT(15, 0)
752 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
754 #if LLVM_VERSION_GE(14, 0)
755 AddressSanitizerOptions opts = AddressSanitizerOptions{
756 /*CompileKernel=*/false,
757 SanitizerOptions->SanitizeAddressRecover,
758 /*UseAfterScope=*/true,
759 AsanDetectStackUseAfterReturnMode::Runtime,
761 #if LLVM_VERSION_LT(16, 0)
762 MPM.addPass(ModuleAddressSanitizerPass(opts));
764 MPM.addPass(AddressSanitizerPass(opts));
767 MPM.addPass(ModuleAddressSanitizerPass(
768 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
769 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
770 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
771 /*UseAfterScope=*/true)));
776 if (SanitizerOptions->SanitizeHWAddress) {
777 OptimizerLastEPCallbacks.push_back(
778 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
779 #if LLVM_VERSION_GE(14, 0)
780 HWAddressSanitizerOptions opts(
781 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
782 /*DisableOptimization=*/false);
783 MPM.addPass(HWAddressSanitizerPass(opts));
785 MPM.addPass(HWAddressSanitizerPass(
786 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
793 if (LLVMPluginsLen) {
794 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
795 SmallVector<StringRef> Plugins;
796 PluginsStr.split(Plugins, ',', -1, false);
797 for (auto PluginPath: Plugins) {
798 auto Plugin = PassPlugin::Load(PluginPath.str());
800 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
803 Plugin->registerPassBuilderCallbacks(PB);
807 ModulePassManager MPM;
808 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
809 if (!NoPrepopulatePasses) {
810 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
811 // At the same time, the LTO pipelines do support O0 and using them is required.
812 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
813 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
814 for (const auto &C : PipelineStartEPCallbacks)
815 PB.registerPipelineStartEPCallback(C);
816 for (const auto &C : OptimizerLastEPCallbacks)
817 PB.registerOptimizerLastEPCallback(C);
819 // Pass false as we manually schedule ThinLTOBufferPasses below.
820 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
822 for (const auto &C : PipelineStartEPCallbacks)
823 PB.registerPipelineStartEPCallback(C);
824 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
825 for (const auto &C : OptimizerLastEPCallbacks)
826 PB.registerOptimizerLastEPCallback(C);
830 case LLVMRustOptStage::PreLinkNoLTO:
831 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
833 case LLVMRustOptStage::PreLinkThinLTO:
834 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
835 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
836 // passes may still run afterwards. This means we need to run the buffer passes again.
837 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
838 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
839 if (OptimizerLastEPCallbacks.empty())
840 NeedThinLTOBufferPasses = false;
841 for (const auto &C : OptimizerLastEPCallbacks)
844 case LLVMRustOptStage::PreLinkFatLTO:
845 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
846 NeedThinLTOBufferPasses = false;
848 case LLVMRustOptStage::ThinLTO:
849 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
850 // It only seems to be needed for C++ specific optimizations.
851 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
853 case LLVMRustOptStage::FatLTO:
854 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
859 // We're not building any of the default pipelines but we still want to
860 // add the verifier, instrumentation, etc passes if they were requested
861 for (const auto &C : PipelineStartEPCallbacks)
863 for (const auto &C : OptimizerLastEPCallbacks)
867 if (ExtraPassesLen) {
868 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
869 std::string ErrMsg = toString(std::move(Err));
870 LLVMRustSetLastError(ErrMsg.c_str());
871 return LLVMRustResult::Failure;
875 if (NeedThinLTOBufferPasses) {
876 MPM.addPass(CanonicalizeAliasesPass());
877 MPM.addPass(NameAnonGlobalPass());
880 // Upgrade all calls to old intrinsics first.
881 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
882 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
884 MPM.run(*TheModule, MAM);
885 return LLVMRustResult::Success;
888 // Callback to demangle function name
890 // * name to be demangled
893 // * output buffer len
894 // Returns len of demangled string, or 0 if demangle failed.
895 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
900 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
902 std::vector<char> Buf;
905 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
907 // Return empty string if demangle failed
908 // or if name does not need to be demangled
909 StringRef CallDemangle(StringRef name) {
914 if (Buf.size() < name.size() * 2) {
915 // Semangled name usually shorter than mangled,
916 // but allocate twice as much memory just in case
917 Buf.resize(name.size() * 2);
920 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
926 auto Demangled = StringRef(Buf.data(), R);
927 if (Demangled == name) {
928 // Do not print anything if demangled name is equal to mangled.
935 void emitFunctionAnnot(const Function *F,
936 formatted_raw_ostream &OS) override {
937 StringRef Demangled = CallDemangle(F->getName());
938 if (Demangled.empty()) {
942 OS << "; " << Demangled << "\n";
945 void emitInstructionAnnot(const Instruction *I,
946 formatted_raw_ostream &OS) override {
949 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
951 Value = CI->getCalledOperand();
952 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
954 Value = II->getCalledOperand();
956 // Could demangle more operations, e. g.
957 // `store %place, @function`.
961 if (!Value->hasName()) {
965 StringRef Demangled = CallDemangle(Value->getName());
966 if (Demangled.empty()) {
970 OS << "; " << Name << " " << Demangled << "\n";
976 extern "C" LLVMRustResult
977 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
978 std::string ErrorInfo;
980 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
982 ErrorInfo = EC.message();
983 if (ErrorInfo != "") {
984 LLVMRustSetLastError(ErrorInfo.c_str());
985 return LLVMRustResult::Failure;
988 RustAssemblyAnnotationWriter AAW(Demangle);
989 formatted_raw_ostream FOS(OS);
990 unwrap(M)->print(FOS, &AAW);
992 return LLVMRustResult::Success;
995 extern "C" void LLVMRustPrintPasses() {
996 LLVMInitializePasses();
997 struct MyListener : PassRegistrationListener {
998 void passEnumerate(const PassInfo *Info) {
999 StringRef PassArg = Info->getPassArgument();
1000 StringRef PassName = Info->getPassName();
1001 if (!PassArg.empty()) {
1002 // These unsigned->signed casts could theoretically overflow, but
1003 // realistically never will (and even if, the result is implementation
1004 // defined rather plain UB).
1005 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1006 (int)PassName.size(), PassName.data());
1011 PassRegistry *PR = PassRegistry::getPassRegistry();
1012 PR->enumerateWith(&Listener);
1015 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1017 auto PreserveFunctions = [=](const GlobalValue &GV) {
1018 for (size_t I = 0; I < Len; I++) {
1019 if (GV.getName() == Symbols[I]) {
1026 internalizeModule(*unwrap(M), PreserveFunctions);
1030 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1031 LLVMTargetMachineRef TMR) {
1032 TargetMachine *Target = unwrap(TMR);
1033 unwrap(Module)->setDataLayout(Target->createDataLayout());
1036 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1037 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1040 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1041 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1044 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1045 LLVMRustCodeModel Model) {
1046 auto CM = fromRust(Model);
1049 unwrap(M)->setCodeModel(*CM);
1052 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1053 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1054 // LLVM, and otherwise it's just blanket rejected from other compilers.
1056 // Most of this implementation is straight copied from LLVM. At the time of
1057 // this writing it wasn't *quite* suitable to reuse more code from upstream
1058 // for our purposes, but we should strive to upstream this support once it's
1059 // ready to go! I figure we may want a bit of testing locally first before
1060 // sending this upstream to LLVM. I hear though they're quite eager to receive
1061 // feedback like this!
1063 // If you're reading this code and wondering "what in the world" or you're
1064 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1065 // then fear not! (ok maybe fear a little). All code here is mostly based
1066 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1068 // You'll find that the general layout here roughly corresponds to the `run`
1069 // method in that file as well as `ProcessThinLTOModule`. Functions are
1070 // specifically commented below as well, but if you're updating this code
1071 // or otherwise trying to understand it, the LLVM source will be useful in
1072 // interpreting the mysteries within.
1074 // Otherwise I'll apologize in advance, it probably requires a relatively
1075 // significant investment on your part to "truly understand" what's going on
1076 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1077 // and various online resources about ThinLTO to make heads or tails of all
1080 // This is a shared data structure which *must* be threadsafe to share
1081 // read-only amongst threads. This also corresponds basically to the arguments
1082 // of the `ProcessThinLTOModule` function in the LLVM source.
1083 struct LLVMRustThinLTOData {
1084 // The combined index that is the global analysis over all modules we're
1085 // performing ThinLTO for. This is mostly managed by LLVM.
1086 ModuleSummaryIndex Index;
1088 // All modules we may look at, stored as in-memory serialized versions. This
1089 // is later used when inlining to ensure we can extract any module to inline
1091 StringMap<MemoryBufferRef> ModuleMap;
1093 // A set that we manage of everything we *don't* want internalized. Note that
1094 // this includes all transitive references right now as well, but it may not
1096 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1098 // Not 100% sure what these are, but they impact what's internalized and
1099 // what's inlined across modules, I believe.
1100 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1101 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1102 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1103 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1105 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1108 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1109 struct LLVMRustThinLTOModule {
1110 const char *identifier;
1115 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1117 static const GlobalValueSummary *
1118 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1119 auto StrongDefForLinker = llvm::find_if(
1120 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1121 auto Linkage = Summary->linkage();
1122 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1123 !GlobalValue::isWeakForLinker(Linkage);
1125 if (StrongDefForLinker != GVSummaryList.end())
1126 return StrongDefForLinker->get();
1128 auto FirstDefForLinker = llvm::find_if(
1129 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1130 auto Linkage = Summary->linkage();
1131 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1133 if (FirstDefForLinker == GVSummaryList.end())
1135 return FirstDefForLinker->get();
1138 // The main entry point for creating the global ThinLTO analysis. The structure
1139 // here is basically the same as before threads are spawned in the `run`
1140 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1141 extern "C" LLVMRustThinLTOData*
1142 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1144 const char **preserved_symbols,
1146 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1148 // Load each module's summary and merge it into one combined index
1149 for (int i = 0; i < num_modules; i++) {
1150 auto module = &modules[i];
1151 StringRef buffer(module->data, module->len);
1152 MemoryBufferRef mem_buffer(buffer, module->identifier);
1154 Ret->ModuleMap[module->identifier] = mem_buffer;
1156 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1157 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1162 // Collect for each module the list of function it defines (GUID -> Summary)
1163 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1165 // Convert the preserved symbols set from string to GUID, this is then needed
1166 // for internalization.
1167 for (int i = 0; i < num_symbols; i++) {
1168 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1169 Ret->GUIDPreservedSymbols.insert(GUID);
1172 // Collect the import/export lists for all modules from the call-graph in the
1175 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1176 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1177 return PrevailingType::Unknown;
1179 // We don't have a complete picture in our use of ThinLTO, just our immediate
1180 // crate, so we need `ImportEnabled = false` to limit internalization.
1181 // Otherwise, we sometimes lose `static` values -- see #60184.
1182 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1183 deadIsPrevailing, /* ImportEnabled = */ false);
1184 ComputeCrossModuleImport(
1186 Ret->ModuleToDefinedGVSummaries,
1191 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1192 // impacts the caching.
1194 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1195 // being lifted from `lib/LTO/LTO.cpp` as well
1196 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1197 for (auto &I : Ret->Index) {
1198 if (I.second.SummaryList.size() > 1)
1199 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1201 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1202 const auto &Prevailing = PrevailingCopy.find(GUID);
1203 if (Prevailing == PrevailingCopy.end())
1205 return Prevailing->second == S;
1207 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1208 GlobalValue::GUID GUID,
1209 GlobalValue::LinkageTypes NewLinkage) {
1210 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1213 // Uses FromPrevailing visibility scheme which works for many binary
1214 // formats. We probably could and should use ELF visibility scheme for many of
1215 // our targets, however.
1217 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1218 Ret->GUIDPreservedSymbols);
1220 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1221 // callback below. This callback below will dictate the linkage for all
1222 // summaries in the index, and we basically just only want to ensure that dead
1223 // symbols are internalized. Otherwise everything that's already external
1224 // linkage will stay as external, and internal will stay as internal.
1225 std::set<GlobalValue::GUID> ExportedGUIDs;
1226 for (auto &List : Ret->Index) {
1227 for (auto &GVS: List.second.SummaryList) {
1228 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1230 auto GUID = GVS->getOriginalName();
1231 if (GVS->flags().Live)
1232 ExportedGUIDs.insert(GUID);
1235 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1236 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1237 return (ExportList != Ret->ExportLists.end() &&
1238 ExportList->second.count(VI)) ||
1239 ExportedGUIDs.count(VI.getGUID());
1241 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1243 return Ret.release();
1247 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1251 // Below are the various passes that happen *per module* when doing ThinLTO.
1253 // In other words, these are the functions that are all run concurrently
1254 // with one another, one per module. The passes here correspond to the analysis
1255 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1256 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1257 // so rustc can save off the intermediate bytecode between each step.
1260 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1261 // When linking an ELF shared object, dso_local should be dropped. We
1262 // conservatively do this for -fpic.
1263 bool ClearDSOLocalOnDeclarations =
1264 TM.getTargetTriple().isOSBinFormatELF() &&
1265 TM.getRelocationModel() != Reloc::Static &&
1266 Mod.getPIELevel() == PIELevel::Default;
1267 return ClearDSOLocalOnDeclarations;
1271 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1272 LLVMTargetMachineRef TM) {
1273 Module &Mod = *unwrap(M);
1274 TargetMachine &Target = *unwrap(TM);
1276 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1277 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1280 LLVMRustSetLastError("renameModuleForThinLTO failed");
1287 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1288 Module &Mod = *unwrap(M);
1289 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1290 #if LLVM_VERSION_GE(14, 0)
1291 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1293 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1299 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1300 Module &Mod = *unwrap(M);
1301 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1302 thinLTOInternalizeModule(Mod, DefinedGlobals);
1307 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1308 LLVMTargetMachineRef TM) {
1309 Module &Mod = *unwrap(M);
1310 TargetMachine &Target = *unwrap(TM);
1312 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1313 auto Loader = [&](StringRef Identifier) {
1314 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1315 auto &Context = Mod.getContext();
1316 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1321 // The rest of this closure is a workaround for
1322 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1323 // we accidentally import wasm custom sections into different modules,
1324 // duplicating them by in the final output artifact.
1326 // The issue is worked around here by manually removing the
1327 // `wasm.custom_sections` named metadata node from any imported module. This
1328 // we know isn't used by any optimization pass so there's no need for it to
1331 // Note that the metadata is currently lazily loaded, so we materialize it
1332 // here before looking up if there's metadata inside. The `FunctionImporter`
1333 // will immediately materialize metadata anyway after an import, so this
1334 // shouldn't be a perf hit.
1335 if (Error Err = (*MOrErr)->materializeMetadata()) {
1336 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1340 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1341 if (WasmCustomSections)
1342 WasmCustomSections->eraseFromParent();
1346 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1347 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1348 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1350 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1356 // This struct and various functions are sort of a hack right now, but the
1357 // problem is that we've got in-memory LLVM modules after we generate and
1358 // optimize all codegen-units for one compilation in rustc. To be compatible
1359 // with the LTO support above we need to serialize the modules plus their
1360 // ThinLTO summary into memory.
1362 // This structure is basically an owned version of a serialize module, with
1363 // a ThinLTO summary attached.
1364 struct LLVMRustThinLTOBuffer {
1368 extern "C" LLVMRustThinLTOBuffer*
1369 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1370 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1372 raw_string_ostream OS(Ret->data);
1376 LoopAnalysisManager LAM;
1377 FunctionAnalysisManager FAM;
1378 CGSCCAnalysisManager CGAM;
1379 ModuleAnalysisManager MAM;
1380 PB.registerModuleAnalyses(MAM);
1381 PB.registerCGSCCAnalyses(CGAM);
1382 PB.registerFunctionAnalyses(FAM);
1383 PB.registerLoopAnalyses(LAM);
1384 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1385 ModulePassManager MPM;
1386 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1387 MPM.run(*unwrap(M), MAM);
1389 WriteBitcodeToFile(*unwrap(M), OS);
1393 return Ret.release();
1397 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1401 extern "C" const void*
1402 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1403 return Buffer->data.data();
1407 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1408 return Buffer->data.length();
1411 // This is what we used to parse upstream bitcode for actual ThinLTO
1412 // processing. We'll call this once per module optimized through ThinLTO, and
1413 // it'll be called concurrently on many threads.
1414 extern "C" LLVMModuleRef
1415 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1418 const char *identifier) {
1419 StringRef Data(data, len);
1420 MemoryBufferRef Buffer(Data, identifier);
1421 unwrap(Context)->enableDebugTypeODRUniquing();
1422 Expected<std::unique_ptr<Module>> SrcOrError =
1423 parseBitcodeFile(Buffer, *unwrap(Context));
1425 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1428 return wrap(std::move(*SrcOrError).release());
1431 // Find the bitcode section in the object file data and return it as a slice.
1432 // Fail if the bitcode section is present but empty.
1434 // On success, the return value is the pointer to the start of the slice and
1435 // `out_len` is filled with the (non-zero) length. On failure, the return value
1436 // is `nullptr` and `out_len` is set to zero.
1437 extern "C" const char*
1438 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1443 StringRef Data(data, len);
1444 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1446 Expected<MemoryBufferRef> BitcodeOrError =
1447 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1448 if (!BitcodeOrError) {
1449 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1453 *out_len = BitcodeOrError->getBufferSize();
1454 return BitcodeOrError->getBufferStart();
1457 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1458 // the comment in `back/lto.rs` for why this exists.
1460 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1462 DICompileUnit **B) {
1463 Module *M = unwrap(Mod);
1464 DICompileUnit **Cur = A;
1465 DICompileUnit **Next = B;
1466 for (DICompileUnit *CU : M->debug_compile_units()) {
1475 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1476 // the comment in `back/lto.rs` for why this exists.
1478 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1479 Module *M = unwrap(Mod);
1481 // If the original source module didn't have a `DICompileUnit` then try to
1482 // merge all the existing compile units. If there aren't actually any though
1483 // then there's not much for us to do so return.
1484 if (Unit == nullptr) {
1485 for (DICompileUnit *CU : M->debug_compile_units()) {
1489 if (Unit == nullptr)
1493 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1494 // process it recursively. Note that we used to specifically iterate over
1495 // instructions to ensure we feed everything into it, but `processModule`
1496 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1497 DebugInfoFinder Finder;
1498 Finder.processModule(*M);
1500 // After we've found all our debuginfo, rewrite all subprograms to point to
1501 // the same `DICompileUnit`.
1502 for (auto &F : Finder.subprograms()) {
1503 F->replaceUnit(Unit);
1506 // Erase any other references to other `DICompileUnit` instances, the verifier
1507 // will later ensure that we don't actually have any other stale references to
1509 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1510 MD->clearOperands();
1511 MD->addOperand(Unit);
1514 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1515 // storing the result in 'KeyOut'.
1516 // Currently, this cache key is a SHA-1 hash of anything that could affect
1517 // the result of optimizing this module (e.g. module imports, exports, liveness
1518 // of access globals, etc).
1519 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1520 // used during the normal linker-plugin incremental thin-LTO process.
1522 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1523 SmallString<40> Key;
1524 llvm::lto::Config conf;
1525 const auto &ImportList = Data->ImportLists.lookup(ModId);
1526 const auto &ExportList = Data->ExportLists.lookup(ModId);
1527 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1528 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1529 std::set<GlobalValue::GUID> CfiFunctionDefs;
1530 std::set<GlobalValue::GUID> CfiFunctionDecls;
1532 // Based on the 'InProcessThinBackend' constructor in LLVM
1533 for (auto &Name : Data->Index.cfiFunctionDefs())
1534 CfiFunctionDefs.insert(
1535 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1536 for (auto &Name : Data->Index.cfiFunctionDecls())
1537 CfiFunctionDecls.insert(
1538 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1540 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1541 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1544 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());