6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/IPO/Internalize.h"
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36 #include "llvm/Transforms/Utils/AddDiscriminators.h"
37 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
38 #include "llvm/LTO/LTO.h"
39 #include "llvm/Bitcode/BitcodeWriter.h"
40 #include "llvm-c/Transforms/PassManagerBuilder.h"
42 #include "llvm/Transforms/Instrumentation.h"
43 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
44 #include "llvm/Support/TimeProfiler.h"
45 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
46 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
47 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
48 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
49 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
50 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
51 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
52 #include "llvm/Transforms/Utils.h"
56 typedef struct LLVMOpaquePass *LLVMPassRef;
57 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
59 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
62 extern "C" void LLVMInitializePasses() {
63 PassRegistry &Registry = *PassRegistry::getPassRegistry();
64 initializeCore(Registry);
65 initializeCodeGen(Registry);
66 initializeScalarOpts(Registry);
67 initializeVectorization(Registry);
68 initializeIPO(Registry);
69 initializeAnalysis(Registry);
70 initializeTransformUtils(Registry);
71 initializeInstCombine(Registry);
72 #if LLVM_VERSION_LT(16, 0)
73 initializeInstrumentation(Registry);
75 initializeTarget(Registry);
78 extern "C" void LLVMTimeTraceProfilerInitialize() {
79 timeTraceProfilerInitialize(
80 /* TimeTraceGranularity */ 0,
81 /* ProcName */ "rustc");
84 extern "C" void LLVMTimeTraceProfilerFinishThread() {
85 timeTraceProfilerFinishThread();
88 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
89 StringRef FN(FileName);
91 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
93 timeTraceProfilerWrite(OS);
94 timeTraceProfilerCleanup();
97 #ifdef LLVM_COMPONENT_X86
98 #define SUBTARGET_X86 SUBTARGET(X86)
100 #define SUBTARGET_X86
103 #ifdef LLVM_COMPONENT_ARM
104 #define SUBTARGET_ARM SUBTARGET(ARM)
106 #define SUBTARGET_ARM
109 #ifdef LLVM_COMPONENT_AARCH64
110 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
112 #define SUBTARGET_AARCH64
115 #ifdef LLVM_COMPONENT_AVR
116 #define SUBTARGET_AVR SUBTARGET(AVR)
118 #define SUBTARGET_AVR
121 #ifdef LLVM_COMPONENT_M68k
122 #define SUBTARGET_M68K SUBTARGET(M68k)
124 #define SUBTARGET_M68K
127 #ifdef LLVM_COMPONENT_MIPS
128 #define SUBTARGET_MIPS SUBTARGET(Mips)
130 #define SUBTARGET_MIPS
133 #ifdef LLVM_COMPONENT_POWERPC
134 #define SUBTARGET_PPC SUBTARGET(PPC)
136 #define SUBTARGET_PPC
139 #ifdef LLVM_COMPONENT_SYSTEMZ
140 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
142 #define SUBTARGET_SYSTEMZ
145 #ifdef LLVM_COMPONENT_MSP430
146 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
148 #define SUBTARGET_MSP430
151 #ifdef LLVM_COMPONENT_RISCV
152 #define SUBTARGET_RISCV SUBTARGET(RISCV)
154 #define SUBTARGET_RISCV
157 #ifdef LLVM_COMPONENT_SPARC
158 #define SUBTARGET_SPARC SUBTARGET(Sparc)
160 #define SUBTARGET_SPARC
163 #ifdef LLVM_COMPONENT_HEXAGON
164 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
166 #define SUBTARGET_HEXAGON
169 #define GEN_SUBTARGETS \
183 #define SUBTARGET(x) \
185 extern const SubtargetFeatureKV x##FeatureKV[]; \
186 extern const SubtargetFeatureKV x##SubTypeKV[]; \
192 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
193 const char *Feature) {
194 TargetMachine *Target = unwrap(TM);
195 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
196 return MCInfo->checkFeatures(std::string("+") + Feature);
199 enum class LLVMRustCodeModel {
208 #if LLVM_VERSION_LT(16, 0)
209 static Optional<CodeModel::Model>
211 static std::optional<CodeModel::Model>
213 fromRust(LLVMRustCodeModel Model) {
215 case LLVMRustCodeModel::Tiny:
216 return CodeModel::Tiny;
217 case LLVMRustCodeModel::Small:
218 return CodeModel::Small;
219 case LLVMRustCodeModel::Kernel:
220 return CodeModel::Kernel;
221 case LLVMRustCodeModel::Medium:
222 return CodeModel::Medium;
223 case LLVMRustCodeModel::Large:
224 return CodeModel::Large;
225 case LLVMRustCodeModel::None:
226 #if LLVM_VERSION_LT(16, 0)
232 report_fatal_error("Bad CodeModel.");
236 enum class LLVMRustCodeGenOptLevel {
243 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
245 case LLVMRustCodeGenOptLevel::None:
246 return CodeGenOpt::None;
247 case LLVMRustCodeGenOptLevel::Less:
248 return CodeGenOpt::Less;
249 case LLVMRustCodeGenOptLevel::Default:
250 return CodeGenOpt::Default;
251 case LLVMRustCodeGenOptLevel::Aggressive:
252 return CodeGenOpt::Aggressive;
254 report_fatal_error("Bad CodeGenOptLevel.");
258 enum class LLVMRustPassBuilderOptLevel {
267 #if LLVM_VERSION_LT(14,0)
268 using OptimizationLevel = PassBuilder::OptimizationLevel;
271 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
273 case LLVMRustPassBuilderOptLevel::O0:
274 return OptimizationLevel::O0;
275 case LLVMRustPassBuilderOptLevel::O1:
276 return OptimizationLevel::O1;
277 case LLVMRustPassBuilderOptLevel::O2:
278 return OptimizationLevel::O2;
279 case LLVMRustPassBuilderOptLevel::O3:
280 return OptimizationLevel::O3;
281 case LLVMRustPassBuilderOptLevel::Os:
282 return OptimizationLevel::Os;
283 case LLVMRustPassBuilderOptLevel::Oz:
284 return OptimizationLevel::Oz;
286 report_fatal_error("Bad PassBuilderOptLevel.");
290 enum class LLVMRustRelocModel {
299 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
301 case LLVMRustRelocModel::Static:
302 return Reloc::Static;
303 case LLVMRustRelocModel::PIC:
305 case LLVMRustRelocModel::DynamicNoPic:
306 return Reloc::DynamicNoPIC;
307 case LLVMRustRelocModel::ROPI:
309 case LLVMRustRelocModel::RWPI:
311 case LLVMRustRelocModel::ROPIRWPI:
312 return Reloc::ROPI_RWPI;
314 report_fatal_error("Bad RelocModel.");
318 /// getLongestEntryLength - Return the length of the longest entry in the table.
319 template<typename KV>
320 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
322 for (auto &I : Table)
323 MaxLen = std::max(MaxLen, std::strlen(I.Key));
327 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
328 const TargetMachine *Target = unwrap(TM);
329 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
330 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
331 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
332 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
333 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
335 printf("Available CPUs for this target:\n");
336 if (HostArch == TargetArch) {
337 const StringRef HostCPU = sys::getHostCPUName();
338 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
339 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
341 for (auto &CPU : CPUTable)
342 printf(" %-*s\n", MaxCPULen, CPU.Key);
346 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
347 const TargetMachine *Target = unwrap(TM);
348 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
349 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
350 return FeatTable.size();
353 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
354 const char** Feature, const char** Desc) {
355 const TargetMachine *Target = unwrap(TM);
356 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
357 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
358 const SubtargetFeatureKV Feat = FeatTable[Index];
365 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
366 printf("Target CPU help is not supported by this LLVM version.\n\n");
369 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
373 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
376 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
377 StringRef Name = sys::getHostCPUName();
382 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
383 const char *TripleStr, const char *CPU, const char *Feature,
384 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
385 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
386 bool FunctionSections,
388 bool UniqueSectionNames,
389 bool TrapUnreachable,
392 bool EmitStackSizeSection,
393 bool RelaxELFRelocations,
395 const char *SplitDwarfFile) {
397 auto OptLevel = fromRust(RustOptLevel);
398 auto RM = fromRust(RustReloc);
399 auto CM = fromRust(RustCM);
402 Triple Trip(Triple::normalize(TripleStr));
403 const llvm::Target *TheTarget =
404 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
405 if (TheTarget == nullptr) {
406 LLVMRustSetLastError(Error.c_str());
410 TargetOptions Options;
412 Options.FloatABIType = FloatABI::Default;
414 Options.FloatABIType = FloatABI::Soft;
416 Options.DataSections = DataSections;
417 Options.FunctionSections = FunctionSections;
418 Options.UniqueSectionNames = UniqueSectionNames;
419 Options.MCOptions.AsmVerbose = AsmComments;
420 Options.MCOptions.PreserveAsmComments = AsmComments;
421 Options.MCOptions.ABIName = ABIStr;
422 if (SplitDwarfFile) {
423 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
425 Options.RelaxELFRelocations = RelaxELFRelocations;
426 Options.UseInitArray = UseInitArray;
428 if (TrapUnreachable) {
429 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
430 // This limits the extent of possible undefined behavior in some cases, as
431 // it prevents control flow from "falling through" into whatever code
432 // happens to be laid out next in memory.
433 Options.TrapUnreachable = true;
437 Options.ThreadModel = ThreadModel::Single;
440 Options.EmitStackSizeSection = EmitStackSizeSection;
442 TargetMachine *TM = TheTarget->createTargetMachine(
443 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
447 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
451 // Unfortunately, the LLVM C API doesn't provide a way to create the
452 // TargetLibraryInfo pass, so we use this method to do so.
453 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
454 bool DisableSimplifyLibCalls) {
455 Triple TargetTriple(unwrap(M)->getTargetTriple());
456 TargetLibraryInfoImpl TLII(TargetTriple);
457 if (DisableSimplifyLibCalls)
458 TLII.disableAllFunctions();
459 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
462 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
463 // Initializing the command-line options more than once is not allowed. So,
464 // check if they've already been initialized. (This could happen if we're
465 // being called from rustpkg, for example). If the arguments change, then
466 // that's just kinda unfortunate.
467 static bool Initialized = false;
471 cl::ParseCommandLineOptions(Argc, Argv);
474 enum class LLVMRustFileType {
479 static CodeGenFileType fromRust(LLVMRustFileType Type) {
481 case LLVMRustFileType::AssemblyFile:
482 return CGFT_AssemblyFile;
483 case LLVMRustFileType::ObjectFile:
484 return CGFT_ObjectFile;
486 report_fatal_error("Bad FileType.");
490 extern "C" LLVMRustResult
491 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
492 LLVMModuleRef M, const char *Path, const char *DwoPath,
493 LLVMRustFileType RustFileType) {
494 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
495 auto FileType = fromRust(RustFileType);
497 std::string ErrorInfo;
499 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
501 ErrorInfo = EC.message();
502 if (ErrorInfo != "") {
503 LLVMRustSetLastError(ErrorInfo.c_str());
504 return LLVMRustResult::Failure;
507 buffer_ostream BOS(OS);
509 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
512 ErrorInfo = EC.message();
513 if (ErrorInfo != "") {
514 LLVMRustSetLastError(ErrorInfo.c_str());
515 return LLVMRustResult::Failure;
517 buffer_ostream DBOS(DOS);
518 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
521 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
525 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
526 // stream (OS), so the only real safe place to delete this is here? Don't we
527 // wish this was written in Rust?
528 LLVMDisposePassManager(PMR);
529 return LLVMRustResult::Success;
532 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
533 const char*, // pass name
534 const char*); // IR name
535 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
537 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
538 if (any_isa<const Module *>(WrappedIr))
539 return any_cast<const Module *>(WrappedIr)->getName().str();
540 if (any_isa<const Function *>(WrappedIr))
541 return any_cast<const Function *>(WrappedIr)->getName().str();
542 if (any_isa<const Loop *>(WrappedIr))
543 return any_cast<const Loop *>(WrappedIr)->getName().str();
544 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
545 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
550 void LLVMSelfProfileInitializeCallbacks(
551 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
552 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
553 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
554 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
555 StringRef Pass, llvm::Any Ir) {
556 std::string PassName = Pass.str();
557 std::string IrName = LLVMRustwrappedIrGetName(Ir);
558 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
561 PIC.registerAfterPassCallback(
562 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
563 const PreservedAnalyses &Preserved) {
564 AfterPassCallback(LlvmSelfProfiler);
567 PIC.registerAfterPassInvalidatedCallback(
568 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
569 AfterPassCallback(LlvmSelfProfiler);
572 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
573 StringRef Pass, llvm::Any Ir) {
574 std::string PassName = Pass.str();
575 std::string IrName = LLVMRustwrappedIrGetName(Ir);
576 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
579 PIC.registerAfterAnalysisCallback(
580 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
581 AfterPassCallback(LlvmSelfProfiler);
585 enum class LLVMRustOptStage {
593 struct LLVMRustSanitizerOptions {
594 bool SanitizeAddress;
595 bool SanitizeAddressRecover;
597 bool SanitizeMemoryRecover;
598 int SanitizeMemoryTrackOrigins;
600 bool SanitizeHWAddress;
601 bool SanitizeHWAddressRecover;
604 extern "C" LLVMRustResult
606 LLVMModuleRef ModuleRef,
607 LLVMTargetMachineRef TMRef,
608 LLVMRustPassBuilderOptLevel OptLevelRust,
609 LLVMRustOptStage OptStage,
610 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
611 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
612 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
613 LLVMRustSanitizerOptions *SanitizerOptions,
614 const char *PGOGenPath, const char *PGOUsePath,
615 bool InstrumentCoverage, const char *InstrProfileOutput,
617 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
618 void* LlvmSelfProfiler,
619 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
620 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
621 const char *ExtraPasses, size_t ExtraPassesLen,
622 const char *LLVMPlugins, size_t LLVMPluginsLen) {
623 Module *TheModule = unwrap(ModuleRef);
624 TargetMachine *TM = unwrap(TMRef);
625 OptimizationLevel OptLevel = fromRust(OptLevelRust);
628 PipelineTuningOptions PTO;
629 PTO.LoopUnrolling = UnrollLoops;
630 PTO.LoopInterleaving = UnrollLoops;
631 PTO.LoopVectorization = LoopVectorize;
632 PTO.SLPVectorization = SLPVectorize;
633 PTO.MergeFunctions = MergeFunctions;
635 // FIXME: We may want to expose this as an option.
636 bool DebugPassManager = false;
638 PassInstrumentationCallbacks PIC;
639 #if LLVM_VERSION_LT(16, 0)
640 StandardInstrumentations SI(DebugPassManager);
642 StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
644 SI.registerCallbacks(PIC);
646 if (LlvmSelfProfiler){
647 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
650 #if LLVM_VERSION_LT(16, 0)
651 Optional<PGOOptions> PGOOpt;
653 std::optional<PGOOptions> PGOOpt;
656 assert(!PGOUsePath && !PGOSampleUsePath);
657 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
658 PGOOptions::NoCSAction, DebugInfoForProfiling);
659 } else if (PGOUsePath) {
660 assert(!PGOSampleUsePath);
661 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
662 PGOOptions::NoCSAction, DebugInfoForProfiling);
663 } else if (PGOSampleUsePath) {
664 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
665 PGOOptions::NoCSAction, DebugInfoForProfiling);
666 } else if (DebugInfoForProfiling) {
667 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
668 PGOOptions::NoCSAction, DebugInfoForProfiling);
671 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
672 LoopAnalysisManager LAM;
673 FunctionAnalysisManager FAM;
674 CGSCCAnalysisManager CGAM;
675 ModuleAnalysisManager MAM;
677 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
679 Triple TargetTriple(TheModule->getTargetTriple());
680 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
681 if (DisableSimplifyLibCalls)
682 TLII->disableAllFunctions();
683 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
685 PB.registerModuleAnalyses(MAM);
686 PB.registerCGSCCAnalyses(CGAM);
687 PB.registerFunctionAnalyses(FAM);
688 PB.registerLoopAnalyses(LAM);
689 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
691 // We manually collect pipeline callbacks so we can apply them at O0, where the
692 // PassBuilder does not create a pipeline.
693 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
694 PipelineStartEPCallbacks;
695 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
696 OptimizerLastEPCallbacks;
699 PipelineStartEPCallbacks.push_back(
700 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
701 MPM.addPass(VerifierPass());
706 if (InstrumentGCOV) {
707 PipelineStartEPCallbacks.push_back(
708 [](ModulePassManager &MPM, OptimizationLevel Level) {
709 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
714 if (InstrumentCoverage) {
715 PipelineStartEPCallbacks.push_back(
716 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
717 InstrProfOptions Options;
718 if (InstrProfileOutput) {
719 Options.InstrProfileOutput = InstrProfileOutput;
721 MPM.addPass(InstrProfiling(Options, false));
726 if (SanitizerOptions) {
727 if (SanitizerOptions->SanitizeMemory) {
728 #if LLVM_VERSION_GE(14, 0)
729 MemorySanitizerOptions Options(
730 SanitizerOptions->SanitizeMemoryTrackOrigins,
731 SanitizerOptions->SanitizeMemoryRecover,
732 /*CompileKernel=*/false,
733 /*EagerChecks=*/true);
735 MemorySanitizerOptions Options(
736 SanitizerOptions->SanitizeMemoryTrackOrigins,
737 SanitizerOptions->SanitizeMemoryRecover,
738 /*CompileKernel=*/false);
740 OptimizerLastEPCallbacks.push_back(
741 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
742 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
743 MPM.addPass(ModuleMemorySanitizerPass(Options));
745 MPM.addPass(MemorySanitizerPass(Options));
747 #if LLVM_VERSION_LT(16, 0)
748 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
754 if (SanitizerOptions->SanitizeThread) {
755 OptimizerLastEPCallbacks.push_back(
756 [](ModulePassManager &MPM, OptimizationLevel Level) {
757 #if LLVM_VERSION_GE(14, 0)
758 MPM.addPass(ModuleThreadSanitizerPass());
760 MPM.addPass(ThreadSanitizerPass());
762 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
767 if (SanitizerOptions->SanitizeAddress) {
768 OptimizerLastEPCallbacks.push_back(
769 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
770 #if LLVM_VERSION_LT(15, 0)
771 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
773 #if LLVM_VERSION_GE(14, 0)
774 AddressSanitizerOptions opts = AddressSanitizerOptions{
775 /*CompileKernel=*/false,
776 SanitizerOptions->SanitizeAddressRecover,
777 /*UseAfterScope=*/true,
778 AsanDetectStackUseAfterReturnMode::Runtime,
780 #if LLVM_VERSION_LT(16, 0)
781 MPM.addPass(ModuleAddressSanitizerPass(opts));
783 MPM.addPass(AddressSanitizerPass(opts));
786 MPM.addPass(ModuleAddressSanitizerPass(
787 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
788 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
789 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
790 /*UseAfterScope=*/true)));
795 if (SanitizerOptions->SanitizeHWAddress) {
796 OptimizerLastEPCallbacks.push_back(
797 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
798 #if LLVM_VERSION_GE(14, 0)
799 HWAddressSanitizerOptions opts(
800 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
801 /*DisableOptimization=*/false);
802 MPM.addPass(HWAddressSanitizerPass(opts));
804 MPM.addPass(HWAddressSanitizerPass(
805 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
812 if (LLVMPluginsLen) {
813 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
814 SmallVector<StringRef> Plugins;
815 PluginsStr.split(Plugins, ',', -1, false);
816 for (auto PluginPath: Plugins) {
817 auto Plugin = PassPlugin::Load(PluginPath.str());
819 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
820 return LLVMRustResult::Failure;
822 Plugin->registerPassBuilderCallbacks(PB);
826 ModulePassManager MPM;
827 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
828 if (!NoPrepopulatePasses) {
829 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
830 // At the same time, the LTO pipelines do support O0 and using them is required.
831 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
832 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
833 for (const auto &C : PipelineStartEPCallbacks)
834 PB.registerPipelineStartEPCallback(C);
835 for (const auto &C : OptimizerLastEPCallbacks)
836 PB.registerOptimizerLastEPCallback(C);
838 // Pass false as we manually schedule ThinLTOBufferPasses below.
839 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
841 for (const auto &C : PipelineStartEPCallbacks)
842 PB.registerPipelineStartEPCallback(C);
843 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
844 for (const auto &C : OptimizerLastEPCallbacks)
845 PB.registerOptimizerLastEPCallback(C);
849 case LLVMRustOptStage::PreLinkNoLTO:
850 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
852 case LLVMRustOptStage::PreLinkThinLTO:
853 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
854 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
855 // passes may still run afterwards. This means we need to run the buffer passes again.
856 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
857 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
858 if (OptimizerLastEPCallbacks.empty())
859 NeedThinLTOBufferPasses = false;
860 for (const auto &C : OptimizerLastEPCallbacks)
863 case LLVMRustOptStage::PreLinkFatLTO:
864 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
865 NeedThinLTOBufferPasses = false;
867 case LLVMRustOptStage::ThinLTO:
868 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
869 // It only seems to be needed for C++ specific optimizations.
870 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
872 case LLVMRustOptStage::FatLTO:
873 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
878 // We're not building any of the default pipelines but we still want to
879 // add the verifier, instrumentation, etc passes if they were requested
880 for (const auto &C : PipelineStartEPCallbacks)
882 for (const auto &C : OptimizerLastEPCallbacks)
886 if (ExtraPassesLen) {
887 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
888 std::string ErrMsg = toString(std::move(Err));
889 LLVMRustSetLastError(ErrMsg.c_str());
890 return LLVMRustResult::Failure;
894 if (NeedThinLTOBufferPasses) {
895 MPM.addPass(CanonicalizeAliasesPass());
896 MPM.addPass(NameAnonGlobalPass());
899 // Upgrade all calls to old intrinsics first.
900 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
901 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
903 MPM.run(*TheModule, MAM);
904 return LLVMRustResult::Success;
907 // Callback to demangle function name
909 // * name to be demangled
912 // * output buffer len
913 // Returns len of demangled string, or 0 if demangle failed.
914 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
919 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
921 std::vector<char> Buf;
924 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
926 // Return empty string if demangle failed
927 // or if name does not need to be demangled
928 StringRef CallDemangle(StringRef name) {
933 if (Buf.size() < name.size() * 2) {
934 // Semangled name usually shorter than mangled,
935 // but allocate twice as much memory just in case
936 Buf.resize(name.size() * 2);
939 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
945 auto Demangled = StringRef(Buf.data(), R);
946 if (Demangled == name) {
947 // Do not print anything if demangled name is equal to mangled.
954 void emitFunctionAnnot(const Function *F,
955 formatted_raw_ostream &OS) override {
956 StringRef Demangled = CallDemangle(F->getName());
957 if (Demangled.empty()) {
961 OS << "; " << Demangled << "\n";
964 void emitInstructionAnnot(const Instruction *I,
965 formatted_raw_ostream &OS) override {
968 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
970 Value = CI->getCalledOperand();
971 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
973 Value = II->getCalledOperand();
975 // Could demangle more operations, e. g.
976 // `store %place, @function`.
980 if (!Value->hasName()) {
984 StringRef Demangled = CallDemangle(Value->getName());
985 if (Demangled.empty()) {
989 OS << "; " << Name << " " << Demangled << "\n";
995 extern "C" LLVMRustResult
996 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
997 std::string ErrorInfo;
999 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1001 ErrorInfo = EC.message();
1002 if (ErrorInfo != "") {
1003 LLVMRustSetLastError(ErrorInfo.c_str());
1004 return LLVMRustResult::Failure;
1007 RustAssemblyAnnotationWriter AAW(Demangle);
1008 formatted_raw_ostream FOS(OS);
1009 unwrap(M)->print(FOS, &AAW);
1011 return LLVMRustResult::Success;
1014 extern "C" void LLVMRustPrintPasses() {
1015 LLVMInitializePasses();
1016 struct MyListener : PassRegistrationListener {
1017 void passEnumerate(const PassInfo *Info) {
1018 StringRef PassArg = Info->getPassArgument();
1019 StringRef PassName = Info->getPassName();
1020 if (!PassArg.empty()) {
1021 // These unsigned->signed casts could theoretically overflow, but
1022 // realistically never will (and even if, the result is implementation
1023 // defined rather plain UB).
1024 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1025 (int)PassName.size(), PassName.data());
1030 PassRegistry *PR = PassRegistry::getPassRegistry();
1031 PR->enumerateWith(&Listener);
1034 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1036 auto PreserveFunctions = [=](const GlobalValue &GV) {
1037 for (size_t I = 0; I < Len; I++) {
1038 if (GV.getName() == Symbols[I]) {
1045 internalizeModule(*unwrap(M), PreserveFunctions);
1049 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1050 LLVMTargetMachineRef TMR) {
1051 TargetMachine *Target = unwrap(TMR);
1052 unwrap(Module)->setDataLayout(Target->createDataLayout());
1055 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1056 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1059 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1060 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1063 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1064 LLVMRustCodeModel Model) {
1065 auto CM = fromRust(Model);
1068 unwrap(M)->setCodeModel(*CM);
1071 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1072 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1073 // LLVM, and otherwise it's just blanket rejected from other compilers.
1075 // Most of this implementation is straight copied from LLVM. At the time of
1076 // this writing it wasn't *quite* suitable to reuse more code from upstream
1077 // for our purposes, but we should strive to upstream this support once it's
1078 // ready to go! I figure we may want a bit of testing locally first before
1079 // sending this upstream to LLVM. I hear though they're quite eager to receive
1080 // feedback like this!
1082 // If you're reading this code and wondering "what in the world" or you're
1083 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1084 // then fear not! (ok maybe fear a little). All code here is mostly based
1085 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1087 // You'll find that the general layout here roughly corresponds to the `run`
1088 // method in that file as well as `ProcessThinLTOModule`. Functions are
1089 // specifically commented below as well, but if you're updating this code
1090 // or otherwise trying to understand it, the LLVM source will be useful in
1091 // interpreting the mysteries within.
1093 // Otherwise I'll apologize in advance, it probably requires a relatively
1094 // significant investment on your part to "truly understand" what's going on
1095 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1096 // and various online resources about ThinLTO to make heads or tails of all
1099 // This is a shared data structure which *must* be threadsafe to share
1100 // read-only amongst threads. This also corresponds basically to the arguments
1101 // of the `ProcessThinLTOModule` function in the LLVM source.
1102 struct LLVMRustThinLTOData {
1103 // The combined index that is the global analysis over all modules we're
1104 // performing ThinLTO for. This is mostly managed by LLVM.
1105 ModuleSummaryIndex Index;
1107 // All modules we may look at, stored as in-memory serialized versions. This
1108 // is later used when inlining to ensure we can extract any module to inline
1110 StringMap<MemoryBufferRef> ModuleMap;
1112 // A set that we manage of everything we *don't* want internalized. Note that
1113 // this includes all transitive references right now as well, but it may not
1115 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1117 // Not 100% sure what these are, but they impact what's internalized and
1118 // what's inlined across modules, I believe.
1119 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1120 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1121 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1122 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1124 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1127 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1128 struct LLVMRustThinLTOModule {
1129 const char *identifier;
1134 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1136 static const GlobalValueSummary *
1137 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1138 auto StrongDefForLinker = llvm::find_if(
1139 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1140 auto Linkage = Summary->linkage();
1141 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1142 !GlobalValue::isWeakForLinker(Linkage);
1144 if (StrongDefForLinker != GVSummaryList.end())
1145 return StrongDefForLinker->get();
1147 auto FirstDefForLinker = llvm::find_if(
1148 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1149 auto Linkage = Summary->linkage();
1150 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1152 if (FirstDefForLinker == GVSummaryList.end())
1154 return FirstDefForLinker->get();
1157 // The main entry point for creating the global ThinLTO analysis. The structure
1158 // here is basically the same as before threads are spawned in the `run`
1159 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1160 extern "C" LLVMRustThinLTOData*
1161 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1163 const char **preserved_symbols,
1165 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1167 // Load each module's summary and merge it into one combined index
1168 for (int i = 0; i < num_modules; i++) {
1169 auto module = &modules[i];
1170 StringRef buffer(module->data, module->len);
1171 MemoryBufferRef mem_buffer(buffer, module->identifier);
1173 Ret->ModuleMap[module->identifier] = mem_buffer;
1175 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1176 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1181 // Collect for each module the list of function it defines (GUID -> Summary)
1182 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1184 // Convert the preserved symbols set from string to GUID, this is then needed
1185 // for internalization.
1186 for (int i = 0; i < num_symbols; i++) {
1187 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1188 Ret->GUIDPreservedSymbols.insert(GUID);
1191 // Collect the import/export lists for all modules from the call-graph in the
1194 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1195 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1196 return PrevailingType::Unknown;
1198 // We don't have a complete picture in our use of ThinLTO, just our immediate
1199 // crate, so we need `ImportEnabled = false` to limit internalization.
1200 // Otherwise, we sometimes lose `static` values -- see #60184.
1201 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1202 deadIsPrevailing, /* ImportEnabled = */ false);
1203 ComputeCrossModuleImport(
1205 Ret->ModuleToDefinedGVSummaries,
1210 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1211 // impacts the caching.
1213 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1214 // being lifted from `lib/LTO/LTO.cpp` as well
1215 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1216 for (auto &I : Ret->Index) {
1217 if (I.second.SummaryList.size() > 1)
1218 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1220 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1221 const auto &Prevailing = PrevailingCopy.find(GUID);
1222 if (Prevailing == PrevailingCopy.end())
1224 return Prevailing->second == S;
1226 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1227 GlobalValue::GUID GUID,
1228 GlobalValue::LinkageTypes NewLinkage) {
1229 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1232 // Uses FromPrevailing visibility scheme which works for many binary
1233 // formats. We probably could and should use ELF visibility scheme for many of
1234 // our targets, however.
1236 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1237 Ret->GUIDPreservedSymbols);
1239 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1240 // callback below. This callback below will dictate the linkage for all
1241 // summaries in the index, and we basically just only want to ensure that dead
1242 // symbols are internalized. Otherwise everything that's already external
1243 // linkage will stay as external, and internal will stay as internal.
1244 std::set<GlobalValue::GUID> ExportedGUIDs;
1245 for (auto &List : Ret->Index) {
1246 for (auto &GVS: List.second.SummaryList) {
1247 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1249 auto GUID = GVS->getOriginalName();
1250 if (GVS->flags().Live)
1251 ExportedGUIDs.insert(GUID);
1254 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1255 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1256 return (ExportList != Ret->ExportLists.end() &&
1257 ExportList->second.count(VI)) ||
1258 ExportedGUIDs.count(VI.getGUID());
1260 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1262 return Ret.release();
1266 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1270 // Below are the various passes that happen *per module* when doing ThinLTO.
1272 // In other words, these are the functions that are all run concurrently
1273 // with one another, one per module. The passes here correspond to the analysis
1274 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1275 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1276 // so rustc can save off the intermediate bytecode between each step.
1279 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1280 // When linking an ELF shared object, dso_local should be dropped. We
1281 // conservatively do this for -fpic.
1282 bool ClearDSOLocalOnDeclarations =
1283 TM.getTargetTriple().isOSBinFormatELF() &&
1284 TM.getRelocationModel() != Reloc::Static &&
1285 Mod.getPIELevel() == PIELevel::Default;
1286 return ClearDSOLocalOnDeclarations;
1290 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1291 LLVMTargetMachineRef TM) {
1292 Module &Mod = *unwrap(M);
1293 TargetMachine &Target = *unwrap(TM);
1295 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1296 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1299 LLVMRustSetLastError("renameModuleForThinLTO failed");
1306 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1307 Module &Mod = *unwrap(M);
1308 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1309 #if LLVM_VERSION_GE(14, 0)
1310 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1312 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1318 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1319 Module &Mod = *unwrap(M);
1320 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1321 thinLTOInternalizeModule(Mod, DefinedGlobals);
1326 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1327 LLVMTargetMachineRef TM) {
1328 Module &Mod = *unwrap(M);
1329 TargetMachine &Target = *unwrap(TM);
1331 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1332 auto Loader = [&](StringRef Identifier) {
1333 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1334 auto &Context = Mod.getContext();
1335 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1340 // The rest of this closure is a workaround for
1341 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1342 // we accidentally import wasm custom sections into different modules,
1343 // duplicating them by in the final output artifact.
1345 // The issue is worked around here by manually removing the
1346 // `wasm.custom_sections` named metadata node from any imported module. This
1347 // we know isn't used by any optimization pass so there's no need for it to
1350 // Note that the metadata is currently lazily loaded, so we materialize it
1351 // here before looking up if there's metadata inside. The `FunctionImporter`
1352 // will immediately materialize metadata anyway after an import, so this
1353 // shouldn't be a perf hit.
1354 if (Error Err = (*MOrErr)->materializeMetadata()) {
1355 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1359 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1360 if (WasmCustomSections)
1361 WasmCustomSections->eraseFromParent();
1365 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1366 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1367 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1369 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1375 // This struct and various functions are sort of a hack right now, but the
1376 // problem is that we've got in-memory LLVM modules after we generate and
1377 // optimize all codegen-units for one compilation in rustc. To be compatible
1378 // with the LTO support above we need to serialize the modules plus their
1379 // ThinLTO summary into memory.
1381 // This structure is basically an owned version of a serialize module, with
1382 // a ThinLTO summary attached.
1383 struct LLVMRustThinLTOBuffer {
1387 extern "C" LLVMRustThinLTOBuffer*
1388 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1389 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1391 raw_string_ostream OS(Ret->data);
1395 LoopAnalysisManager LAM;
1396 FunctionAnalysisManager FAM;
1397 CGSCCAnalysisManager CGAM;
1398 ModuleAnalysisManager MAM;
1399 PB.registerModuleAnalyses(MAM);
1400 PB.registerCGSCCAnalyses(CGAM);
1401 PB.registerFunctionAnalyses(FAM);
1402 PB.registerLoopAnalyses(LAM);
1403 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1404 ModulePassManager MPM;
1405 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1406 MPM.run(*unwrap(M), MAM);
1408 WriteBitcodeToFile(*unwrap(M), OS);
1412 return Ret.release();
1416 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1420 extern "C" const void*
1421 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1422 return Buffer->data.data();
1426 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1427 return Buffer->data.length();
1430 // This is what we used to parse upstream bitcode for actual ThinLTO
1431 // processing. We'll call this once per module optimized through ThinLTO, and
1432 // it'll be called concurrently on many threads.
1433 extern "C" LLVMModuleRef
1434 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1437 const char *identifier) {
1438 StringRef Data(data, len);
1439 MemoryBufferRef Buffer(Data, identifier);
1440 unwrap(Context)->enableDebugTypeODRUniquing();
1441 Expected<std::unique_ptr<Module>> SrcOrError =
1442 parseBitcodeFile(Buffer, *unwrap(Context));
1444 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1447 return wrap(std::move(*SrcOrError).release());
1450 // Find the bitcode section in the object file data and return it as a slice.
1451 // Fail if the bitcode section is present but empty.
1453 // On success, the return value is the pointer to the start of the slice and
1454 // `out_len` is filled with the (non-zero) length. On failure, the return value
1455 // is `nullptr` and `out_len` is set to zero.
1456 extern "C" const char*
1457 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1462 StringRef Data(data, len);
1463 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1465 Expected<MemoryBufferRef> BitcodeOrError =
1466 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1467 if (!BitcodeOrError) {
1468 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1472 *out_len = BitcodeOrError->getBufferSize();
1473 return BitcodeOrError->getBufferStart();
1476 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1477 // the comment in `back/lto.rs` for why this exists.
1479 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1481 DICompileUnit **B) {
1482 Module *M = unwrap(Mod);
1483 DICompileUnit **Cur = A;
1484 DICompileUnit **Next = B;
1485 for (DICompileUnit *CU : M->debug_compile_units()) {
1494 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1495 // the comment in `back/lto.rs` for why this exists.
1497 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1498 Module *M = unwrap(Mod);
1500 // If the original source module didn't have a `DICompileUnit` then try to
1501 // merge all the existing compile units. If there aren't actually any though
1502 // then there's not much for us to do so return.
1503 if (Unit == nullptr) {
1504 for (DICompileUnit *CU : M->debug_compile_units()) {
1508 if (Unit == nullptr)
1512 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1513 // process it recursively. Note that we used to specifically iterate over
1514 // instructions to ensure we feed everything into it, but `processModule`
1515 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1516 DebugInfoFinder Finder;
1517 Finder.processModule(*M);
1519 // After we've found all our debuginfo, rewrite all subprograms to point to
1520 // the same `DICompileUnit`.
1521 for (auto &F : Finder.subprograms()) {
1522 F->replaceUnit(Unit);
1525 // Erase any other references to other `DICompileUnit` instances, the verifier
1526 // will later ensure that we don't actually have any other stale references to
1528 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1529 MD->clearOperands();
1530 MD->addOperand(Unit);
1533 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1534 // storing the result in 'KeyOut'.
1535 // Currently, this cache key is a SHA-1 hash of anything that could affect
1536 // the result of optimizing this module (e.g. module imports, exports, liveness
1537 // of access globals, etc).
1538 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1539 // used during the normal linker-plugin incremental thin-LTO process.
1541 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1542 SmallString<40> Key;
1543 llvm::lto::Config conf;
1544 const auto &ImportList = Data->ImportLists.lookup(ModId);
1545 const auto &ExportList = Data->ExportLists.lookup(ModId);
1546 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1547 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1548 std::set<GlobalValue::GUID> CfiFunctionDefs;
1549 std::set<GlobalValue::GUID> CfiFunctionDecls;
1551 // Based on the 'InProcessThinBackend' constructor in LLVM
1552 for (auto &Name : Data->Index.cfiFunctionDefs())
1553 CfiFunctionDefs.insert(
1554 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1555 for (auto &Name : Data->Index.cfiFunctionDecls())
1556 CfiFunctionDecls.insert(
1557 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1559 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1560 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1563 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());