6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/IPO/Internalize.h"
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36 #include "llvm/Transforms/Utils/AddDiscriminators.h"
37 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
38 #include "llvm/LTO/LTO.h"
39 #include "llvm/Bitcode/BitcodeWriter.h"
40 #include "llvm-c/Transforms/PassManagerBuilder.h"
42 #include "llvm/Transforms/Instrumentation.h"
43 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
44 #include "llvm/Support/TimeProfiler.h"
45 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
46 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
47 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
48 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
49 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
50 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
51 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
52 #include "llvm/Transforms/Utils.h"
56 typedef struct LLVMOpaquePass *LLVMPassRef;
57 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
59 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
62 extern "C" void LLVMInitializePasses() {
63 PassRegistry &Registry = *PassRegistry::getPassRegistry();
64 initializeCore(Registry);
65 initializeCodeGen(Registry);
66 initializeScalarOpts(Registry);
67 initializeVectorization(Registry);
68 initializeIPO(Registry);
69 initializeAnalysis(Registry);
70 initializeTransformUtils(Registry);
71 initializeInstCombine(Registry);
72 #if LLVM_VERSION_LT(16, 0)
73 initializeInstrumentation(Registry);
75 initializeTarget(Registry);
78 extern "C" void LLVMTimeTraceProfilerInitialize() {
79 timeTraceProfilerInitialize(
80 /* TimeTraceGranularity */ 0,
81 /* ProcName */ "rustc");
84 extern "C" void LLVMTimeTraceProfilerFinishThread() {
85 timeTraceProfilerFinishThread();
88 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
89 StringRef FN(FileName);
91 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
93 timeTraceProfilerWrite(OS);
94 timeTraceProfilerCleanup();
97 #ifdef LLVM_COMPONENT_X86
98 #define SUBTARGET_X86 SUBTARGET(X86)
100 #define SUBTARGET_X86
103 #ifdef LLVM_COMPONENT_ARM
104 #define SUBTARGET_ARM SUBTARGET(ARM)
106 #define SUBTARGET_ARM
109 #ifdef LLVM_COMPONENT_AARCH64
110 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
112 #define SUBTARGET_AARCH64
115 #ifdef LLVM_COMPONENT_AVR
116 #define SUBTARGET_AVR SUBTARGET(AVR)
118 #define SUBTARGET_AVR
121 #ifdef LLVM_COMPONENT_M68k
122 #define SUBTARGET_M68K SUBTARGET(M68k)
124 #define SUBTARGET_M68K
127 #ifdef LLVM_COMPONENT_MIPS
128 #define SUBTARGET_MIPS SUBTARGET(Mips)
130 #define SUBTARGET_MIPS
133 #ifdef LLVM_COMPONENT_POWERPC
134 #define SUBTARGET_PPC SUBTARGET(PPC)
136 #define SUBTARGET_PPC
139 #ifdef LLVM_COMPONENT_SYSTEMZ
140 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
142 #define SUBTARGET_SYSTEMZ
145 #ifdef LLVM_COMPONENT_MSP430
146 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
148 #define SUBTARGET_MSP430
151 #ifdef LLVM_COMPONENT_RISCV
152 #define SUBTARGET_RISCV SUBTARGET(RISCV)
154 #define SUBTARGET_RISCV
157 #ifdef LLVM_COMPONENT_SPARC
158 #define SUBTARGET_SPARC SUBTARGET(Sparc)
160 #define SUBTARGET_SPARC
163 #ifdef LLVM_COMPONENT_HEXAGON
164 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
166 #define SUBTARGET_HEXAGON
169 #define GEN_SUBTARGETS \
183 #define SUBTARGET(x) \
185 extern const SubtargetFeatureKV x##FeatureKV[]; \
186 extern const SubtargetFeatureKV x##SubTypeKV[]; \
192 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
193 const char *Feature) {
194 TargetMachine *Target = unwrap(TM);
195 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
196 return MCInfo->checkFeatures(std::string("+") + Feature);
199 enum class LLVMRustCodeModel {
208 #if LLVM_VERSION_LT(16, 0)
209 static Optional<CodeModel::Model>
211 static std::optional<CodeModel::Model>
213 fromRust(LLVMRustCodeModel Model) {
215 case LLVMRustCodeModel::Tiny:
216 return CodeModel::Tiny;
217 case LLVMRustCodeModel::Small:
218 return CodeModel::Small;
219 case LLVMRustCodeModel::Kernel:
220 return CodeModel::Kernel;
221 case LLVMRustCodeModel::Medium:
222 return CodeModel::Medium;
223 case LLVMRustCodeModel::Large:
224 return CodeModel::Large;
225 case LLVMRustCodeModel::None:
228 report_fatal_error("Bad CodeModel.");
232 enum class LLVMRustCodeGenOptLevel {
239 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
241 case LLVMRustCodeGenOptLevel::None:
242 return CodeGenOpt::None;
243 case LLVMRustCodeGenOptLevel::Less:
244 return CodeGenOpt::Less;
245 case LLVMRustCodeGenOptLevel::Default:
246 return CodeGenOpt::Default;
247 case LLVMRustCodeGenOptLevel::Aggressive:
248 return CodeGenOpt::Aggressive;
250 report_fatal_error("Bad CodeGenOptLevel.");
254 enum class LLVMRustPassBuilderOptLevel {
263 #if LLVM_VERSION_LT(14,0)
264 using OptimizationLevel = PassBuilder::OptimizationLevel;
267 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
269 case LLVMRustPassBuilderOptLevel::O0:
270 return OptimizationLevel::O0;
271 case LLVMRustPassBuilderOptLevel::O1:
272 return OptimizationLevel::O1;
273 case LLVMRustPassBuilderOptLevel::O2:
274 return OptimizationLevel::O2;
275 case LLVMRustPassBuilderOptLevel::O3:
276 return OptimizationLevel::O3;
277 case LLVMRustPassBuilderOptLevel::Os:
278 return OptimizationLevel::Os;
279 case LLVMRustPassBuilderOptLevel::Oz:
280 return OptimizationLevel::Oz;
282 report_fatal_error("Bad PassBuilderOptLevel.");
286 enum class LLVMRustRelocModel {
295 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
297 case LLVMRustRelocModel::Static:
298 return Reloc::Static;
299 case LLVMRustRelocModel::PIC:
301 case LLVMRustRelocModel::DynamicNoPic:
302 return Reloc::DynamicNoPIC;
303 case LLVMRustRelocModel::ROPI:
305 case LLVMRustRelocModel::RWPI:
307 case LLVMRustRelocModel::ROPIRWPI:
308 return Reloc::ROPI_RWPI;
310 report_fatal_error("Bad RelocModel.");
314 /// getLongestEntryLength - Return the length of the longest entry in the table.
315 template<typename KV>
316 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
318 for (auto &I : Table)
319 MaxLen = std::max(MaxLen, std::strlen(I.Key));
323 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
324 const TargetMachine *Target = unwrap(TM);
325 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
326 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
327 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
328 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
329 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
331 printf("Available CPUs for this target:\n");
332 if (HostArch == TargetArch) {
333 const StringRef HostCPU = sys::getHostCPUName();
334 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
335 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
337 for (auto &CPU : CPUTable)
338 printf(" %-*s\n", MaxCPULen, CPU.Key);
342 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
343 const TargetMachine *Target = unwrap(TM);
344 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
345 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
346 return FeatTable.size();
349 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
350 const char** Feature, const char** Desc) {
351 const TargetMachine *Target = unwrap(TM);
352 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
353 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
354 const SubtargetFeatureKV Feat = FeatTable[Index];
361 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
362 printf("Target CPU help is not supported by this LLVM version.\n\n");
365 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
369 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
372 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
373 StringRef Name = sys::getHostCPUName();
378 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
379 const char *TripleStr, const char *CPU, const char *Feature,
380 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
381 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
382 bool FunctionSections,
384 bool UniqueSectionNames,
385 bool TrapUnreachable,
388 bool EmitStackSizeSection,
389 bool RelaxELFRelocations,
391 const char *SplitDwarfFile) {
393 auto OptLevel = fromRust(RustOptLevel);
394 auto RM = fromRust(RustReloc);
395 auto CM = fromRust(RustCM);
398 Triple Trip(Triple::normalize(TripleStr));
399 const llvm::Target *TheTarget =
400 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
401 if (TheTarget == nullptr) {
402 LLVMRustSetLastError(Error.c_str());
406 TargetOptions Options;
408 Options.FloatABIType = FloatABI::Default;
410 Options.FloatABIType = FloatABI::Soft;
412 Options.DataSections = DataSections;
413 Options.FunctionSections = FunctionSections;
414 Options.UniqueSectionNames = UniqueSectionNames;
415 Options.MCOptions.AsmVerbose = AsmComments;
416 Options.MCOptions.PreserveAsmComments = AsmComments;
417 Options.MCOptions.ABIName = ABIStr;
418 if (SplitDwarfFile) {
419 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
421 Options.RelaxELFRelocations = RelaxELFRelocations;
422 Options.UseInitArray = UseInitArray;
424 if (TrapUnreachable) {
425 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
426 // This limits the extent of possible undefined behavior in some cases, as
427 // it prevents control flow from "falling through" into whatever code
428 // happens to be laid out next in memory.
429 Options.TrapUnreachable = true;
433 Options.ThreadModel = ThreadModel::Single;
436 Options.EmitStackSizeSection = EmitStackSizeSection;
438 TargetMachine *TM = TheTarget->createTargetMachine(
439 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
443 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
447 // Unfortunately, the LLVM C API doesn't provide a way to create the
448 // TargetLibraryInfo pass, so we use this method to do so.
449 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
450 bool DisableSimplifyLibCalls) {
451 Triple TargetTriple(unwrap(M)->getTargetTriple());
452 TargetLibraryInfoImpl TLII(TargetTriple);
453 if (DisableSimplifyLibCalls)
454 TLII.disableAllFunctions();
455 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
458 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
459 // Initializing the command-line options more than once is not allowed. So,
460 // check if they've already been initialized. (This could happen if we're
461 // being called from rustpkg, for example). If the arguments change, then
462 // that's just kinda unfortunate.
463 static bool Initialized = false;
467 cl::ParseCommandLineOptions(Argc, Argv);
470 enum class LLVMRustFileType {
475 static CodeGenFileType fromRust(LLVMRustFileType Type) {
477 case LLVMRustFileType::AssemblyFile:
478 return CGFT_AssemblyFile;
479 case LLVMRustFileType::ObjectFile:
480 return CGFT_ObjectFile;
482 report_fatal_error("Bad FileType.");
486 extern "C" LLVMRustResult
487 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
488 LLVMModuleRef M, const char *Path, const char *DwoPath,
489 LLVMRustFileType RustFileType) {
490 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
491 auto FileType = fromRust(RustFileType);
493 std::string ErrorInfo;
495 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
497 ErrorInfo = EC.message();
498 if (ErrorInfo != "") {
499 LLVMRustSetLastError(ErrorInfo.c_str());
500 return LLVMRustResult::Failure;
503 buffer_ostream BOS(OS);
505 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
508 ErrorInfo = EC.message();
509 if (ErrorInfo != "") {
510 LLVMRustSetLastError(ErrorInfo.c_str());
511 return LLVMRustResult::Failure;
513 buffer_ostream DBOS(DOS);
514 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
517 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
521 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
522 // stream (OS), so the only real safe place to delete this is here? Don't we
523 // wish this was written in Rust?
524 LLVMDisposePassManager(PMR);
525 return LLVMRustResult::Success;
528 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
529 const char*, // pass name
530 const char*); // IR name
531 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
533 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
534 if (any_isa<const Module *>(WrappedIr))
535 return any_cast<const Module *>(WrappedIr)->getName().str();
536 if (any_isa<const Function *>(WrappedIr))
537 return any_cast<const Function *>(WrappedIr)->getName().str();
538 if (any_isa<const Loop *>(WrappedIr))
539 return any_cast<const Loop *>(WrappedIr)->getName().str();
540 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
541 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
546 void LLVMSelfProfileInitializeCallbacks(
547 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
548 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
549 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
550 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
551 StringRef Pass, llvm::Any Ir) {
552 std::string PassName = Pass.str();
553 std::string IrName = LLVMRustwrappedIrGetName(Ir);
554 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
557 PIC.registerAfterPassCallback(
558 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
559 const PreservedAnalyses &Preserved) {
560 AfterPassCallback(LlvmSelfProfiler);
563 PIC.registerAfterPassInvalidatedCallback(
564 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
565 AfterPassCallback(LlvmSelfProfiler);
568 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
569 StringRef Pass, llvm::Any Ir) {
570 std::string PassName = Pass.str();
571 std::string IrName = LLVMRustwrappedIrGetName(Ir);
572 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
575 PIC.registerAfterAnalysisCallback(
576 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
577 AfterPassCallback(LlvmSelfProfiler);
581 enum class LLVMRustOptStage {
589 struct LLVMRustSanitizerOptions {
590 bool SanitizeAddress;
591 bool SanitizeAddressRecover;
593 bool SanitizeMemoryRecover;
594 int SanitizeMemoryTrackOrigins;
596 bool SanitizeHWAddress;
597 bool SanitizeHWAddressRecover;
600 extern "C" LLVMRustResult
602 LLVMModuleRef ModuleRef,
603 LLVMTargetMachineRef TMRef,
604 LLVMRustPassBuilderOptLevel OptLevelRust,
605 LLVMRustOptStage OptStage,
606 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
607 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
608 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
609 LLVMRustSanitizerOptions *SanitizerOptions,
610 const char *PGOGenPath, const char *PGOUsePath,
611 bool InstrumentCoverage, const char *InstrProfileOutput,
613 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
614 void* LlvmSelfProfiler,
615 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
616 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
617 const char *ExtraPasses, size_t ExtraPassesLen,
618 const char *LLVMPlugins, size_t LLVMPluginsLen) {
619 Module *TheModule = unwrap(ModuleRef);
620 TargetMachine *TM = unwrap(TMRef);
621 OptimizationLevel OptLevel = fromRust(OptLevelRust);
624 PipelineTuningOptions PTO;
625 PTO.LoopUnrolling = UnrollLoops;
626 PTO.LoopInterleaving = UnrollLoops;
627 PTO.LoopVectorization = LoopVectorize;
628 PTO.SLPVectorization = SLPVectorize;
629 PTO.MergeFunctions = MergeFunctions;
631 // FIXME: We may want to expose this as an option.
632 bool DebugPassManager = false;
634 PassInstrumentationCallbacks PIC;
635 #if LLVM_VERSION_LT(16, 0)
636 StandardInstrumentations SI(DebugPassManager);
638 StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
640 SI.registerCallbacks(PIC);
642 if (LlvmSelfProfiler){
643 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
646 #if LLVM_VERSION_LT(16, 0)
647 Optional<PGOOptions> PGOOpt;
649 std::optional<PGOOptions> PGOOpt;
652 assert(!PGOUsePath && !PGOSampleUsePath);
653 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
654 PGOOptions::NoCSAction, DebugInfoForProfiling);
655 } else if (PGOUsePath) {
656 assert(!PGOSampleUsePath);
657 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
658 PGOOptions::NoCSAction, DebugInfoForProfiling);
659 } else if (PGOSampleUsePath) {
660 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
661 PGOOptions::NoCSAction, DebugInfoForProfiling);
662 } else if (DebugInfoForProfiling) {
663 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
664 PGOOptions::NoCSAction, DebugInfoForProfiling);
667 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
668 LoopAnalysisManager LAM;
669 FunctionAnalysisManager FAM;
670 CGSCCAnalysisManager CGAM;
671 ModuleAnalysisManager MAM;
673 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
675 Triple TargetTriple(TheModule->getTargetTriple());
676 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
677 if (DisableSimplifyLibCalls)
678 TLII->disableAllFunctions();
679 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
681 PB.registerModuleAnalyses(MAM);
682 PB.registerCGSCCAnalyses(CGAM);
683 PB.registerFunctionAnalyses(FAM);
684 PB.registerLoopAnalyses(LAM);
685 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
687 // We manually collect pipeline callbacks so we can apply them at O0, where the
688 // PassBuilder does not create a pipeline.
689 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
690 PipelineStartEPCallbacks;
691 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
692 OptimizerLastEPCallbacks;
695 PipelineStartEPCallbacks.push_back(
696 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
697 MPM.addPass(VerifierPass());
702 if (InstrumentGCOV) {
703 PipelineStartEPCallbacks.push_back(
704 [](ModulePassManager &MPM, OptimizationLevel Level) {
705 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
710 if (InstrumentCoverage) {
711 PipelineStartEPCallbacks.push_back(
712 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
713 InstrProfOptions Options;
714 if (InstrProfileOutput) {
715 Options.InstrProfileOutput = InstrProfileOutput;
717 MPM.addPass(InstrProfiling(Options, false));
722 if (SanitizerOptions) {
723 if (SanitizerOptions->SanitizeMemory) {
724 #if LLVM_VERSION_GE(14, 0)
725 MemorySanitizerOptions Options(
726 SanitizerOptions->SanitizeMemoryTrackOrigins,
727 SanitizerOptions->SanitizeMemoryRecover,
728 /*CompileKernel=*/false,
729 /*EagerChecks=*/true);
731 MemorySanitizerOptions Options(
732 SanitizerOptions->SanitizeMemoryTrackOrigins,
733 SanitizerOptions->SanitizeMemoryRecover,
734 /*CompileKernel=*/false);
736 OptimizerLastEPCallbacks.push_back(
737 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
738 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
739 MPM.addPass(ModuleMemorySanitizerPass(Options));
741 MPM.addPass(MemorySanitizerPass(Options));
743 #if LLVM_VERSION_LT(16, 0)
744 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
750 if (SanitizerOptions->SanitizeThread) {
751 OptimizerLastEPCallbacks.push_back(
752 [](ModulePassManager &MPM, OptimizationLevel Level) {
753 #if LLVM_VERSION_GE(14, 0)
754 MPM.addPass(ModuleThreadSanitizerPass());
756 MPM.addPass(ThreadSanitizerPass());
758 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
763 if (SanitizerOptions->SanitizeAddress) {
764 OptimizerLastEPCallbacks.push_back(
765 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
766 #if LLVM_VERSION_LT(15, 0)
767 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
769 #if LLVM_VERSION_GE(14, 0)
770 AddressSanitizerOptions opts = AddressSanitizerOptions{
771 /*CompileKernel=*/false,
772 SanitizerOptions->SanitizeAddressRecover,
773 /*UseAfterScope=*/true,
774 AsanDetectStackUseAfterReturnMode::Runtime,
776 #if LLVM_VERSION_LT(16, 0)
777 MPM.addPass(ModuleAddressSanitizerPass(opts));
779 MPM.addPass(AddressSanitizerPass(opts));
782 MPM.addPass(ModuleAddressSanitizerPass(
783 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
784 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
785 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
786 /*UseAfterScope=*/true)));
791 if (SanitizerOptions->SanitizeHWAddress) {
792 OptimizerLastEPCallbacks.push_back(
793 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
794 #if LLVM_VERSION_GE(14, 0)
795 HWAddressSanitizerOptions opts(
796 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
797 /*DisableOptimization=*/false);
798 MPM.addPass(HWAddressSanitizerPass(opts));
800 MPM.addPass(HWAddressSanitizerPass(
801 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
808 if (LLVMPluginsLen) {
809 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
810 SmallVector<StringRef> Plugins;
811 PluginsStr.split(Plugins, ',', -1, false);
812 for (auto PluginPath: Plugins) {
813 auto Plugin = PassPlugin::Load(PluginPath.str());
815 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
816 return LLVMRustResult::Failure;
818 Plugin->registerPassBuilderCallbacks(PB);
822 ModulePassManager MPM;
823 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
824 if (!NoPrepopulatePasses) {
825 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
826 // At the same time, the LTO pipelines do support O0 and using them is required.
827 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
828 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
829 for (const auto &C : PipelineStartEPCallbacks)
830 PB.registerPipelineStartEPCallback(C);
831 for (const auto &C : OptimizerLastEPCallbacks)
832 PB.registerOptimizerLastEPCallback(C);
834 // Pass false as we manually schedule ThinLTOBufferPasses below.
835 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
837 for (const auto &C : PipelineStartEPCallbacks)
838 PB.registerPipelineStartEPCallback(C);
839 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
840 for (const auto &C : OptimizerLastEPCallbacks)
841 PB.registerOptimizerLastEPCallback(C);
845 case LLVMRustOptStage::PreLinkNoLTO:
846 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
848 case LLVMRustOptStage::PreLinkThinLTO:
849 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
850 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
851 // passes may still run afterwards. This means we need to run the buffer passes again.
852 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
853 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
854 if (OptimizerLastEPCallbacks.empty())
855 NeedThinLTOBufferPasses = false;
856 for (const auto &C : OptimizerLastEPCallbacks)
859 case LLVMRustOptStage::PreLinkFatLTO:
860 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
861 NeedThinLTOBufferPasses = false;
863 case LLVMRustOptStage::ThinLTO:
864 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
865 // It only seems to be needed for C++ specific optimizations.
866 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
868 case LLVMRustOptStage::FatLTO:
869 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
874 // We're not building any of the default pipelines but we still want to
875 // add the verifier, instrumentation, etc passes if they were requested
876 for (const auto &C : PipelineStartEPCallbacks)
878 for (const auto &C : OptimizerLastEPCallbacks)
882 if (ExtraPassesLen) {
883 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
884 std::string ErrMsg = toString(std::move(Err));
885 LLVMRustSetLastError(ErrMsg.c_str());
886 return LLVMRustResult::Failure;
890 if (NeedThinLTOBufferPasses) {
891 MPM.addPass(CanonicalizeAliasesPass());
892 MPM.addPass(NameAnonGlobalPass());
895 // Upgrade all calls to old intrinsics first.
896 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
897 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
899 MPM.run(*TheModule, MAM);
900 return LLVMRustResult::Success;
903 // Callback to demangle function name
905 // * name to be demangled
908 // * output buffer len
909 // Returns len of demangled string, or 0 if demangle failed.
910 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
915 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
917 std::vector<char> Buf;
920 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
922 // Return empty string if demangle failed
923 // or if name does not need to be demangled
924 StringRef CallDemangle(StringRef name) {
929 if (Buf.size() < name.size() * 2) {
930 // Semangled name usually shorter than mangled,
931 // but allocate twice as much memory just in case
932 Buf.resize(name.size() * 2);
935 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
941 auto Demangled = StringRef(Buf.data(), R);
942 if (Demangled == name) {
943 // Do not print anything if demangled name is equal to mangled.
950 void emitFunctionAnnot(const Function *F,
951 formatted_raw_ostream &OS) override {
952 StringRef Demangled = CallDemangle(F->getName());
953 if (Demangled.empty()) {
957 OS << "; " << Demangled << "\n";
960 void emitInstructionAnnot(const Instruction *I,
961 formatted_raw_ostream &OS) override {
964 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
966 Value = CI->getCalledOperand();
967 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
969 Value = II->getCalledOperand();
971 // Could demangle more operations, e. g.
972 // `store %place, @function`.
976 if (!Value->hasName()) {
980 StringRef Demangled = CallDemangle(Value->getName());
981 if (Demangled.empty()) {
985 OS << "; " << Name << " " << Demangled << "\n";
991 extern "C" LLVMRustResult
992 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
993 std::string ErrorInfo;
995 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
997 ErrorInfo = EC.message();
998 if (ErrorInfo != "") {
999 LLVMRustSetLastError(ErrorInfo.c_str());
1000 return LLVMRustResult::Failure;
1003 RustAssemblyAnnotationWriter AAW(Demangle);
1004 formatted_raw_ostream FOS(OS);
1005 unwrap(M)->print(FOS, &AAW);
1007 return LLVMRustResult::Success;
1010 extern "C" void LLVMRustPrintPasses() {
1011 LLVMInitializePasses();
1012 struct MyListener : PassRegistrationListener {
1013 void passEnumerate(const PassInfo *Info) {
1014 StringRef PassArg = Info->getPassArgument();
1015 StringRef PassName = Info->getPassName();
1016 if (!PassArg.empty()) {
1017 // These unsigned->signed casts could theoretically overflow, but
1018 // realistically never will (and even if, the result is implementation
1019 // defined rather plain UB).
1020 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1021 (int)PassName.size(), PassName.data());
1026 PassRegistry *PR = PassRegistry::getPassRegistry();
1027 PR->enumerateWith(&Listener);
1030 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1032 auto PreserveFunctions = [=](const GlobalValue &GV) {
1033 for (size_t I = 0; I < Len; I++) {
1034 if (GV.getName() == Symbols[I]) {
1041 internalizeModule(*unwrap(M), PreserveFunctions);
1045 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1046 LLVMTargetMachineRef TMR) {
1047 TargetMachine *Target = unwrap(TMR);
1048 unwrap(Module)->setDataLayout(Target->createDataLayout());
1051 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1052 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1055 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1056 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1059 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1060 LLVMRustCodeModel Model) {
1061 auto CM = fromRust(Model);
1064 unwrap(M)->setCodeModel(*CM);
1067 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1068 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1069 // LLVM, and otherwise it's just blanket rejected from other compilers.
1071 // Most of this implementation is straight copied from LLVM. At the time of
1072 // this writing it wasn't *quite* suitable to reuse more code from upstream
1073 // for our purposes, but we should strive to upstream this support once it's
1074 // ready to go! I figure we may want a bit of testing locally first before
1075 // sending this upstream to LLVM. I hear though they're quite eager to receive
1076 // feedback like this!
1078 // If you're reading this code and wondering "what in the world" or you're
1079 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1080 // then fear not! (ok maybe fear a little). All code here is mostly based
1081 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1083 // You'll find that the general layout here roughly corresponds to the `run`
1084 // method in that file as well as `ProcessThinLTOModule`. Functions are
1085 // specifically commented below as well, but if you're updating this code
1086 // or otherwise trying to understand it, the LLVM source will be useful in
1087 // interpreting the mysteries within.
1089 // Otherwise I'll apologize in advance, it probably requires a relatively
1090 // significant investment on your part to "truly understand" what's going on
1091 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1092 // and various online resources about ThinLTO to make heads or tails of all
1095 // This is a shared data structure which *must* be threadsafe to share
1096 // read-only amongst threads. This also corresponds basically to the arguments
1097 // of the `ProcessThinLTOModule` function in the LLVM source.
1098 struct LLVMRustThinLTOData {
1099 // The combined index that is the global analysis over all modules we're
1100 // performing ThinLTO for. This is mostly managed by LLVM.
1101 ModuleSummaryIndex Index;
1103 // All modules we may look at, stored as in-memory serialized versions. This
1104 // is later used when inlining to ensure we can extract any module to inline
1106 StringMap<MemoryBufferRef> ModuleMap;
1108 // A set that we manage of everything we *don't* want internalized. Note that
1109 // this includes all transitive references right now as well, but it may not
1111 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1113 // Not 100% sure what these are, but they impact what's internalized and
1114 // what's inlined across modules, I believe.
1115 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1116 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1117 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1118 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1120 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1123 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1124 struct LLVMRustThinLTOModule {
1125 const char *identifier;
1130 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1132 static const GlobalValueSummary *
1133 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1134 auto StrongDefForLinker = llvm::find_if(
1135 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1136 auto Linkage = Summary->linkage();
1137 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1138 !GlobalValue::isWeakForLinker(Linkage);
1140 if (StrongDefForLinker != GVSummaryList.end())
1141 return StrongDefForLinker->get();
1143 auto FirstDefForLinker = llvm::find_if(
1144 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1145 auto Linkage = Summary->linkage();
1146 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1148 if (FirstDefForLinker == GVSummaryList.end())
1150 return FirstDefForLinker->get();
1153 // The main entry point for creating the global ThinLTO analysis. The structure
1154 // here is basically the same as before threads are spawned in the `run`
1155 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1156 extern "C" LLVMRustThinLTOData*
1157 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1159 const char **preserved_symbols,
1161 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1163 // Load each module's summary and merge it into one combined index
1164 for (int i = 0; i < num_modules; i++) {
1165 auto module = &modules[i];
1166 StringRef buffer(module->data, module->len);
1167 MemoryBufferRef mem_buffer(buffer, module->identifier);
1169 Ret->ModuleMap[module->identifier] = mem_buffer;
1171 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1172 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1177 // Collect for each module the list of function it defines (GUID -> Summary)
1178 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1180 // Convert the preserved symbols set from string to GUID, this is then needed
1181 // for internalization.
1182 for (int i = 0; i < num_symbols; i++) {
1183 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1184 Ret->GUIDPreservedSymbols.insert(GUID);
1187 // Collect the import/export lists for all modules from the call-graph in the
1190 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1191 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1192 return PrevailingType::Unknown;
1194 // We don't have a complete picture in our use of ThinLTO, just our immediate
1195 // crate, so we need `ImportEnabled = false` to limit internalization.
1196 // Otherwise, we sometimes lose `static` values -- see #60184.
1197 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1198 deadIsPrevailing, /* ImportEnabled = */ false);
1199 ComputeCrossModuleImport(
1201 Ret->ModuleToDefinedGVSummaries,
1206 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1207 // impacts the caching.
1209 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1210 // being lifted from `lib/LTO/LTO.cpp` as well
1211 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1212 for (auto &I : Ret->Index) {
1213 if (I.second.SummaryList.size() > 1)
1214 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1216 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1217 const auto &Prevailing = PrevailingCopy.find(GUID);
1218 if (Prevailing == PrevailingCopy.end())
1220 return Prevailing->second == S;
1222 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1223 GlobalValue::GUID GUID,
1224 GlobalValue::LinkageTypes NewLinkage) {
1225 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1228 // Uses FromPrevailing visibility scheme which works for many binary
1229 // formats. We probably could and should use ELF visibility scheme for many of
1230 // our targets, however.
1232 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1233 Ret->GUIDPreservedSymbols);
1235 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1236 // callback below. This callback below will dictate the linkage for all
1237 // summaries in the index, and we basically just only want to ensure that dead
1238 // symbols are internalized. Otherwise everything that's already external
1239 // linkage will stay as external, and internal will stay as internal.
1240 std::set<GlobalValue::GUID> ExportedGUIDs;
1241 for (auto &List : Ret->Index) {
1242 for (auto &GVS: List.second.SummaryList) {
1243 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1245 auto GUID = GVS->getOriginalName();
1246 if (GVS->flags().Live)
1247 ExportedGUIDs.insert(GUID);
1250 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1251 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1252 return (ExportList != Ret->ExportLists.end() &&
1253 ExportList->second.count(VI)) ||
1254 ExportedGUIDs.count(VI.getGUID());
1256 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1258 return Ret.release();
1262 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1266 // Below are the various passes that happen *per module* when doing ThinLTO.
1268 // In other words, these are the functions that are all run concurrently
1269 // with one another, one per module. The passes here correspond to the analysis
1270 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1271 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1272 // so rustc can save off the intermediate bytecode between each step.
1275 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1276 // When linking an ELF shared object, dso_local should be dropped. We
1277 // conservatively do this for -fpic.
1278 bool ClearDSOLocalOnDeclarations =
1279 TM.getTargetTriple().isOSBinFormatELF() &&
1280 TM.getRelocationModel() != Reloc::Static &&
1281 Mod.getPIELevel() == PIELevel::Default;
1282 return ClearDSOLocalOnDeclarations;
1286 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1287 LLVMTargetMachineRef TM) {
1288 Module &Mod = *unwrap(M);
1289 TargetMachine &Target = *unwrap(TM);
1291 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1292 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1295 LLVMRustSetLastError("renameModuleForThinLTO failed");
1302 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1303 Module &Mod = *unwrap(M);
1304 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1305 #if LLVM_VERSION_GE(14, 0)
1306 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1308 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1314 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1315 Module &Mod = *unwrap(M);
1316 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1317 thinLTOInternalizeModule(Mod, DefinedGlobals);
1322 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1323 LLVMTargetMachineRef TM) {
1324 Module &Mod = *unwrap(M);
1325 TargetMachine &Target = *unwrap(TM);
1327 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1328 auto Loader = [&](StringRef Identifier) {
1329 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1330 auto &Context = Mod.getContext();
1331 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1336 // The rest of this closure is a workaround for
1337 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1338 // we accidentally import wasm custom sections into different modules,
1339 // duplicating them by in the final output artifact.
1341 // The issue is worked around here by manually removing the
1342 // `wasm.custom_sections` named metadata node from any imported module. This
1343 // we know isn't used by any optimization pass so there's no need for it to
1346 // Note that the metadata is currently lazily loaded, so we materialize it
1347 // here before looking up if there's metadata inside. The `FunctionImporter`
1348 // will immediately materialize metadata anyway after an import, so this
1349 // shouldn't be a perf hit.
1350 if (Error Err = (*MOrErr)->materializeMetadata()) {
1351 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1355 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1356 if (WasmCustomSections)
1357 WasmCustomSections->eraseFromParent();
1361 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1362 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1363 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1365 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1371 // This struct and various functions are sort of a hack right now, but the
1372 // problem is that we've got in-memory LLVM modules after we generate and
1373 // optimize all codegen-units for one compilation in rustc. To be compatible
1374 // with the LTO support above we need to serialize the modules plus their
1375 // ThinLTO summary into memory.
1377 // This structure is basically an owned version of a serialize module, with
1378 // a ThinLTO summary attached.
1379 struct LLVMRustThinLTOBuffer {
1383 extern "C" LLVMRustThinLTOBuffer*
1384 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1385 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1387 raw_string_ostream OS(Ret->data);
1391 LoopAnalysisManager LAM;
1392 FunctionAnalysisManager FAM;
1393 CGSCCAnalysisManager CGAM;
1394 ModuleAnalysisManager MAM;
1395 PB.registerModuleAnalyses(MAM);
1396 PB.registerCGSCCAnalyses(CGAM);
1397 PB.registerFunctionAnalyses(FAM);
1398 PB.registerLoopAnalyses(LAM);
1399 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1400 ModulePassManager MPM;
1401 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1402 MPM.run(*unwrap(M), MAM);
1404 WriteBitcodeToFile(*unwrap(M), OS);
1408 return Ret.release();
1412 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1416 extern "C" const void*
1417 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1418 return Buffer->data.data();
1422 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1423 return Buffer->data.length();
1426 // This is what we used to parse upstream bitcode for actual ThinLTO
1427 // processing. We'll call this once per module optimized through ThinLTO, and
1428 // it'll be called concurrently on many threads.
1429 extern "C" LLVMModuleRef
1430 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1433 const char *identifier) {
1434 StringRef Data(data, len);
1435 MemoryBufferRef Buffer(Data, identifier);
1436 unwrap(Context)->enableDebugTypeODRUniquing();
1437 Expected<std::unique_ptr<Module>> SrcOrError =
1438 parseBitcodeFile(Buffer, *unwrap(Context));
1440 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1443 return wrap(std::move(*SrcOrError).release());
1446 // Find the bitcode section in the object file data and return it as a slice.
1447 // Fail if the bitcode section is present but empty.
1449 // On success, the return value is the pointer to the start of the slice and
1450 // `out_len` is filled with the (non-zero) length. On failure, the return value
1451 // is `nullptr` and `out_len` is set to zero.
1452 extern "C" const char*
1453 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1458 StringRef Data(data, len);
1459 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1461 Expected<MemoryBufferRef> BitcodeOrError =
1462 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1463 if (!BitcodeOrError) {
1464 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1468 *out_len = BitcodeOrError->getBufferSize();
1469 return BitcodeOrError->getBufferStart();
1472 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1473 // the comment in `back/lto.rs` for why this exists.
1475 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1477 DICompileUnit **B) {
1478 Module *M = unwrap(Mod);
1479 DICompileUnit **Cur = A;
1480 DICompileUnit **Next = B;
1481 for (DICompileUnit *CU : M->debug_compile_units()) {
1490 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1491 // the comment in `back/lto.rs` for why this exists.
1493 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1494 Module *M = unwrap(Mod);
1496 // If the original source module didn't have a `DICompileUnit` then try to
1497 // merge all the existing compile units. If there aren't actually any though
1498 // then there's not much for us to do so return.
1499 if (Unit == nullptr) {
1500 for (DICompileUnit *CU : M->debug_compile_units()) {
1504 if (Unit == nullptr)
1508 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1509 // process it recursively. Note that we used to specifically iterate over
1510 // instructions to ensure we feed everything into it, but `processModule`
1511 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1512 DebugInfoFinder Finder;
1513 Finder.processModule(*M);
1515 // After we've found all our debuginfo, rewrite all subprograms to point to
1516 // the same `DICompileUnit`.
1517 for (auto &F : Finder.subprograms()) {
1518 F->replaceUnit(Unit);
1521 // Erase any other references to other `DICompileUnit` instances, the verifier
1522 // will later ensure that we don't actually have any other stale references to
1524 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1525 MD->clearOperands();
1526 MD->addOperand(Unit);
1529 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1530 // storing the result in 'KeyOut'.
1531 // Currently, this cache key is a SHA-1 hash of anything that could affect
1532 // the result of optimizing this module (e.g. module imports, exports, liveness
1533 // of access globals, etc).
1534 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1535 // used during the normal linker-plugin incremental thin-LTO process.
1537 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1538 SmallString<40> Key;
1539 llvm::lto::Config conf;
1540 const auto &ImportList = Data->ImportLists.lookup(ModId);
1541 const auto &ExportList = Data->ExportLists.lookup(ModId);
1542 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1543 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1544 std::set<GlobalValue::GUID> CfiFunctionDefs;
1545 std::set<GlobalValue::GUID> CfiFunctionDecls;
1547 // Based on the 'InProcessThinBackend' constructor in LLVM
1548 for (auto &Name : Data->Index.cfiFunctionDefs())
1549 CfiFunctionDefs.insert(
1550 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1551 for (auto &Name : Data->Index.cfiFunctionDecls())
1552 CfiFunctionDecls.insert(
1553 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1555 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1556 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1559 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());