6 #include "LLVMWrapper.h"
8 #include "llvm/Analysis/AliasAnalysis.h"
9 #include "llvm/Analysis/TargetLibraryInfo.h"
10 #include "llvm/Analysis/TargetTransformInfo.h"
11 #include "llvm/CodeGen/TargetSubtargetInfo.h"
12 #include "llvm/InitializePasses.h"
13 #include "llvm/IR/AutoUpgrade.h"
14 #include "llvm/IR/AssemblyAnnotationWriter.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Verifier.h"
17 #include "llvm/Object/ObjectFile.h"
18 #include "llvm/Object/IRObjectFile.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Passes/PassPlugin.h"
21 #include "llvm/Passes/StandardInstrumentations.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #if LLVM_VERSION_LT(14, 0)
26 #include "llvm/Support/TargetRegistry.h"
28 #include "llvm/MC/TargetRegistry.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
32 #include "llvm/Transforms/IPO/AlwaysInliner.h"
33 #include "llvm/Transforms/IPO/FunctionImport.h"
34 #include "llvm/Transforms/IPO/Internalize.h"
35 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36 #include "llvm/Transforms/Utils/AddDiscriminators.h"
37 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
38 #include "llvm/LTO/LTO.h"
39 #include "llvm/Bitcode/BitcodeWriter.h"
40 #include "llvm-c/Transforms/PassManagerBuilder.h"
42 #include "llvm/Transforms/Instrumentation.h"
43 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
44 #include "llvm/Support/TimeProfiler.h"
45 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
46 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
47 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
48 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
49 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
50 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
51 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
52 #include "llvm/Transforms/Utils.h"
56 typedef struct LLVMOpaquePass *LLVMPassRef;
57 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
59 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
60 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
62 extern "C" void LLVMInitializePasses() {
63 PassRegistry &Registry = *PassRegistry::getPassRegistry();
64 initializeCore(Registry);
65 initializeCodeGen(Registry);
66 initializeScalarOpts(Registry);
67 initializeVectorization(Registry);
68 initializeIPO(Registry);
69 initializeAnalysis(Registry);
70 initializeTransformUtils(Registry);
71 initializeInstCombine(Registry);
72 #if LLVM_VERSION_LT(16, 0)
73 initializeInstrumentation(Registry);
75 initializeTarget(Registry);
78 extern "C" void LLVMTimeTraceProfilerInitialize() {
79 timeTraceProfilerInitialize(
80 /* TimeTraceGranularity */ 0,
81 /* ProcName */ "rustc");
84 extern "C" void LLVMTimeTraceProfilerFinishThread() {
85 timeTraceProfilerFinishThread();
88 extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
89 StringRef FN(FileName);
91 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
93 timeTraceProfilerWrite(OS);
94 timeTraceProfilerCleanup();
97 #ifdef LLVM_COMPONENT_X86
98 #define SUBTARGET_X86 SUBTARGET(X86)
100 #define SUBTARGET_X86
103 #ifdef LLVM_COMPONENT_ARM
104 #define SUBTARGET_ARM SUBTARGET(ARM)
106 #define SUBTARGET_ARM
109 #ifdef LLVM_COMPONENT_AARCH64
110 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
112 #define SUBTARGET_AARCH64
115 #ifdef LLVM_COMPONENT_AVR
116 #define SUBTARGET_AVR SUBTARGET(AVR)
118 #define SUBTARGET_AVR
121 #ifdef LLVM_COMPONENT_M68k
122 #define SUBTARGET_M68K SUBTARGET(M68k)
124 #define SUBTARGET_M68K
127 #ifdef LLVM_COMPONENT_MIPS
128 #define SUBTARGET_MIPS SUBTARGET(Mips)
130 #define SUBTARGET_MIPS
133 #ifdef LLVM_COMPONENT_POWERPC
134 #define SUBTARGET_PPC SUBTARGET(PPC)
136 #define SUBTARGET_PPC
139 #ifdef LLVM_COMPONENT_SYSTEMZ
140 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
142 #define SUBTARGET_SYSTEMZ
145 #ifdef LLVM_COMPONENT_MSP430
146 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
148 #define SUBTARGET_MSP430
151 #ifdef LLVM_COMPONENT_RISCV
152 #define SUBTARGET_RISCV SUBTARGET(RISCV)
154 #define SUBTARGET_RISCV
157 #ifdef LLVM_COMPONENT_SPARC
158 #define SUBTARGET_SPARC SUBTARGET(Sparc)
160 #define SUBTARGET_SPARC
163 #ifdef LLVM_COMPONENT_HEXAGON
164 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
166 #define SUBTARGET_HEXAGON
169 #define GEN_SUBTARGETS \
183 #define SUBTARGET(x) \
185 extern const SubtargetFeatureKV x##FeatureKV[]; \
186 extern const SubtargetFeatureKV x##SubTypeKV[]; \
192 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
193 const char *Feature) {
194 TargetMachine *Target = unwrap(TM);
195 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
196 return MCInfo->checkFeatures(std::string("+") + Feature);
199 enum class LLVMRustCodeModel {
208 static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) {
210 case LLVMRustCodeModel::Tiny:
211 return CodeModel::Tiny;
212 case LLVMRustCodeModel::Small:
213 return CodeModel::Small;
214 case LLVMRustCodeModel::Kernel:
215 return CodeModel::Kernel;
216 case LLVMRustCodeModel::Medium:
217 return CodeModel::Medium;
218 case LLVMRustCodeModel::Large:
219 return CodeModel::Large;
220 case LLVMRustCodeModel::None:
223 report_fatal_error("Bad CodeModel.");
227 enum class LLVMRustCodeGenOptLevel {
234 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
236 case LLVMRustCodeGenOptLevel::None:
237 return CodeGenOpt::None;
238 case LLVMRustCodeGenOptLevel::Less:
239 return CodeGenOpt::Less;
240 case LLVMRustCodeGenOptLevel::Default:
241 return CodeGenOpt::Default;
242 case LLVMRustCodeGenOptLevel::Aggressive:
243 return CodeGenOpt::Aggressive;
245 report_fatal_error("Bad CodeGenOptLevel.");
249 enum class LLVMRustPassBuilderOptLevel {
258 #if LLVM_VERSION_LT(14,0)
259 using OptimizationLevel = PassBuilder::OptimizationLevel;
262 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
264 case LLVMRustPassBuilderOptLevel::O0:
265 return OptimizationLevel::O0;
266 case LLVMRustPassBuilderOptLevel::O1:
267 return OptimizationLevel::O1;
268 case LLVMRustPassBuilderOptLevel::O2:
269 return OptimizationLevel::O2;
270 case LLVMRustPassBuilderOptLevel::O3:
271 return OptimizationLevel::O3;
272 case LLVMRustPassBuilderOptLevel::Os:
273 return OptimizationLevel::Os;
274 case LLVMRustPassBuilderOptLevel::Oz:
275 return OptimizationLevel::Oz;
277 report_fatal_error("Bad PassBuilderOptLevel.");
281 enum class LLVMRustRelocModel {
290 static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
292 case LLVMRustRelocModel::Static:
293 return Reloc::Static;
294 case LLVMRustRelocModel::PIC:
296 case LLVMRustRelocModel::DynamicNoPic:
297 return Reloc::DynamicNoPIC;
298 case LLVMRustRelocModel::ROPI:
300 case LLVMRustRelocModel::RWPI:
302 case LLVMRustRelocModel::ROPIRWPI:
303 return Reloc::ROPI_RWPI;
305 report_fatal_error("Bad RelocModel.");
309 /// getLongestEntryLength - Return the length of the longest entry in the table.
310 template<typename KV>
311 static size_t getLongestEntryLength(ArrayRef<KV> Table) {
313 for (auto &I : Table)
314 MaxLen = std::max(MaxLen, std::strlen(I.Key));
318 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
319 const TargetMachine *Target = unwrap(TM);
320 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
321 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
322 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
323 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
324 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
326 printf("Available CPUs for this target:\n");
327 if (HostArch == TargetArch) {
328 const StringRef HostCPU = sys::getHostCPUName();
329 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
330 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
332 for (auto &CPU : CPUTable)
333 printf(" %-*s\n", MaxCPULen, CPU.Key);
337 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
338 const TargetMachine *Target = unwrap(TM);
339 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
340 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
341 return FeatTable.size();
344 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
345 const char** Feature, const char** Desc) {
346 const TargetMachine *Target = unwrap(TM);
347 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
348 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
349 const SubtargetFeatureKV Feat = FeatTable[Index];
356 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
357 printf("Target CPU help is not supported by this LLVM version.\n\n");
360 extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) {
364 extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {}
367 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
368 StringRef Name = sys::getHostCPUName();
373 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
374 const char *TripleStr, const char *CPU, const char *Feature,
375 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
376 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
377 bool FunctionSections,
379 bool UniqueSectionNames,
380 bool TrapUnreachable,
383 bool EmitStackSizeSection,
384 bool RelaxELFRelocations,
386 const char *SplitDwarfFile) {
388 auto OptLevel = fromRust(RustOptLevel);
389 auto RM = fromRust(RustReloc);
390 auto CM = fromRust(RustCM);
393 Triple Trip(Triple::normalize(TripleStr));
394 const llvm::Target *TheTarget =
395 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
396 if (TheTarget == nullptr) {
397 LLVMRustSetLastError(Error.c_str());
401 TargetOptions Options;
403 Options.FloatABIType = FloatABI::Default;
405 Options.FloatABIType = FloatABI::Soft;
407 Options.DataSections = DataSections;
408 Options.FunctionSections = FunctionSections;
409 Options.UniqueSectionNames = UniqueSectionNames;
410 Options.MCOptions.AsmVerbose = AsmComments;
411 Options.MCOptions.PreserveAsmComments = AsmComments;
412 Options.MCOptions.ABIName = ABIStr;
413 if (SplitDwarfFile) {
414 Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
416 Options.RelaxELFRelocations = RelaxELFRelocations;
417 Options.UseInitArray = UseInitArray;
419 if (TrapUnreachable) {
420 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
421 // This limits the extent of possible undefined behavior in some cases, as
422 // it prevents control flow from "falling through" into whatever code
423 // happens to be laid out next in memory.
424 Options.TrapUnreachable = true;
428 Options.ThreadModel = ThreadModel::Single;
431 Options.EmitStackSizeSection = EmitStackSizeSection;
433 TargetMachine *TM = TheTarget->createTargetMachine(
434 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
438 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
442 // Unfortunately, the LLVM C API doesn't provide a way to create the
443 // TargetLibraryInfo pass, so we use this method to do so.
444 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
445 bool DisableSimplifyLibCalls) {
446 Triple TargetTriple(unwrap(M)->getTargetTriple());
447 TargetLibraryInfoImpl TLII(TargetTriple);
448 if (DisableSimplifyLibCalls)
449 TLII.disableAllFunctions();
450 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
453 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
454 // Initializing the command-line options more than once is not allowed. So,
455 // check if they've already been initialized. (This could happen if we're
456 // being called from rustpkg, for example). If the arguments change, then
457 // that's just kinda unfortunate.
458 static bool Initialized = false;
462 cl::ParseCommandLineOptions(Argc, Argv);
465 enum class LLVMRustFileType {
470 static CodeGenFileType fromRust(LLVMRustFileType Type) {
472 case LLVMRustFileType::AssemblyFile:
473 return CGFT_AssemblyFile;
474 case LLVMRustFileType::ObjectFile:
475 return CGFT_ObjectFile;
477 report_fatal_error("Bad FileType.");
481 extern "C" LLVMRustResult
482 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
483 LLVMModuleRef M, const char *Path, const char *DwoPath,
484 LLVMRustFileType RustFileType) {
485 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
486 auto FileType = fromRust(RustFileType);
488 std::string ErrorInfo;
490 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
492 ErrorInfo = EC.message();
493 if (ErrorInfo != "") {
494 LLVMRustSetLastError(ErrorInfo.c_str());
495 return LLVMRustResult::Failure;
498 buffer_ostream BOS(OS);
500 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
503 ErrorInfo = EC.message();
504 if (ErrorInfo != "") {
505 LLVMRustSetLastError(ErrorInfo.c_str());
506 return LLVMRustResult::Failure;
508 buffer_ostream DBOS(DOS);
509 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
512 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
516 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
517 // stream (OS), so the only real safe place to delete this is here? Don't we
518 // wish this was written in Rust?
519 LLVMDisposePassManager(PMR);
520 return LLVMRustResult::Success;
523 extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
524 const char*, // pass name
525 const char*); // IR name
526 extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
528 std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
529 if (any_isa<const Module *>(WrappedIr))
530 return any_cast<const Module *>(WrappedIr)->getName().str();
531 if (any_isa<const Function *>(WrappedIr))
532 return any_cast<const Function *>(WrappedIr)->getName().str();
533 if (any_isa<const Loop *>(WrappedIr))
534 return any_cast<const Loop *>(WrappedIr)->getName().str();
535 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
536 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
541 void LLVMSelfProfileInitializeCallbacks(
542 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
543 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
544 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
545 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
546 StringRef Pass, llvm::Any Ir) {
547 std::string PassName = Pass.str();
548 std::string IrName = LLVMRustwrappedIrGetName(Ir);
549 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
552 PIC.registerAfterPassCallback(
553 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
554 const PreservedAnalyses &Preserved) {
555 AfterPassCallback(LlvmSelfProfiler);
558 PIC.registerAfterPassInvalidatedCallback(
559 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
560 AfterPassCallback(LlvmSelfProfiler);
563 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
564 StringRef Pass, llvm::Any Ir) {
565 std::string PassName = Pass.str();
566 std::string IrName = LLVMRustwrappedIrGetName(Ir);
567 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
570 PIC.registerAfterAnalysisCallback(
571 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
572 AfterPassCallback(LlvmSelfProfiler);
576 enum class LLVMRustOptStage {
584 struct LLVMRustSanitizerOptions {
585 bool SanitizeAddress;
586 bool SanitizeAddressRecover;
588 bool SanitizeMemoryRecover;
589 int SanitizeMemoryTrackOrigins;
591 bool SanitizeHWAddress;
592 bool SanitizeHWAddressRecover;
595 extern "C" LLVMRustResult
597 LLVMModuleRef ModuleRef,
598 LLVMTargetMachineRef TMRef,
599 LLVMRustPassBuilderOptLevel OptLevelRust,
600 LLVMRustOptStage OptStage,
601 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
602 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
603 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
604 LLVMRustSanitizerOptions *SanitizerOptions,
605 const char *PGOGenPath, const char *PGOUsePath,
606 bool InstrumentCoverage, const char *InstrProfileOutput,
608 const char *PGOSampleUsePath, bool DebugInfoForProfiling,
609 void* LlvmSelfProfiler,
610 LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
611 LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
612 const char *ExtraPasses, size_t ExtraPassesLen,
613 const char *LLVMPlugins, size_t LLVMPluginsLen) {
614 Module *TheModule = unwrap(ModuleRef);
615 TargetMachine *TM = unwrap(TMRef);
616 OptimizationLevel OptLevel = fromRust(OptLevelRust);
619 PipelineTuningOptions PTO;
620 PTO.LoopUnrolling = UnrollLoops;
621 PTO.LoopInterleaving = UnrollLoops;
622 PTO.LoopVectorization = LoopVectorize;
623 PTO.SLPVectorization = SLPVectorize;
624 PTO.MergeFunctions = MergeFunctions;
626 // FIXME: We may want to expose this as an option.
627 bool DebugPassManager = false;
629 PassInstrumentationCallbacks PIC;
630 #if LLVM_VERSION_LT(16, 0)
631 StandardInstrumentations SI(DebugPassManager);
633 StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
635 SI.registerCallbacks(PIC);
637 if (LlvmSelfProfiler){
638 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
641 Optional<PGOOptions> PGOOpt;
643 assert(!PGOUsePath && !PGOSampleUsePath);
644 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
645 PGOOptions::NoCSAction, DebugInfoForProfiling);
646 } else if (PGOUsePath) {
647 assert(!PGOSampleUsePath);
648 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
649 PGOOptions::NoCSAction, DebugInfoForProfiling);
650 } else if (PGOSampleUsePath) {
651 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
652 PGOOptions::NoCSAction, DebugInfoForProfiling);
653 } else if (DebugInfoForProfiling) {
654 PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
655 PGOOptions::NoCSAction, DebugInfoForProfiling);
658 PassBuilder PB(TM, PTO, PGOOpt, &PIC);
659 LoopAnalysisManager LAM;
660 FunctionAnalysisManager FAM;
661 CGSCCAnalysisManager CGAM;
662 ModuleAnalysisManager MAM;
664 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
666 Triple TargetTriple(TheModule->getTargetTriple());
667 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
668 if (DisableSimplifyLibCalls)
669 TLII->disableAllFunctions();
670 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
672 PB.registerModuleAnalyses(MAM);
673 PB.registerCGSCCAnalyses(CGAM);
674 PB.registerFunctionAnalyses(FAM);
675 PB.registerLoopAnalyses(LAM);
676 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
678 // We manually collect pipeline callbacks so we can apply them at O0, where the
679 // PassBuilder does not create a pipeline.
680 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
681 PipelineStartEPCallbacks;
682 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
683 OptimizerLastEPCallbacks;
686 PipelineStartEPCallbacks.push_back(
687 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
688 MPM.addPass(VerifierPass());
693 if (InstrumentGCOV) {
694 PipelineStartEPCallbacks.push_back(
695 [](ModulePassManager &MPM, OptimizationLevel Level) {
696 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
701 if (InstrumentCoverage) {
702 PipelineStartEPCallbacks.push_back(
703 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
704 InstrProfOptions Options;
705 if (InstrProfileOutput) {
706 Options.InstrProfileOutput = InstrProfileOutput;
708 MPM.addPass(InstrProfiling(Options, false));
713 if (SanitizerOptions) {
714 if (SanitizerOptions->SanitizeMemory) {
715 #if LLVM_VERSION_GE(14, 0)
716 MemorySanitizerOptions Options(
717 SanitizerOptions->SanitizeMemoryTrackOrigins,
718 SanitizerOptions->SanitizeMemoryRecover,
719 /*CompileKernel=*/false,
720 /*EagerChecks=*/true);
722 MemorySanitizerOptions Options(
723 SanitizerOptions->SanitizeMemoryTrackOrigins,
724 SanitizerOptions->SanitizeMemoryRecover,
725 /*CompileKernel=*/false);
727 OptimizerLastEPCallbacks.push_back(
728 [Options](ModulePassManager &MPM, OptimizationLevel Level) {
729 #if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
730 MPM.addPass(ModuleMemorySanitizerPass(Options));
732 MPM.addPass(MemorySanitizerPass(Options));
734 #if LLVM_VERSION_LT(16, 0)
735 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
741 if (SanitizerOptions->SanitizeThread) {
742 OptimizerLastEPCallbacks.push_back(
743 [](ModulePassManager &MPM, OptimizationLevel Level) {
744 #if LLVM_VERSION_GE(14, 0)
745 MPM.addPass(ModuleThreadSanitizerPass());
747 MPM.addPass(ThreadSanitizerPass());
749 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
754 if (SanitizerOptions->SanitizeAddress) {
755 OptimizerLastEPCallbacks.push_back(
756 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
757 #if LLVM_VERSION_LT(15, 0)
758 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
760 #if LLVM_VERSION_GE(14, 0)
761 AddressSanitizerOptions opts = AddressSanitizerOptions{
762 /*CompileKernel=*/false,
763 SanitizerOptions->SanitizeAddressRecover,
764 /*UseAfterScope=*/true,
765 AsanDetectStackUseAfterReturnMode::Runtime,
767 #if LLVM_VERSION_LT(16, 0)
768 MPM.addPass(ModuleAddressSanitizerPass(opts));
770 MPM.addPass(AddressSanitizerPass(opts));
773 MPM.addPass(ModuleAddressSanitizerPass(
774 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
775 MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
776 /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
777 /*UseAfterScope=*/true)));
782 if (SanitizerOptions->SanitizeHWAddress) {
783 OptimizerLastEPCallbacks.push_back(
784 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
785 #if LLVM_VERSION_GE(14, 0)
786 HWAddressSanitizerOptions opts(
787 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
788 /*DisableOptimization=*/false);
789 MPM.addPass(HWAddressSanitizerPass(opts));
791 MPM.addPass(HWAddressSanitizerPass(
792 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
799 if (LLVMPluginsLen) {
800 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
801 SmallVector<StringRef> Plugins;
802 PluginsStr.split(Plugins, ',', -1, false);
803 for (auto PluginPath: Plugins) {
804 auto Plugin = PassPlugin::Load(PluginPath.str());
806 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
807 return LLVMRustResult::Failure;
809 Plugin->registerPassBuilderCallbacks(PB);
813 ModulePassManager MPM;
814 bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
815 if (!NoPrepopulatePasses) {
816 // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead.
817 // At the same time, the LTO pipelines do support O0 and using them is required.
818 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
819 if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
820 for (const auto &C : PipelineStartEPCallbacks)
821 PB.registerPipelineStartEPCallback(C);
822 for (const auto &C : OptimizerLastEPCallbacks)
823 PB.registerOptimizerLastEPCallback(C);
825 // Pass false as we manually schedule ThinLTOBufferPasses below.
826 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
828 for (const auto &C : PipelineStartEPCallbacks)
829 PB.registerPipelineStartEPCallback(C);
830 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
831 for (const auto &C : OptimizerLastEPCallbacks)
832 PB.registerOptimizerLastEPCallback(C);
836 case LLVMRustOptStage::PreLinkNoLTO:
837 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
839 case LLVMRustOptStage::PreLinkThinLTO:
840 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
841 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
842 // passes may still run afterwards. This means we need to run the buffer passes again.
843 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
844 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
845 if (OptimizerLastEPCallbacks.empty())
846 NeedThinLTOBufferPasses = false;
847 for (const auto &C : OptimizerLastEPCallbacks)
850 case LLVMRustOptStage::PreLinkFatLTO:
851 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
852 NeedThinLTOBufferPasses = false;
854 case LLVMRustOptStage::ThinLTO:
855 // FIXME: Does it make sense to pass the ModuleSummaryIndex?
856 // It only seems to be needed for C++ specific optimizations.
857 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
859 case LLVMRustOptStage::FatLTO:
860 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
865 // We're not building any of the default pipelines but we still want to
866 // add the verifier, instrumentation, etc passes if they were requested
867 for (const auto &C : PipelineStartEPCallbacks)
869 for (const auto &C : OptimizerLastEPCallbacks)
873 if (ExtraPassesLen) {
874 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
875 std::string ErrMsg = toString(std::move(Err));
876 LLVMRustSetLastError(ErrMsg.c_str());
877 return LLVMRustResult::Failure;
881 if (NeedThinLTOBufferPasses) {
882 MPM.addPass(CanonicalizeAliasesPass());
883 MPM.addPass(NameAnonGlobalPass());
886 // Upgrade all calls to old intrinsics first.
887 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
888 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
890 MPM.run(*TheModule, MAM);
891 return LLVMRustResult::Success;
894 // Callback to demangle function name
896 // * name to be demangled
899 // * output buffer len
900 // Returns len of demangled string, or 0 if demangle failed.
901 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
906 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
908 std::vector<char> Buf;
911 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
913 // Return empty string if demangle failed
914 // or if name does not need to be demangled
915 StringRef CallDemangle(StringRef name) {
920 if (Buf.size() < name.size() * 2) {
921 // Semangled name usually shorter than mangled,
922 // but allocate twice as much memory just in case
923 Buf.resize(name.size() * 2);
926 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
932 auto Demangled = StringRef(Buf.data(), R);
933 if (Demangled == name) {
934 // Do not print anything if demangled name is equal to mangled.
941 void emitFunctionAnnot(const Function *F,
942 formatted_raw_ostream &OS) override {
943 StringRef Demangled = CallDemangle(F->getName());
944 if (Demangled.empty()) {
948 OS << "; " << Demangled << "\n";
951 void emitInstructionAnnot(const Instruction *I,
952 formatted_raw_ostream &OS) override {
955 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
957 Value = CI->getCalledOperand();
958 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
960 Value = II->getCalledOperand();
962 // Could demangle more operations, e. g.
963 // `store %place, @function`.
967 if (!Value->hasName()) {
971 StringRef Demangled = CallDemangle(Value->getName());
972 if (Demangled.empty()) {
976 OS << "; " << Name << " " << Demangled << "\n";
982 extern "C" LLVMRustResult
983 LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
984 std::string ErrorInfo;
986 raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
988 ErrorInfo = EC.message();
989 if (ErrorInfo != "") {
990 LLVMRustSetLastError(ErrorInfo.c_str());
991 return LLVMRustResult::Failure;
994 RustAssemblyAnnotationWriter AAW(Demangle);
995 formatted_raw_ostream FOS(OS);
996 unwrap(M)->print(FOS, &AAW);
998 return LLVMRustResult::Success;
1001 extern "C" void LLVMRustPrintPasses() {
1002 LLVMInitializePasses();
1003 struct MyListener : PassRegistrationListener {
1004 void passEnumerate(const PassInfo *Info) {
1005 StringRef PassArg = Info->getPassArgument();
1006 StringRef PassName = Info->getPassName();
1007 if (!PassArg.empty()) {
1008 // These unsigned->signed casts could theoretically overflow, but
1009 // realistically never will (and even if, the result is implementation
1010 // defined rather plain UB).
1011 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1012 (int)PassName.size(), PassName.data());
1017 PassRegistry *PR = PassRegistry::getPassRegistry();
1018 PR->enumerateWith(&Listener);
1021 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1023 auto PreserveFunctions = [=](const GlobalValue &GV) {
1024 for (size_t I = 0; I < Len; I++) {
1025 if (GV.getName() == Symbols[I]) {
1032 internalizeModule(*unwrap(M), PreserveFunctions);
1036 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1037 LLVMTargetMachineRef TMR) {
1038 TargetMachine *Target = unwrap(TMR);
1039 unwrap(Module)->setDataLayout(Target->createDataLayout());
1042 extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1043 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1046 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1047 unwrap(M)->setPIELevel(PIELevel::Level::Large);
1050 extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1051 LLVMRustCodeModel Model) {
1052 auto CM = fromRust(Model);
1055 unwrap(M)->setCodeModel(*CM);
1058 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
1059 // right now. This ThinLTO support is only enabled on "recent ish" versions of
1060 // LLVM, and otherwise it's just blanket rejected from other compilers.
1062 // Most of this implementation is straight copied from LLVM. At the time of
1063 // this writing it wasn't *quite* suitable to reuse more code from upstream
1064 // for our purposes, but we should strive to upstream this support once it's
1065 // ready to go! I figure we may want a bit of testing locally first before
1066 // sending this upstream to LLVM. I hear though they're quite eager to receive
1067 // feedback like this!
1069 // If you're reading this code and wondering "what in the world" or you're
1070 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
1071 // then fear not! (ok maybe fear a little). All code here is mostly based
1072 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1074 // You'll find that the general layout here roughly corresponds to the `run`
1075 // method in that file as well as `ProcessThinLTOModule`. Functions are
1076 // specifically commented below as well, but if you're updating this code
1077 // or otherwise trying to understand it, the LLVM source will be useful in
1078 // interpreting the mysteries within.
1080 // Otherwise I'll apologize in advance, it probably requires a relatively
1081 // significant investment on your part to "truly understand" what's going on
1082 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
1083 // and various online resources about ThinLTO to make heads or tails of all
1086 // This is a shared data structure which *must* be threadsafe to share
1087 // read-only amongst threads. This also corresponds basically to the arguments
1088 // of the `ProcessThinLTOModule` function in the LLVM source.
1089 struct LLVMRustThinLTOData {
1090 // The combined index that is the global analysis over all modules we're
1091 // performing ThinLTO for. This is mostly managed by LLVM.
1092 ModuleSummaryIndex Index;
1094 // All modules we may look at, stored as in-memory serialized versions. This
1095 // is later used when inlining to ensure we can extract any module to inline
1097 StringMap<MemoryBufferRef> ModuleMap;
1099 // A set that we manage of everything we *don't* want internalized. Note that
1100 // this includes all transitive references right now as well, but it may not
1102 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1104 // Not 100% sure what these are, but they impact what's internalized and
1105 // what's inlined across modules, I believe.
1106 StringMap<FunctionImporter::ImportMapTy> ImportLists;
1107 StringMap<FunctionImporter::ExportSetTy> ExportLists;
1108 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1109 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1111 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1114 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
1115 struct LLVMRustThinLTOModule {
1116 const char *identifier;
1121 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1123 static const GlobalValueSummary *
1124 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1125 auto StrongDefForLinker = llvm::find_if(
1126 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1127 auto Linkage = Summary->linkage();
1128 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1129 !GlobalValue::isWeakForLinker(Linkage);
1131 if (StrongDefForLinker != GVSummaryList.end())
1132 return StrongDefForLinker->get();
1134 auto FirstDefForLinker = llvm::find_if(
1135 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1136 auto Linkage = Summary->linkage();
1137 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1139 if (FirstDefForLinker == GVSummaryList.end())
1141 return FirstDefForLinker->get();
1144 // The main entry point for creating the global ThinLTO analysis. The structure
1145 // here is basically the same as before threads are spawned in the `run`
1146 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1147 extern "C" LLVMRustThinLTOData*
1148 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1150 const char **preserved_symbols,
1152 auto Ret = std::make_unique<LLVMRustThinLTOData>();
1154 // Load each module's summary and merge it into one combined index
1155 for (int i = 0; i < num_modules; i++) {
1156 auto module = &modules[i];
1157 StringRef buffer(module->data, module->len);
1158 MemoryBufferRef mem_buffer(buffer, module->identifier);
1160 Ret->ModuleMap[module->identifier] = mem_buffer;
1162 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1163 LLVMRustSetLastError(toString(std::move(Err)).c_str());
1168 // Collect for each module the list of function it defines (GUID -> Summary)
1169 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1171 // Convert the preserved symbols set from string to GUID, this is then needed
1172 // for internalization.
1173 for (int i = 0; i < num_symbols; i++) {
1174 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1175 Ret->GUIDPreservedSymbols.insert(GUID);
1178 // Collect the import/export lists for all modules from the call-graph in the
1181 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1182 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1183 return PrevailingType::Unknown;
1185 // We don't have a complete picture in our use of ThinLTO, just our immediate
1186 // crate, so we need `ImportEnabled = false` to limit internalization.
1187 // Otherwise, we sometimes lose `static` values -- see #60184.
1188 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1189 deadIsPrevailing, /* ImportEnabled = */ false);
1190 ComputeCrossModuleImport(
1192 Ret->ModuleToDefinedGVSummaries,
1197 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1198 // impacts the caching.
1200 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1201 // being lifted from `lib/LTO/LTO.cpp` as well
1202 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1203 for (auto &I : Ret->Index) {
1204 if (I.second.SummaryList.size() > 1)
1205 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1207 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1208 const auto &Prevailing = PrevailingCopy.find(GUID);
1209 if (Prevailing == PrevailingCopy.end())
1211 return Prevailing->second == S;
1213 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1214 GlobalValue::GUID GUID,
1215 GlobalValue::LinkageTypes NewLinkage) {
1216 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1219 // Uses FromPrevailing visibility scheme which works for many binary
1220 // formats. We probably could and should use ELF visibility scheme for many of
1221 // our targets, however.
1223 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1224 Ret->GUIDPreservedSymbols);
1226 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1227 // callback below. This callback below will dictate the linkage for all
1228 // summaries in the index, and we basically just only want to ensure that dead
1229 // symbols are internalized. Otherwise everything that's already external
1230 // linkage will stay as external, and internal will stay as internal.
1231 std::set<GlobalValue::GUID> ExportedGUIDs;
1232 for (auto &List : Ret->Index) {
1233 for (auto &GVS: List.second.SummaryList) {
1234 if (GlobalValue::isLocalLinkage(GVS->linkage()))
1236 auto GUID = GVS->getOriginalName();
1237 if (GVS->flags().Live)
1238 ExportedGUIDs.insert(GUID);
1241 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1242 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1243 return (ExportList != Ret->ExportLists.end() &&
1244 ExportList->second.count(VI)) ||
1245 ExportedGUIDs.count(VI.getGUID());
1247 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1249 return Ret.release();
1253 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1257 // Below are the various passes that happen *per module* when doing ThinLTO.
1259 // In other words, these are the functions that are all run concurrently
1260 // with one another, one per module. The passes here correspond to the analysis
1261 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1262 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1263 // so rustc can save off the intermediate bytecode between each step.
1266 clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1267 // When linking an ELF shared object, dso_local should be dropped. We
1268 // conservatively do this for -fpic.
1269 bool ClearDSOLocalOnDeclarations =
1270 TM.getTargetTriple().isOSBinFormatELF() &&
1271 TM.getRelocationModel() != Reloc::Static &&
1272 Mod.getPIELevel() == PIELevel::Default;
1273 return ClearDSOLocalOnDeclarations;
1277 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1278 LLVMTargetMachineRef TM) {
1279 Module &Mod = *unwrap(M);
1280 TargetMachine &Target = *unwrap(TM);
1282 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1283 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1286 LLVMRustSetLastError("renameModuleForThinLTO failed");
1293 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1294 Module &Mod = *unwrap(M);
1295 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1296 #if LLVM_VERSION_GE(14, 0)
1297 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1299 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1305 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1306 Module &Mod = *unwrap(M);
1307 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1308 thinLTOInternalizeModule(Mod, DefinedGlobals);
1313 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1314 LLVMTargetMachineRef TM) {
1315 Module &Mod = *unwrap(M);
1316 TargetMachine &Target = *unwrap(TM);
1318 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1319 auto Loader = [&](StringRef Identifier) {
1320 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1321 auto &Context = Mod.getContext();
1322 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1327 // The rest of this closure is a workaround for
1328 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1329 // we accidentally import wasm custom sections into different modules,
1330 // duplicating them by in the final output artifact.
1332 // The issue is worked around here by manually removing the
1333 // `wasm.custom_sections` named metadata node from any imported module. This
1334 // we know isn't used by any optimization pass so there's no need for it to
1337 // Note that the metadata is currently lazily loaded, so we materialize it
1338 // here before looking up if there's metadata inside. The `FunctionImporter`
1339 // will immediately materialize metadata anyway after an import, so this
1340 // shouldn't be a perf hit.
1341 if (Error Err = (*MOrErr)->materializeMetadata()) {
1342 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1346 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1347 if (WasmCustomSections)
1348 WasmCustomSections->eraseFromParent();
1352 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1353 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1354 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1356 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1362 // This struct and various functions are sort of a hack right now, but the
1363 // problem is that we've got in-memory LLVM modules after we generate and
1364 // optimize all codegen-units for one compilation in rustc. To be compatible
1365 // with the LTO support above we need to serialize the modules plus their
1366 // ThinLTO summary into memory.
1368 // This structure is basically an owned version of a serialize module, with
1369 // a ThinLTO summary attached.
1370 struct LLVMRustThinLTOBuffer {
1374 extern "C" LLVMRustThinLTOBuffer*
1375 LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1376 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1378 raw_string_ostream OS(Ret->data);
1382 LoopAnalysisManager LAM;
1383 FunctionAnalysisManager FAM;
1384 CGSCCAnalysisManager CGAM;
1385 ModuleAnalysisManager MAM;
1386 PB.registerModuleAnalyses(MAM);
1387 PB.registerCGSCCAnalyses(CGAM);
1388 PB.registerFunctionAnalyses(FAM);
1389 PB.registerLoopAnalyses(LAM);
1390 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1391 ModulePassManager MPM;
1392 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1393 MPM.run(*unwrap(M), MAM);
1395 WriteBitcodeToFile(*unwrap(M), OS);
1399 return Ret.release();
1403 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1407 extern "C" const void*
1408 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1409 return Buffer->data.data();
1413 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1414 return Buffer->data.length();
1417 // This is what we used to parse upstream bitcode for actual ThinLTO
1418 // processing. We'll call this once per module optimized through ThinLTO, and
1419 // it'll be called concurrently on many threads.
1420 extern "C" LLVMModuleRef
1421 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1424 const char *identifier) {
1425 StringRef Data(data, len);
1426 MemoryBufferRef Buffer(Data, identifier);
1427 unwrap(Context)->enableDebugTypeODRUniquing();
1428 Expected<std::unique_ptr<Module>> SrcOrError =
1429 parseBitcodeFile(Buffer, *unwrap(Context));
1431 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1434 return wrap(std::move(*SrcOrError).release());
1437 // Find the bitcode section in the object file data and return it as a slice.
1438 // Fail if the bitcode section is present but empty.
1440 // On success, the return value is the pointer to the start of the slice and
1441 // `out_len` is filled with the (non-zero) length. On failure, the return value
1442 // is `nullptr` and `out_len` is set to zero.
1443 extern "C" const char*
1444 LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1449 StringRef Data(data, len);
1450 MemoryBufferRef Buffer(Data, ""); // The id is unused.
1452 Expected<MemoryBufferRef> BitcodeOrError =
1453 object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1454 if (!BitcodeOrError) {
1455 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1459 *out_len = BitcodeOrError->getBufferSize();
1460 return BitcodeOrError->getBufferStart();
1463 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1464 // the comment in `back/lto.rs` for why this exists.
1466 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1468 DICompileUnit **B) {
1469 Module *M = unwrap(Mod);
1470 DICompileUnit **Cur = A;
1471 DICompileUnit **Next = B;
1472 for (DICompileUnit *CU : M->debug_compile_units()) {
1481 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1482 // the comment in `back/lto.rs` for why this exists.
1484 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1485 Module *M = unwrap(Mod);
1487 // If the original source module didn't have a `DICompileUnit` then try to
1488 // merge all the existing compile units. If there aren't actually any though
1489 // then there's not much for us to do so return.
1490 if (Unit == nullptr) {
1491 for (DICompileUnit *CU : M->debug_compile_units()) {
1495 if (Unit == nullptr)
1499 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1500 // process it recursively. Note that we used to specifically iterate over
1501 // instructions to ensure we feed everything into it, but `processModule`
1502 // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
1503 DebugInfoFinder Finder;
1504 Finder.processModule(*M);
1506 // After we've found all our debuginfo, rewrite all subprograms to point to
1507 // the same `DICompileUnit`.
1508 for (auto &F : Finder.subprograms()) {
1509 F->replaceUnit(Unit);
1512 // Erase any other references to other `DICompileUnit` instances, the verifier
1513 // will later ensure that we don't actually have any other stale references to
1515 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1516 MD->clearOperands();
1517 MD->addOperand(Unit);
1520 // Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1521 // storing the result in 'KeyOut'.
1522 // Currently, this cache key is a SHA-1 hash of anything that could affect
1523 // the result of optimizing this module (e.g. module imports, exports, liveness
1524 // of access globals, etc).
1525 // The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1526 // used during the normal linker-plugin incremental thin-LTO process.
1528 LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1529 SmallString<40> Key;
1530 llvm::lto::Config conf;
1531 const auto &ImportList = Data->ImportLists.lookup(ModId);
1532 const auto &ExportList = Data->ExportLists.lookup(ModId);
1533 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1534 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1535 std::set<GlobalValue::GUID> CfiFunctionDefs;
1536 std::set<GlobalValue::GUID> CfiFunctionDecls;
1538 // Based on the 'InProcessThinBackend' constructor in LLVM
1539 for (auto &Name : Data->Index.cfiFunctionDefs())
1540 CfiFunctionDefs.insert(
1541 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1542 for (auto &Name : Data->Index.cfiFunctionDecls())
1543 CfiFunctionDecls.insert(
1544 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1546 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1547 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1550 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());