8 #include "llvm/Analysis/TargetLibraryInfo.h"
9 #include "llvm/Analysis/TargetTransformInfo.h"
10 #include "llvm/CodeGen/TargetSubtargetInfo.h"
11 #include "llvm/IR/AutoUpgrade.h"
12 #include "llvm/IR/AssemblyAnnotationWriter.h"
13 #include "llvm/IR/IntrinsicInst.h"
14 #include "llvm/Support/CBindingWrapping.h"
15 #include "llvm/Support/FileSystem.h"
16 #include "llvm/Support/Host.h"
17 #include "llvm/Target/TargetMachine.h"
18 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
19 #include "llvm/Transforms/IPO/AlwaysInliner.h"
20 #include "llvm/Transforms/IPO/FunctionImport.h"
21 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
22 #include "llvm/LTO/LTO.h"
24 #include "llvm-c/Transforms/PassManagerBuilder.h"
27 using namespace llvm::legacy;
29 extern cl::opt<bool> EnableARMEHABI;
31 typedef struct LLVMOpaquePass *LLVMPassRef;
32 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
34 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
35 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
36 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
37 LLVMPassManagerBuilderRef)
39 extern "C" void LLVMInitializePasses() {
40 PassRegistry &Registry = *PassRegistry::getPassRegistry();
41 initializeCore(Registry);
42 initializeCodeGen(Registry);
43 initializeScalarOpts(Registry);
44 initializeVectorization(Registry);
45 initializeIPO(Registry);
46 initializeAnalysis(Registry);
47 initializeTransformUtils(Registry);
48 initializeInstCombine(Registry);
49 initializeInstrumentation(Registry);
50 initializeTarget(Registry);
53 enum class LLVMRustPassKind {
59 static LLVMRustPassKind toRust(PassKind Kind) {
62 return LLVMRustPassKind::Function;
64 return LLVMRustPassKind::Module;
66 return LLVMRustPassKind::Other;
70 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
71 StringRef SR(PassName);
72 PassRegistry *PR = PassRegistry::getPassRegistry();
74 const PassInfo *PI = PR->getPassInfo(SR);
76 return wrap(PI->createPass());
81 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
83 Pass *Pass = unwrap(RustPass);
84 return toRust(Pass->getPassKind());
87 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
89 Pass *Pass = unwrap(RustPass);
90 PassManagerBase *PMB = unwrap(PMR);
95 void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
96 LLVMPassManagerBuilderRef PMBR,
97 LLVMPassManagerRef PMR
99 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
102 #ifdef LLVM_COMPONENT_X86
103 #define SUBTARGET_X86 SUBTARGET(X86)
105 #define SUBTARGET_X86
108 #ifdef LLVM_COMPONENT_ARM
109 #define SUBTARGET_ARM SUBTARGET(ARM)
111 #define SUBTARGET_ARM
114 #ifdef LLVM_COMPONENT_AARCH64
115 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
117 #define SUBTARGET_AARCH64
120 #ifdef LLVM_COMPONENT_MIPS
121 #define SUBTARGET_MIPS SUBTARGET(Mips)
123 #define SUBTARGET_MIPS
126 #ifdef LLVM_COMPONENT_POWERPC
127 #define SUBTARGET_PPC SUBTARGET(PPC)
129 #define SUBTARGET_PPC
132 #ifdef LLVM_COMPONENT_SYSTEMZ
133 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
135 #define SUBTARGET_SYSTEMZ
138 #ifdef LLVM_COMPONENT_MSP430
139 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
141 #define SUBTARGET_MSP430
144 #ifdef LLVM_COMPONENT_RISCV
145 #define SUBTARGET_RISCV SUBTARGET(RISCV)
147 #define SUBTARGET_RISCV
150 #ifdef LLVM_COMPONENT_SPARC
151 #define SUBTARGET_SPARC SUBTARGET(Sparc)
153 #define SUBTARGET_SPARC
156 #ifdef LLVM_COMPONENT_HEXAGON
157 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
159 #define SUBTARGET_HEXAGON
162 #define GEN_SUBTARGETS \
174 #define SUBTARGET(x) \
176 extern const SubtargetFeatureKV x##FeatureKV[]; \
177 extern const SubtargetFeatureKV x##SubTypeKV[]; \
183 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
184 const char *Feature) {
185 TargetMachine *Target = unwrap(TM);
186 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
187 return MCInfo->checkFeatures(std::string("+") + Feature);
190 enum class LLVMRustCodeModel {
199 static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
201 case LLVMRustCodeModel::Small:
202 return CodeModel::Small;
203 case LLVMRustCodeModel::Kernel:
204 return CodeModel::Kernel;
205 case LLVMRustCodeModel::Medium:
206 return CodeModel::Medium;
207 case LLVMRustCodeModel::Large:
208 return CodeModel::Large;
210 report_fatal_error("Bad CodeModel.");
214 enum class LLVMRustCodeGenOptLevel {
222 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
224 case LLVMRustCodeGenOptLevel::None:
225 return CodeGenOpt::None;
226 case LLVMRustCodeGenOptLevel::Less:
227 return CodeGenOpt::Less;
228 case LLVMRustCodeGenOptLevel::Default:
229 return CodeGenOpt::Default;
230 case LLVMRustCodeGenOptLevel::Aggressive:
231 return CodeGenOpt::Aggressive;
233 report_fatal_error("Bad CodeGenOptLevel.");
237 enum class LLVMRustRelocMode {
247 static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
249 case LLVMRustRelocMode::Default:
251 case LLVMRustRelocMode::Static:
252 return Reloc::Static;
253 case LLVMRustRelocMode::PIC:
255 case LLVMRustRelocMode::DynamicNoPic:
256 return Reloc::DynamicNoPIC;
257 case LLVMRustRelocMode::ROPI:
259 case LLVMRustRelocMode::RWPI:
261 case LLVMRustRelocMode::ROPIRWPI:
262 return Reloc::ROPI_RWPI;
264 report_fatal_error("Bad RelocModel.");
268 /// getLongestEntryLength - Return the length of the longest entry in the table.
270 static size_t getLongestEntryLength(ArrayRef<SubtargetFeatureKV> Table) {
272 for (auto &I : Table)
273 MaxLen = std::max(MaxLen, std::strlen(I.Key));
277 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
278 const TargetMachine *Target = unwrap(TM);
279 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
280 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
281 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
282 const ArrayRef<SubtargetFeatureKV> CPUTable = MCInfo->getCPUTable();
283 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
285 printf("Available CPUs for this target:\n");
286 if (HostArch == TargetArch) {
287 const StringRef HostCPU = sys::getHostCPUName();
288 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
289 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
291 for (auto &CPU : CPUTable)
292 printf(" %-*s - %s.\n", MaxCPULen, CPU.Key, CPU.Desc);
296 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
297 const TargetMachine *Target = unwrap(TM);
298 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
299 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
300 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
302 printf("Available features for this target:\n");
303 for (auto &Feature : FeatTable)
304 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
307 printf("Use +feature to enable a feature, or -feature to disable it.\n"
308 "For example, rustc -C -target-cpu=mycpu -C "
309 "target-feature=+feature1,-feature2\n\n");
314 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
315 printf("Target CPU help is not supported by this LLVM version.\n\n");
318 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
319 printf("Target features help is not supported by this LLVM version.\n\n");
323 extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
324 StringRef Name = sys::getHostCPUName();
329 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
330 const char *TripleStr, const char *CPU, const char *Feature,
331 LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
332 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
333 bool PositionIndependentExecutable, bool FunctionSections,
335 bool TrapUnreachable,
338 bool EmitStackSizeSection) {
340 auto OptLevel = fromRust(RustOptLevel);
341 auto RM = fromRust(RustReloc);
344 Triple Trip(Triple::normalize(TripleStr));
345 const llvm::Target *TheTarget =
346 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
347 if (TheTarget == nullptr) {
348 LLVMRustSetLastError(Error.c_str());
352 TargetOptions Options;
354 Options.FloatABIType = FloatABI::Default;
356 Options.FloatABIType = FloatABI::Soft;
358 Options.DataSections = DataSections;
359 Options.FunctionSections = FunctionSections;
360 Options.MCOptions.AsmVerbose = AsmComments;
361 Options.MCOptions.PreserveAsmComments = AsmComments;
363 if (TrapUnreachable) {
364 // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
365 // This limits the extent of possible undefined behavior in some cases, as
366 // it prevents control flow from "falling through" into whatever code
367 // happens to be laid out next in memory.
368 Options.TrapUnreachable = true;
372 Options.ThreadModel = ThreadModel::Single;
375 Options.EmitStackSizeSection = EmitStackSizeSection;
377 Optional<CodeModel::Model> CM;
378 if (RustCM != LLVMRustCodeModel::None)
379 CM = fromRust(RustCM);
380 TargetMachine *TM = TheTarget->createTargetMachine(
381 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
385 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
389 // Unfortunately, LLVM doesn't expose a C API to add the corresponding analysis
390 // passes for a target to a pass manager. We export that functionality through
392 extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM,
393 LLVMPassManagerRef PMR,
395 PassManagerBase *PM = unwrap(PMR);
397 createTargetTransformInfoWrapperPass(unwrap(TM)->getTargetIRAnalysis()));
400 extern "C" void LLVMRustConfigurePassManagerBuilder(
401 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
402 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
403 const char* PGOGenPath, const char* PGOUsePath) {
404 #if LLVM_VERSION_GE(7, 0)
405 unwrap(PMBR)->MergeFunctions = MergeFunctions;
407 unwrap(PMBR)->SLPVectorize = SLPVectorize;
408 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
409 unwrap(PMBR)->LoopVectorize = LoopVectorize;
410 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
414 unwrap(PMBR)->EnablePGOInstrGen = true;
415 unwrap(PMBR)->PGOInstrGen = PGOGenPath;
419 unwrap(PMBR)->PGOInstrUse = PGOUsePath;
423 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
424 // field of a PassManagerBuilder, we expose our own method of doing so.
425 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
427 bool DisableSimplifyLibCalls) {
428 Triple TargetTriple(unwrap(M)->getTargetTriple());
429 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
430 if (DisableSimplifyLibCalls)
431 TLI->disableAllFunctions();
432 unwrap(PMBR)->LibraryInfo = TLI;
435 // Unfortunately, the LLVM C API doesn't provide a way to create the
436 // TargetLibraryInfo pass, so we use this method to do so.
437 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
438 bool DisableSimplifyLibCalls) {
439 Triple TargetTriple(unwrap(M)->getTargetTriple());
440 TargetLibraryInfoImpl TLII(TargetTriple);
441 if (DisableSimplifyLibCalls)
442 TLII.disableAllFunctions();
443 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
446 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
447 // all the functions in a module, so we do that manually here. You'll find
448 // similar code in clang's BackendUtil.cpp file.
449 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
451 llvm::legacy::FunctionPassManager *P =
452 unwrap<llvm::legacy::FunctionPassManager>(PMR);
453 P->doInitialization();
455 // Upgrade all calls to old intrinsics first.
456 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
457 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
459 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
461 if (!I->isDeclaration())
467 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
468 // Initializing the command-line options more than once is not allowed. So,
469 // check if they've already been initialized. (This could happen if we're
470 // being called from rustpkg, for example). If the arguments change, then
471 // that's just kinda unfortunate.
472 static bool Initialized = false;
476 cl::ParseCommandLineOptions(Argc, Argv);
479 enum class LLVMRustFileType {
485 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
487 case LLVMRustFileType::AssemblyFile:
488 return TargetMachine::CGFT_AssemblyFile;
489 case LLVMRustFileType::ObjectFile:
490 return TargetMachine::CGFT_ObjectFile;
492 report_fatal_error("Bad FileType.");
496 extern "C" LLVMRustResult
497 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
498 LLVMModuleRef M, const char *Path,
499 LLVMRustFileType RustFileType) {
500 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
501 auto FileType = fromRust(RustFileType);
503 std::string ErrorInfo;
505 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
507 ErrorInfo = EC.message();
508 if (ErrorInfo != "") {
509 LLVMRustSetLastError(ErrorInfo.c_str());
510 return LLVMRustResult::Failure;
513 #if LLVM_VERSION_GE(7, 0)
514 buffer_ostream BOS(OS);
515 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
517 unwrap(Target)->addPassesToEmitFile(*PM, OS, FileType, false);
521 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
522 // stream (OS), so the only real safe place to delete this is here? Don't we
523 // wish this was written in Rust?
525 return LLVMRustResult::Success;
529 // Callback to demangle function name
531 // * name to be demangled
534 // * output buffer len
535 // Returns len of demangled string, or 0 if demangle failed.
536 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
541 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
543 std::vector<char> Buf;
546 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
548 // Return empty string if demangle failed
549 // or if name does not need to be demangled
550 StringRef CallDemangle(StringRef name) {
555 if (Buf.size() < name.size() * 2) {
556 // Semangled name usually shorter than mangled,
557 // but allocate twice as much memory just in case
558 Buf.resize(name.size() * 2);
561 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
567 auto Demangled = StringRef(Buf.data(), R);
568 if (Demangled == name) {
569 // Do not print anything if demangled name is equal to mangled.
576 void emitFunctionAnnot(const Function *F,
577 formatted_raw_ostream &OS) override {
578 StringRef Demangled = CallDemangle(F->getName());
579 if (Demangled.empty()) {
583 OS << "; " << Demangled << "\n";
586 void emitInstructionAnnot(const Instruction *I,
587 formatted_raw_ostream &OS) override {
590 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
592 Value = CI->getCalledValue();
593 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
595 Value = II->getCalledValue();
597 // Could demangle more operations, e. g.
598 // `store %place, @function`.
602 if (!Value->hasName()) {
606 StringRef Demangled = CallDemangle(Value->getName());
607 if (Demangled.empty()) {
611 OS << "; " << Name << " " << Demangled << "\n";
615 class RustPrintModulePass : public ModulePass {
620 RustPrintModulePass() : ModulePass(ID), OS(nullptr), Demangle(nullptr) {}
621 RustPrintModulePass(raw_ostream &OS, DemangleFn Demangle)
622 : ModulePass(ID), OS(&OS), Demangle(Demangle) {}
624 bool runOnModule(Module &M) override {
625 RustAssemblyAnnotationWriter AW(Demangle);
627 M.print(*OS, &AW, false);
632 void getAnalysisUsage(AnalysisUsage &AU) const override {
633 AU.setPreservesAll();
636 static StringRef name() { return "RustPrintModulePass"; }
642 void initializeRustPrintModulePassPass(PassRegistry&);
645 char RustPrintModulePass::ID = 0;
646 INITIALIZE_PASS(RustPrintModulePass, "print-rust-module",
647 "Print rust module to stderr", false, false)
649 extern "C" LLVMRustResult
650 LLVMRustPrintModule(LLVMPassManagerRef PMR, LLVMModuleRef M,
651 const char *Path, DemangleFn Demangle) {
652 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
653 std::string ErrorInfo;
656 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
658 ErrorInfo = EC.message();
659 if (ErrorInfo != "") {
660 LLVMRustSetLastError(ErrorInfo.c_str());
661 return LLVMRustResult::Failure;
664 formatted_raw_ostream FOS(OS);
666 PM->add(new RustPrintModulePass(FOS, Demangle));
670 return LLVMRustResult::Success;
673 extern "C" void LLVMRustPrintPasses() {
674 LLVMInitializePasses();
675 struct MyListener : PassRegistrationListener {
676 void passEnumerate(const PassInfo *Info) {
677 StringRef PassArg = Info->getPassArgument();
678 StringRef PassName = Info->getPassName();
679 if (!PassArg.empty()) {
680 // These unsigned->signed casts could theoretically overflow, but
681 // realistically never will (and even if, the result is implementation
682 // defined rather plain UB).
683 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
684 (int)PassName.size(), PassName.data());
689 PassRegistry *PR = PassRegistry::getPassRegistry();
690 PR->enumerateWith(&Listener);
693 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
695 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
698 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
700 llvm::legacy::PassManager passes;
702 auto PreserveFunctions = [=](const GlobalValue &GV) {
703 for (size_t I = 0; I < Len; I++) {
704 if (GV.getName() == Symbols[I]) {
711 passes.add(llvm::createInternalizePass(PreserveFunctions));
713 passes.run(*unwrap(M));
716 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
717 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
719 GV->setDoesNotThrow();
720 Function *F = dyn_cast<Function>(GV);
724 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
725 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
726 if (isa<InvokeInst>(I)) {
727 InvokeInst *CI = cast<InvokeInst>(I);
728 CI->setDoesNotThrow();
736 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
737 LLVMTargetMachineRef TMR) {
738 TargetMachine *Target = unwrap(TMR);
739 unwrap(Module)->setDataLayout(Target->createDataLayout());
742 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
743 unwrap(M)->setPIELevel(PIELevel::Level::Large);
746 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
747 // right now. This ThinLTO support is only enabled on "recent ish" versions of
748 // LLVM, and otherwise it's just blanket rejected from other compilers.
750 // Most of this implementation is straight copied from LLVM. At the time of
751 // this writing it wasn't *quite* suitable to reuse more code from upstream
752 // for our purposes, but we should strive to upstream this support once it's
753 // ready to go! I figure we may want a bit of testing locally first before
754 // sending this upstream to LLVM. I hear though they're quite eager to receive
755 // feedback like this!
757 // If you're reading this code and wondering "what in the world" or you're
758 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
759 // then fear not! (ok maybe fear a little). All code here is mostly based
760 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
762 // You'll find that the general layout here roughly corresponds to the `run`
763 // method in that file as well as `ProcessThinLTOModule`. Functions are
764 // specifically commented below as well, but if you're updating this code
765 // or otherwise trying to understand it, the LLVM source will be useful in
766 // interpreting the mysteries within.
768 // Otherwise I'll apologize in advance, it probably requires a relatively
769 // significant investment on your part to "truly understand" what's going on
770 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
771 // and various online resources about ThinLTO to make heads or tails of all
774 // This is a shared data structure which *must* be threadsafe to share
775 // read-only amongst threads. This also corresponds basically to the arguments
776 // of the `ProcessThinLTOModule` function in the LLVM source.
777 struct LLVMRustThinLTOData {
778 // The combined index that is the global analysis over all modules we're
779 // performing ThinLTO for. This is mostly managed by LLVM.
780 ModuleSummaryIndex Index;
782 // All modules we may look at, stored as in-memory serialized versions. This
783 // is later used when inlining to ensure we can extract any module to inline
785 StringMap<MemoryBufferRef> ModuleMap;
787 // A set that we manage of everything we *don't* want internalized. Note that
788 // this includes all transitive references right now as well, but it may not
790 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
792 // Not 100% sure what these are, but they impact what's internalized and
793 // what's inlined across modules, I believe.
794 StringMap<FunctionImporter::ImportMapTy> ImportLists;
795 StringMap<FunctionImporter::ExportSetTy> ExportLists;
796 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
798 #if LLVM_VERSION_GE(7, 0)
799 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
803 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
804 struct LLVMRustThinLTOModule {
805 const char *identifier;
810 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
812 static const GlobalValueSummary *
813 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
814 auto StrongDefForLinker = llvm::find_if(
815 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
816 auto Linkage = Summary->linkage();
817 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
818 !GlobalValue::isWeakForLinker(Linkage);
820 if (StrongDefForLinker != GVSummaryList.end())
821 return StrongDefForLinker->get();
823 auto FirstDefForLinker = llvm::find_if(
824 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
825 auto Linkage = Summary->linkage();
826 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
828 if (FirstDefForLinker == GVSummaryList.end())
830 return FirstDefForLinker->get();
833 // The main entry point for creating the global ThinLTO analysis. The structure
834 // here is basically the same as before threads are spawned in the `run`
835 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
836 extern "C" LLVMRustThinLTOData*
837 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
839 const char **preserved_symbols,
841 auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
843 // Load each module's summary and merge it into one combined index
844 for (int i = 0; i < num_modules; i++) {
845 auto module = &modules[i];
846 StringRef buffer(module->data, module->len);
847 MemoryBufferRef mem_buffer(buffer, module->identifier);
849 Ret->ModuleMap[module->identifier] = mem_buffer;
851 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
852 LLVMRustSetLastError(toString(std::move(Err)).c_str());
857 // Collect for each module the list of function it defines (GUID -> Summary)
858 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
860 // Convert the preserved symbols set from string to GUID, this is then needed
861 // for internalization.
862 for (int i = 0; i < num_symbols; i++) {
863 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
864 Ret->GUIDPreservedSymbols.insert(GUID);
867 // Collect the import/export lists for all modules from the call-graph in the
870 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
871 #if LLVM_VERSION_GE(7, 0)
872 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
873 return PrevailingType::Unknown;
875 #if LLVM_VERSION_GE(8, 0)
876 // We don't have a complete picture in our use of ThinLTO, just our immediate
877 // crate, so we need `ImportEnabled = false` to limit internalization.
878 // Otherwise, we sometimes lose `static` values -- see #60184.
879 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
880 deadIsPrevailing, /* ImportEnabled = */ false);
882 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
885 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols);
887 ComputeCrossModuleImport(
889 Ret->ModuleToDefinedGVSummaries,
894 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
895 // impacts the caching.
897 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
898 // being lifted from `lib/LTO/LTO.cpp` as well
899 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
900 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
901 for (auto &I : Ret->Index) {
902 if (I.second.SummaryList.size() > 1)
903 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
905 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
906 const auto &Prevailing = PrevailingCopy.find(GUID);
907 if (Prevailing == PrevailingCopy.end())
909 return Prevailing->second == S;
911 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
912 GlobalValue::GUID GUID,
913 GlobalValue::LinkageTypes NewLinkage) {
914 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
916 #if LLVM_VERSION_GE(8, 0)
917 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
919 thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
922 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
923 // callback below. This callback below will dictate the linkage for all
924 // summaries in the index, and we basically just only want to ensure that dead
925 // symbols are internalized. Otherwise everything that's already external
926 // linkage will stay as external, and internal will stay as internal.
927 std::set<GlobalValue::GUID> ExportedGUIDs;
928 for (auto &List : Ret->Index) {
929 for (auto &GVS: List.second.SummaryList) {
930 if (GlobalValue::isLocalLinkage(GVS->linkage()))
932 auto GUID = GVS->getOriginalName();
933 if (GVS->flags().Live)
934 ExportedGUIDs.insert(GUID);
937 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
938 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
939 return (ExportList != Ret->ExportLists.end() &&
940 ExportList->second.count(GUID)) ||
941 ExportedGUIDs.count(GUID);
943 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
945 return Ret.release();
949 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
953 // Below are the various passes that happen *per module* when doing ThinLTO.
955 // In other words, these are the functions that are all run concurrently
956 // with one another, one per module. The passes here correspond to the analysis
957 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
958 // `ProcessThinLTOModule` function. Here they're split up into separate steps
959 // so rustc can save off the intermediate bytecode between each step.
962 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
963 Module &Mod = *unwrap(M);
964 if (renameModuleForThinLTO(Mod, Data->Index)) {
965 LLVMRustSetLastError("renameModuleForThinLTO failed");
972 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
973 Module &Mod = *unwrap(M);
974 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
975 #if LLVM_VERSION_GE(8, 0)
976 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
978 thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
984 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
985 Module &Mod = *unwrap(M);
986 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
987 thinLTOInternalizeModule(Mod, DefinedGlobals);
992 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
993 Module &Mod = *unwrap(M);
995 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
996 auto Loader = [&](StringRef Identifier) {
997 const auto &Memory = Data->ModuleMap.lookup(Identifier);
998 auto &Context = Mod.getContext();
999 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1004 // The rest of this closure is a workaround for
1005 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1006 // we accidentally import wasm custom sections into different modules,
1007 // duplicating them by in the final output artifact.
1009 // The issue is worked around here by manually removing the
1010 // `wasm.custom_sections` named metadata node from any imported module. This
1011 // we know isn't used by any optimization pass so there's no need for it to
1014 // Note that the metadata is currently lazily loaded, so we materialize it
1015 // here before looking up if there's metadata inside. The `FunctionImporter`
1016 // will immediately materialize metadata anyway after an import, so this
1017 // shouldn't be a perf hit.
1018 if (Error Err = (*MOrErr)->materializeMetadata()) {
1019 Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1023 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1024 if (WasmCustomSections)
1025 WasmCustomSections->eraseFromParent();
1029 FunctionImporter Importer(Data->Index, Loader);
1030 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1032 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1038 extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1039 const char*, // importing module name
1040 const char*); // imported module name
1042 // Calls `module_name_callback` for each module import done by ThinLTO.
1043 // The callback is provided with regular null-terminated C strings.
1045 LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1046 LLVMRustModuleNameCallback module_name_callback,
1047 void* callback_payload) {
1048 for (const auto& importing_module : data->ImportLists) {
1049 const std::string importing_module_id = importing_module.getKey().str();
1050 const auto& imports = importing_module.getValue();
1051 for (const auto& imported_module : imports) {
1052 const std::string imported_module_id = imported_module.getKey().str();
1053 module_name_callback(callback_payload,
1054 importing_module_id.c_str(),
1055 imported_module_id.c_str());
1060 // This struct and various functions are sort of a hack right now, but the
1061 // problem is that we've got in-memory LLVM modules after we generate and
1062 // optimize all codegen-units for one compilation in rustc. To be compatible
1063 // with the LTO support above we need to serialize the modules plus their
1064 // ThinLTO summary into memory.
1066 // This structure is basically an owned version of a serialize module, with
1067 // a ThinLTO summary attached.
1068 struct LLVMRustThinLTOBuffer {
1072 extern "C" LLVMRustThinLTOBuffer*
1073 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1074 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1076 raw_string_ostream OS(Ret->data);
1078 legacy::PassManager PM;
1079 PM.add(createWriteThinLTOBitcodePass(OS));
1083 return Ret.release();
1087 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1091 extern "C" const void*
1092 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1093 return Buffer->data.data();
1097 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1098 return Buffer->data.length();
1101 // This is what we used to parse upstream bitcode for actual ThinLTO
1102 // processing. We'll call this once per module optimized through ThinLTO, and
1103 // it'll be called concurrently on many threads.
1104 extern "C" LLVMModuleRef
1105 LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1108 const char *identifier) {
1109 StringRef Data(data, len);
1110 MemoryBufferRef Buffer(Data, identifier);
1111 unwrap(Context)->enableDebugTypeODRUniquing();
1112 Expected<std::unique_ptr<Module>> SrcOrError =
1113 parseBitcodeFile(Buffer, *unwrap(Context));
1115 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1118 return wrap(std::move(*SrcOrError).release());
1121 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1122 // the comment in `back/lto.rs` for why this exists.
1124 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1126 DICompileUnit **B) {
1127 Module *M = unwrap(Mod);
1128 DICompileUnit **Cur = A;
1129 DICompileUnit **Next = B;
1130 for (DICompileUnit *CU : M->debug_compile_units()) {
1139 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1140 // the comment in `back/lto.rs` for why this exists.
1142 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1143 Module *M = unwrap(Mod);
1145 // If the original source module didn't have a `DICompileUnit` then try to
1146 // merge all the existing compile units. If there aren't actually any though
1147 // then there's not much for us to do so return.
1148 if (Unit == nullptr) {
1149 for (DICompileUnit *CU : M->debug_compile_units()) {
1153 if (Unit == nullptr)
1157 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1158 // process it recursively. Note that we specifically iterate over instructions
1159 // to ensure we feed everything into it.
1160 DebugInfoFinder Finder;
1161 Finder.processModule(*M);
1162 for (Function &F : M->functions()) {
1163 for (auto &FI : F) {
1164 for (Instruction &BI : FI) {
1165 if (auto Loc = BI.getDebugLoc())
1166 Finder.processLocation(*M, Loc);
1167 if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1168 Finder.processValue(*M, DVI);
1169 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1170 Finder.processDeclare(*M, DDI);
1175 // After we've found all our debuginfo, rewrite all subprograms to point to
1176 // the same `DICompileUnit`.
1177 for (auto &F : Finder.subprograms()) {
1178 F->replaceUnit(Unit);
1181 // Erase any other references to other `DICompileUnit` instances, the verifier
1182 // will later ensure that we don't actually have any other stale references to
1184 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1185 MD->clearOperands();
1186 MD->addOperand(Unit);