1 // Copyright 2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
18 #include "llvm/Analysis/TargetLibraryInfo.h"
19 #include "llvm/Analysis/TargetTransformInfo.h"
20 #include "llvm/IR/AutoUpgrade.h"
21 #include "llvm/IR/AssemblyAnnotationWriter.h"
22 #include "llvm/Support/CBindingWrapping.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/Host.h"
25 #include "llvm/Target/TargetMachine.h"
26 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
28 #if LLVM_VERSION_GE(6, 0)
29 #include "llvm/CodeGen/TargetSubtargetInfo.h"
30 #include "llvm/IR/IntrinsicInst.h"
32 #include "llvm/Target/TargetSubtargetInfo.h"
35 #if LLVM_VERSION_GE(4, 0)
36 #include "llvm/Transforms/IPO/AlwaysInliner.h"
37 #include "llvm/Transforms/IPO/FunctionImport.h"
38 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
39 #include "llvm/LTO/LTO.h"
40 #if LLVM_VERSION_LE(4, 0)
41 #include "llvm/Object/ModuleSummaryIndexObjectFile.h"
45 #include "llvm-c/Transforms/PassManagerBuilder.h"
48 using namespace llvm::legacy;
50 extern cl::opt<bool> EnableARMEHABI;
52 typedef struct LLVMOpaquePass *LLVMPassRef;
53 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
55 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
56 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
57 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
58 LLVMPassManagerBuilderRef)
60 extern "C" void LLVMInitializePasses() {
61 PassRegistry &Registry = *PassRegistry::getPassRegistry();
62 initializeCore(Registry);
63 initializeCodeGen(Registry);
64 initializeScalarOpts(Registry);
65 initializeVectorization(Registry);
66 initializeIPO(Registry);
67 initializeAnalysis(Registry);
68 initializeTransformUtils(Registry);
69 initializeInstCombine(Registry);
70 initializeInstrumentation(Registry);
71 initializeTarget(Registry);
74 enum class LLVMRustPassKind {
80 static LLVMRustPassKind toRust(PassKind Kind) {
83 return LLVMRustPassKind::Function;
85 return LLVMRustPassKind::Module;
87 return LLVMRustPassKind::Other;
91 extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
92 StringRef SR(PassName);
93 PassRegistry *PR = PassRegistry::getPassRegistry();
95 const PassInfo *PI = PR->getPassInfo(SR);
97 return wrap(PI->createPass());
102 extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
104 Pass *Pass = unwrap(RustPass);
105 return toRust(Pass->getPassKind());
108 extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
110 Pass *Pass = unwrap(RustPass);
111 PassManagerBase *PMB = unwrap(PMR);
116 bool LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
117 LLVMPassManagerBuilderRef PMBR,
118 LLVMPassManagerRef PMR
120 #if LLVM_VERSION_GE(4, 0)
121 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
128 #ifdef LLVM_COMPONENT_X86
129 #define SUBTARGET_X86 SUBTARGET(X86)
131 #define SUBTARGET_X86
134 #ifdef LLVM_COMPONENT_ARM
135 #define SUBTARGET_ARM SUBTARGET(ARM)
137 #define SUBTARGET_ARM
140 #ifdef LLVM_COMPONENT_AARCH64
141 #define SUBTARGET_AARCH64 SUBTARGET(AArch64)
143 #define SUBTARGET_AARCH64
146 #ifdef LLVM_COMPONENT_MIPS
147 #define SUBTARGET_MIPS SUBTARGET(Mips)
149 #define SUBTARGET_MIPS
152 #ifdef LLVM_COMPONENT_POWERPC
153 #define SUBTARGET_PPC SUBTARGET(PPC)
155 #define SUBTARGET_PPC
158 #ifdef LLVM_COMPONENT_SYSTEMZ
159 #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
161 #define SUBTARGET_SYSTEMZ
164 #ifdef LLVM_COMPONENT_MSP430
165 #define SUBTARGET_MSP430 SUBTARGET(MSP430)
167 #define SUBTARGET_MSP430
170 #ifdef LLVM_COMPONENT_SPARC
171 #define SUBTARGET_SPARC SUBTARGET(Sparc)
173 #define SUBTARGET_SPARC
176 #ifdef LLVM_COMPONENT_HEXAGON
177 #define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
179 #define SUBTARGET_HEXAGON
182 #define GEN_SUBTARGETS \
193 #define SUBTARGET(x) \
195 extern const SubtargetFeatureKV x##FeatureKV[]; \
196 extern const SubtargetFeatureKV x##SubTypeKV[]; \
202 extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
203 const char *Feature) {
205 TargetMachine *Target = unwrap(TM);
206 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
207 const FeatureBitset &Bits = MCInfo->getFeatureBits();
208 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
210 for (auto &FeatureEntry : FeatTable)
211 if (!strcmp(FeatureEntry.Key, Feature))
212 return (Bits & FeatureEntry.Value) == FeatureEntry.Value;
217 enum class LLVMRustCodeModel {
226 static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
228 case LLVMRustCodeModel::Small:
229 return CodeModel::Small;
230 case LLVMRustCodeModel::Kernel:
231 return CodeModel::Kernel;
232 case LLVMRustCodeModel::Medium:
233 return CodeModel::Medium;
234 case LLVMRustCodeModel::Large:
235 return CodeModel::Large;
237 report_fatal_error("Bad CodeModel.");
241 enum class LLVMRustCodeGenOptLevel {
249 static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
251 case LLVMRustCodeGenOptLevel::None:
252 return CodeGenOpt::None;
253 case LLVMRustCodeGenOptLevel::Less:
254 return CodeGenOpt::Less;
255 case LLVMRustCodeGenOptLevel::Default:
256 return CodeGenOpt::Default;
257 case LLVMRustCodeGenOptLevel::Aggressive:
258 return CodeGenOpt::Aggressive;
260 report_fatal_error("Bad CodeGenOptLevel.");
264 enum class LLVMRustRelocMode {
274 static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
276 case LLVMRustRelocMode::Default:
278 case LLVMRustRelocMode::Static:
279 return Reloc::Static;
280 case LLVMRustRelocMode::PIC:
282 case LLVMRustRelocMode::DynamicNoPic:
283 return Reloc::DynamicNoPIC;
284 #if LLVM_VERSION_GE(4, 0)
285 case LLVMRustRelocMode::ROPI:
287 case LLVMRustRelocMode::RWPI:
289 case LLVMRustRelocMode::ROPIRWPI:
290 return Reloc::ROPI_RWPI;
296 report_fatal_error("Bad RelocModel.");
300 /// getLongestEntryLength - Return the length of the longest entry in the table.
302 static size_t getLongestEntryLength(ArrayRef<SubtargetFeatureKV> Table) {
304 for (auto &I : Table)
305 MaxLen = std::max(MaxLen, std::strlen(I.Key));
309 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
310 const TargetMachine *Target = unwrap(TM);
311 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
312 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
313 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
314 const ArrayRef<SubtargetFeatureKV> CPUTable = MCInfo->getCPUTable();
315 unsigned MaxCPULen = getLongestEntryLength(CPUTable);
317 printf("Available CPUs for this target:\n");
318 if (HostArch == TargetArch) {
319 const StringRef HostCPU = sys::getHostCPUName();
320 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n",
321 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
323 for (auto &CPU : CPUTable)
324 printf(" %-*s - %s.\n", MaxCPULen, CPU.Key, CPU.Desc);
328 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
329 const TargetMachine *Target = unwrap(TM);
330 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
331 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
332 unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
334 printf("Available features for this target:\n");
335 for (auto &Feature : FeatTable)
336 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
339 printf("Use +feature to enable a feature, or -feature to disable it.\n"
340 "For example, rustc -C -target-cpu=mycpu -C "
341 "target-feature=+feature1,-feature2\n\n");
346 extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
347 printf("Target CPU help is not supported by this LLVM version.\n\n");
350 extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
351 printf("Target features help is not supported by this LLVM version.\n\n");
355 extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
356 const char *TripleStr, const char *CPU, const char *Feature,
357 LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
358 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
359 bool PositionIndependentExecutable, bool FunctionSections,
361 bool TrapUnreachable,
364 auto OptLevel = fromRust(RustOptLevel);
365 auto RM = fromRust(RustReloc);
368 Triple Trip(Triple::normalize(TripleStr));
369 const llvm::Target *TheTarget =
370 TargetRegistry::lookupTarget(Trip.getTriple(), Error);
371 if (TheTarget == nullptr) {
372 LLVMRustSetLastError(Error.c_str());
376 StringRef RealCPU = CPU;
377 if (RealCPU == "native") {
378 RealCPU = sys::getHostCPUName();
381 TargetOptions Options;
383 Options.FloatABIType = FloatABI::Default;
385 Options.FloatABIType = FloatABI::Soft;
387 Options.DataSections = DataSections;
388 Options.FunctionSections = FunctionSections;
390 if (TrapUnreachable) {
391 // Tell LLVM to translate `unreachable` into an explicit trap instruction.
392 // This limits the extent of possible undefined behavior in some cases, as
393 // it prevents control flow from "falling through" into whatever code
394 // happens to be laid out next in memory.
395 Options.TrapUnreachable = true;
399 Options.ThreadModel = ThreadModel::Single;
402 #if LLVM_VERSION_GE(6, 0)
403 Optional<CodeModel::Model> CM;
405 CodeModel::Model CM = CodeModel::Model::Default;
407 if (RustCM != LLVMRustCodeModel::None)
408 CM = fromRust(RustCM);
409 TargetMachine *TM = TheTarget->createTargetMachine(
410 Trip.getTriple(), RealCPU, Feature, Options, RM, CM, OptLevel);
414 extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
418 // Unfortunately, LLVM doesn't expose a C API to add the corresponding analysis
419 // passes for a target to a pass manager. We export that functionality through
421 extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM,
422 LLVMPassManagerRef PMR,
424 PassManagerBase *PM = unwrap(PMR);
426 createTargetTransformInfoWrapperPass(unwrap(TM)->getTargetIRAnalysis()));
429 extern "C" void LLVMRustConfigurePassManagerBuilder(
430 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
431 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize) {
432 // Ignore mergefunc for now as enabling it causes crashes.
433 // unwrap(PMBR)->MergeFunctions = MergeFunctions;
434 unwrap(PMBR)->SLPVectorize = SLPVectorize;
435 unwrap(PMBR)->OptLevel = fromRust(OptLevel);
436 unwrap(PMBR)->LoopVectorize = LoopVectorize;
439 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
440 // field of a PassManagerBuilder, we expose our own method of doing so.
441 extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
443 bool DisableSimplifyLibCalls) {
444 Triple TargetTriple(unwrap(M)->getTargetTriple());
445 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
446 if (DisableSimplifyLibCalls)
447 TLI->disableAllFunctions();
448 unwrap(PMBR)->LibraryInfo = TLI;
451 // Unfortunately, the LLVM C API doesn't provide a way to create the
452 // TargetLibraryInfo pass, so we use this method to do so.
453 extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
454 bool DisableSimplifyLibCalls) {
455 Triple TargetTriple(unwrap(M)->getTargetTriple());
456 TargetLibraryInfoImpl TLII(TargetTriple);
457 if (DisableSimplifyLibCalls)
458 TLII.disableAllFunctions();
459 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
462 // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
463 // all the functions in a module, so we do that manually here. You'll find
464 // similar code in clang's BackendUtil.cpp file.
465 extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
467 llvm::legacy::FunctionPassManager *P =
468 unwrap<llvm::legacy::FunctionPassManager>(PMR);
469 P->doInitialization();
471 // Upgrade all calls to old intrinsics first.
472 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
473 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
475 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
477 if (!I->isDeclaration())
483 extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
484 // Initializing the command-line options more than once is not allowed. So,
485 // check if they've already been initialized. (This could happen if we're
486 // being called from rustpkg, for example). If the arguments change, then
487 // that's just kinda unfortunate.
488 static bool Initialized = false;
492 cl::ParseCommandLineOptions(Argc, Argv);
495 enum class LLVMRustFileType {
501 static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
503 case LLVMRustFileType::AssemblyFile:
504 return TargetMachine::CGFT_AssemblyFile;
505 case LLVMRustFileType::ObjectFile:
506 return TargetMachine::CGFT_ObjectFile;
508 report_fatal_error("Bad FileType.");
512 extern "C" LLVMRustResult
513 LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
514 LLVMModuleRef M, const char *Path,
515 LLVMRustFileType RustFileType) {
516 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
517 auto FileType = fromRust(RustFileType);
519 std::string ErrorInfo;
521 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
523 ErrorInfo = EC.message();
524 if (ErrorInfo != "") {
525 LLVMRustSetLastError(ErrorInfo.c_str());
526 return LLVMRustResult::Failure;
529 unwrap(Target)->addPassesToEmitFile(*PM, OS, FileType, false);
532 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
533 // stream (OS), so the only real safe place to delete this is here? Don't we
534 // wish this was written in Rust?
536 return LLVMRustResult::Success;
540 // Callback to demangle function name
542 // * name to be demangled
545 // * output buffer len
546 // Returns len of demangled string, or 0 if demangle failed.
547 typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
552 class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
554 std::vector<char> Buf;
557 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
559 // Return empty string if demangle failed
560 // or if name does not need to be demangled
561 StringRef CallDemangle(StringRef name) {
566 if (Buf.size() < name.size() * 2) {
567 // Semangled name usually shorter than mangled,
568 // but allocate twice as much memory just in case
569 Buf.resize(name.size() * 2);
572 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
578 auto Demangled = StringRef(Buf.data(), R);
579 if (Demangled == name) {
580 // Do not print anything if demangled name is equal to mangled.
587 void emitFunctionAnnot(const Function *F,
588 formatted_raw_ostream &OS) override {
589 StringRef Demangled = CallDemangle(F->getName());
590 if (Demangled.empty()) {
594 OS << "; " << Demangled << "\n";
597 void emitInstructionAnnot(const Instruction *I,
598 formatted_raw_ostream &OS) override {
601 if (const CallInst *CI = dyn_cast<CallInst>(I)) {
603 Value = CI->getCalledValue();
604 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
606 Value = II->getCalledValue();
608 // Could demangle more operations, e. g.
609 // `store %place, @function`.
613 if (!Value->hasName()) {
617 StringRef Demangled = CallDemangle(Value->getName());
618 if (Demangled.empty()) {
622 OS << "; " << Name << " " << Demangled << "\n";
626 class RustPrintModulePass : public ModulePass {
631 RustPrintModulePass() : ModulePass(ID), OS(nullptr), Demangle(nullptr) {}
632 RustPrintModulePass(raw_ostream &OS, DemangleFn Demangle)
633 : ModulePass(ID), OS(&OS), Demangle(Demangle) {}
635 bool runOnModule(Module &M) override {
636 RustAssemblyAnnotationWriter AW(Demangle);
638 M.print(*OS, &AW, false);
643 void getAnalysisUsage(AnalysisUsage &AU) const override {
644 AU.setPreservesAll();
647 static StringRef name() { return "RustPrintModulePass"; }
653 void initializeRustPrintModulePassPass(PassRegistry&);
656 char RustPrintModulePass::ID = 0;
657 INITIALIZE_PASS(RustPrintModulePass, "print-rust-module",
658 "Print rust module to stderr", false, false)
660 extern "C" void LLVMRustPrintModule(LLVMPassManagerRef PMR, LLVMModuleRef M,
661 const char *Path, DemangleFn Demangle) {
662 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
663 std::string ErrorInfo;
666 raw_fd_ostream OS(Path, EC, sys::fs::F_None);
668 ErrorInfo = EC.message();
670 formatted_raw_ostream FOS(OS);
672 PM->add(new RustPrintModulePass(FOS, Demangle));
677 extern "C" void LLVMRustPrintPasses() {
678 LLVMInitializePasses();
679 struct MyListener : PassRegistrationListener {
680 void passEnumerate(const PassInfo *Info) {
681 #if LLVM_VERSION_GE(4, 0)
682 StringRef PassArg = Info->getPassArgument();
683 StringRef PassName = Info->getPassName();
684 if (!PassArg.empty()) {
685 // These unsigned->signed casts could theoretically overflow, but
686 // realistically never will (and even if, the result is implementation
687 // defined rather plain UB).
688 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
689 (int)PassName.size(), PassName.data());
692 if (Info->getPassArgument() && *Info->getPassArgument()) {
693 printf("%15s - %s\n", Info->getPassArgument(), Info->getPassName());
699 PassRegistry *PR = PassRegistry::getPassRegistry();
700 PR->enumerateWith(&Listener);
703 extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
705 #if LLVM_VERSION_GE(4, 0)
706 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
708 unwrap(PMBR)->Inliner = createAlwaysInlinerPass(AddLifetimes);
712 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
714 llvm::legacy::PassManager passes;
716 auto PreserveFunctions = [=](const GlobalValue &GV) {
717 for (size_t I = 0; I < Len; I++) {
718 if (GV.getName() == Symbols[I]) {
725 passes.add(llvm::createInternalizePass(PreserveFunctions));
727 passes.run(*unwrap(M));
730 extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
731 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
733 GV->setDoesNotThrow();
734 Function *F = dyn_cast<Function>(GV);
738 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
739 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
740 if (isa<InvokeInst>(I)) {
741 InvokeInst *CI = cast<InvokeInst>(I);
742 CI->setDoesNotThrow();
750 LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
751 LLVMTargetMachineRef TMR) {
752 TargetMachine *Target = unwrap(TMR);
753 unwrap(Module)->setDataLayout(Target->createDataLayout());
756 extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
757 unwrap(M)->setPIELevel(PIELevel::Level::Large);
761 LLVMRustThinLTOAvailable() {
762 #if LLVM_VERSION_GE(4, 0)
769 #if LLVM_VERSION_GE(4, 0)
771 // Here you'll find an implementation of ThinLTO as used by the Rust compiler
772 // right now. This ThinLTO support is only enabled on "recent ish" versions of
773 // LLVM, and otherwise it's just blanket rejected from other compilers.
775 // Most of this implementation is straight copied from LLVM. At the time of
776 // this writing it wasn't *quite* suitable to reuse more code from upstream
777 // for our purposes, but we should strive to upstream this support once it's
778 // ready to go! I figure we may want a bit of testing locally first before
779 // sending this upstream to LLVM. I hear though they're quite eager to receive
780 // feedback like this!
782 // If you're reading this code and wondering "what in the world" or you're
783 // working "good lord by LLVM upgrade is *still* failing due to these bindings"
784 // then fear not! (ok maybe fear a little). All code here is mostly based
785 // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
787 // You'll find that the general layout here roughly corresponds to the `run`
788 // method in that file as well as `ProcessThinLTOModule`. Functions are
789 // specifically commented below as well, but if you're updating this code
790 // or otherwise trying to understand it, the LLVM source will be useful in
791 // interpreting the mysteries within.
793 // Otherwise I'll apologize in advance, it probably requires a relatively
794 // significant investment on your part to "truly understand" what's going on
795 // here. Not saying I do myself, but it took me awhile staring at LLVM's source
796 // and various online resources about ThinLTO to make heads or tails of all
800 LLVMRustWriteThinBitcodeToFile(LLVMPassManagerRef PMR,
802 const char *BcFile) {
803 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
805 llvm::raw_fd_ostream bc(BcFile, EC, llvm::sys::fs::F_None);
807 LLVMRustSetLastError(EC.message().c_str());
810 PM->add(createWriteThinLTOBitcodePass(bc));
816 // This is a shared data structure which *must* be threadsafe to share
817 // read-only amongst threads. This also corresponds basically to the arguments
818 // of the `ProcessThinLTOModule` function in the LLVM source.
819 struct LLVMRustThinLTOData {
820 // The combined index that is the global analysis over all modules we're
821 // performing ThinLTO for. This is mostly managed by LLVM.
822 ModuleSummaryIndex Index;
824 // All modules we may look at, stored as in-memory serialized versions. This
825 // is later used when inlining to ensure we can extract any module to inline
827 StringMap<MemoryBufferRef> ModuleMap;
829 // A set that we manage of everything we *don't* want internalized. Note that
830 // this includes all transitive references right now as well, but it may not
832 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
834 // Not 100% sure what these are, but they impact what's internalized and
835 // what's inlined across modules, I believe.
836 StringMap<FunctionImporter::ImportMapTy> ImportLists;
837 StringMap<FunctionImporter::ExportSetTy> ExportLists;
838 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
840 #if LLVM_VERSION_GE(7, 0)
841 LLVMRustThinLTOData() : Index(/* isPerformingAnalysis = */ false) {}
845 // Just an argument to the `LLVMRustCreateThinLTOData` function below.
846 struct LLVMRustThinLTOModule {
847 const char *identifier;
852 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
854 static const GlobalValueSummary *
855 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
856 auto StrongDefForLinker = llvm::find_if(
857 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
858 auto Linkage = Summary->linkage();
859 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
860 !GlobalValue::isWeakForLinker(Linkage);
862 if (StrongDefForLinker != GVSummaryList.end())
863 return StrongDefForLinker->get();
865 auto FirstDefForLinker = llvm::find_if(
866 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
867 auto Linkage = Summary->linkage();
868 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
870 if (FirstDefForLinker == GVSummaryList.end())
872 return FirstDefForLinker->get();
875 // The main entry point for creating the global ThinLTO analysis. The structure
876 // here is basically the same as before threads are spawned in the `run`
877 // function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
878 extern "C" LLVMRustThinLTOData*
879 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
881 const char **preserved_symbols,
883 auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
885 // Load each module's summary and merge it into one combined index
886 for (int i = 0; i < num_modules; i++) {
887 auto module = &modules[i];
888 StringRef buffer(module->data, module->len);
889 MemoryBufferRef mem_buffer(buffer, module->identifier);
891 Ret->ModuleMap[module->identifier] = mem_buffer;
893 #if LLVM_VERSION_GE(5, 0)
894 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
895 LLVMRustSetLastError(toString(std::move(Err)).c_str());
899 Expected<std::unique_ptr<object::ModuleSummaryIndexObjectFile>> ObjOrErr =
900 object::ModuleSummaryIndexObjectFile::create(mem_buffer);
902 LLVMRustSetLastError(toString(ObjOrErr.takeError()).c_str());
905 auto Index = (*ObjOrErr)->takeIndex();
906 Ret->Index.mergeFrom(std::move(Index), i);
910 // Collect for each module the list of function it defines (GUID -> Summary)
911 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
913 // Convert the preserved symbols set from string to GUID, this is then needed
914 // for internalization.
915 for (int i = 0; i < num_symbols; i++) {
916 auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
917 Ret->GUIDPreservedSymbols.insert(GUID);
920 // Collect the import/export lists for all modules from the call-graph in the
923 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
924 #if LLVM_VERSION_GE(5, 0)
925 #if LLVM_VERSION_GE(7, 0)
926 auto deadIsPrevailing = [&](GlobalValue::GUID G) {
927 return PrevailingType::Unknown;
929 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
931 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols);
933 ComputeCrossModuleImport(
935 Ret->ModuleToDefinedGVSummaries,
940 auto DeadSymbols = computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols);
941 ComputeCrossModuleImport(
943 Ret->ModuleToDefinedGVSummaries,
950 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
951 // impacts the caching.
953 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
954 // being lifted from `lib/LTO/LTO.cpp` as well
955 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
956 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
957 for (auto &I : Ret->Index) {
958 #if LLVM_VERSION_GE(5, 0)
959 if (I.second.SummaryList.size() > 1)
960 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
962 if (I.second.size() > 1)
963 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second);
966 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
967 const auto &Prevailing = PrevailingCopy.find(GUID);
968 if (Prevailing == PrevailingCopy.end())
970 return Prevailing->second == S;
972 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
973 GlobalValue::GUID GUID,
974 GlobalValue::LinkageTypes NewLinkage) {
975 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
977 thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
979 // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
980 // callback below. This callback below will dictate the linkage for all
981 // summaries in the index, and we basically just only want to ensure that dead
982 // symbols are internalized. Otherwise everything that's already external
983 // linkage will stay as external, and internal will stay as internal.
984 std::set<GlobalValue::GUID> ExportedGUIDs;
985 for (auto &List : Ret->Index) {
986 #if LLVM_VERSION_GE(5, 0)
987 for (auto &GVS: List.second.SummaryList) {
989 for (auto &GVS: List.second) {
991 if (GlobalValue::isLocalLinkage(GVS->linkage()))
993 auto GUID = GVS->getOriginalName();
994 #if LLVM_VERSION_GE(5, 0)
995 if (GVS->flags().Live)
997 if (!DeadSymbols.count(GUID))
999 ExportedGUIDs.insert(GUID);
1002 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1003 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1004 return (ExportList != Ret->ExportLists.end() &&
1005 ExportList->second.count(GUID)) ||
1006 ExportedGUIDs.count(GUID);
1008 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1010 return Ret.release();
1014 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1018 // Below are the various passes that happen *per module* when doing ThinLTO.
1020 // In other words, these are the functions that are all run concurrently
1021 // with one another, one per module. The passes here correspond to the analysis
1022 // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1023 // `ProcessThinLTOModule` function. Here they're split up into separate steps
1024 // so rustc can save off the intermediate bytecode between each step.
1027 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1028 Module &Mod = *unwrap(M);
1029 if (renameModuleForThinLTO(Mod, Data->Index)) {
1030 LLVMRustSetLastError("renameModuleForThinLTO failed");
1037 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1038 Module &Mod = *unwrap(M);
1039 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1040 thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
1045 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1046 Module &Mod = *unwrap(M);
1047 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1048 thinLTOInternalizeModule(Mod, DefinedGlobals);
1053 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1054 Module &Mod = *unwrap(M);
1055 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1056 auto Loader = [&](StringRef Identifier) {
1057 const auto &Memory = Data->ModuleMap.lookup(Identifier);
1058 auto &Context = Mod.getContext();
1059 return getLazyBitcodeModule(Memory, Context, true, true);
1061 FunctionImporter Importer(Data->Index, Loader);
1062 Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1064 LLVMRustSetLastError(toString(Result.takeError()).c_str());
1070 // This struct and various functions are sort of a hack right now, but the
1071 // problem is that we've got in-memory LLVM modules after we generate and
1072 // optimize all codegen-units for one compilation in rustc. To be compatible
1073 // with the LTO support above we need to serialize the modules plus their
1074 // ThinLTO summary into memory.
1076 // This structure is basically an owned version of a serialize module, with
1077 // a ThinLTO summary attached.
1078 struct LLVMRustThinLTOBuffer {
1082 extern "C" LLVMRustThinLTOBuffer*
1083 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1084 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1086 raw_string_ostream OS(Ret->data);
1088 legacy::PassManager PM;
1089 PM.add(createWriteThinLTOBitcodePass(OS));
1093 return Ret.release();
1097 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1101 extern "C" const void*
1102 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1103 return Buffer->data.data();
1107 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1108 return Buffer->data.length();
1111 // This is what we used to parse upstream bitcode for actual ThinLTO
1112 // processing. We'll call this once per module optimized through ThinLTO, and
1113 // it'll be called concurrently on many threads.
1114 extern "C" LLVMModuleRef
1115 LLVMRustParseBitcodeForThinLTO(LLVMContextRef Context,
1118 const char *identifier) {
1119 StringRef Data(data, len);
1120 MemoryBufferRef Buffer(Data, identifier);
1121 unwrap(Context)->enableDebugTypeODRUniquing();
1122 Expected<std::unique_ptr<Module>> SrcOrError =
1123 parseBitcodeFile(Buffer, *unwrap(Context));
1125 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1128 return wrap(std::move(*SrcOrError).release());
1131 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1132 // the comment in `back/lto.rs` for why this exists.
1134 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1136 DICompileUnit **B) {
1137 Module *M = unwrap(Mod);
1138 DICompileUnit **Cur = A;
1139 DICompileUnit **Next = B;
1140 for (DICompileUnit *CU : M->debug_compile_units()) {
1149 // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1150 // the comment in `back/lto.rs` for why this exists.
1152 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1153 Module *M = unwrap(Mod);
1155 // If the original source module didn't have a `DICompileUnit` then try to
1156 // merge all the existing compile units. If there aren't actually any though
1157 // then there's not much for us to do so return.
1158 if (Unit == nullptr) {
1159 for (DICompileUnit *CU : M->debug_compile_units()) {
1163 if (Unit == nullptr)
1167 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1168 // process it recursively. Note that we specifically iterate over instructions
1169 // to ensure we feed everything into it.
1170 DebugInfoFinder Finder;
1171 Finder.processModule(*M);
1172 for (Function &F : M->functions()) {
1173 for (auto &FI : F) {
1174 for (Instruction &BI : FI) {
1175 if (auto Loc = BI.getDebugLoc())
1176 Finder.processLocation(*M, Loc);
1177 if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1178 Finder.processValue(*M, DVI);
1179 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1180 Finder.processDeclare(*M, DDI);
1185 // After we've found all our debuginfo, rewrite all subprograms to point to
1186 // the same `DICompileUnit`.
1187 for (auto &F : Finder.subprograms()) {
1188 F->replaceUnit(Unit);
1191 // Erase any other references to other `DICompileUnit` instances, the verifier
1192 // will later ensure that we don't actually have any other stale references to
1194 auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1195 MD->clearOperands();
1196 MD->addOperand(Unit);
1200 LLVMRustThinLTORemoveAvailableExternally(LLVMModuleRef Mod) {
1201 Module *M = unwrap(Mod);
1202 for (Function &F : M->functions()) {
1203 if (F.hasAvailableExternallyLinkage())
1211 LLVMRustWriteThinBitcodeToFile(LLVMPassManagerRef PMR,
1213 const char *BcFile) {
1214 report_fatal_error("ThinLTO not available");
1217 struct LLVMRustThinLTOData {
1220 struct LLVMRustThinLTOModule {
1223 extern "C" LLVMRustThinLTOData*
1224 LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1226 const char **preserved_symbols,
1228 report_fatal_error("ThinLTO not available");
1232 LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1233 report_fatal_error("ThinLTO not available");
1237 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1238 report_fatal_error("ThinLTO not available");
1242 LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1243 report_fatal_error("ThinLTO not available");
1247 LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1248 report_fatal_error("ThinLTO not available");
1252 LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1253 report_fatal_error("ThinLTO not available");
1256 struct LLVMRustThinLTOBuffer {
1259 extern "C" LLVMRustThinLTOBuffer*
1260 LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1261 report_fatal_error("ThinLTO not available");
1265 LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1266 report_fatal_error("ThinLTO not available");
1269 extern "C" const void*
1270 LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1271 report_fatal_error("ThinLTO not available");
1275 LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1276 report_fatal_error("ThinLTO not available");
1279 extern "C" LLVMModuleRef
1280 LLVMRustParseBitcodeForThinLTO(LLVMContextRef Context,
1283 const char *identifier) {
1284 report_fatal_error("ThinLTO not available");
1288 LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1290 DICompileUnit **B) {
1291 report_fatal_error("ThinLTO not available");
1295 LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod) {
1296 report_fatal_error("ThinLTO not available");
1300 LLVMRustThinLTORemoveAvailableExternally(LLVMModuleRef Mod) {
1301 report_fatal_error("ThinLTO not available");
1304 #endif // LLVM_VERSION_GE(4, 0)