src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  39     if !builder.config.extended {
  40         return false;
  41     }
  42     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  43 }
  44
  45 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  46 pub struct Docs {
  47     pub host: TargetSelection,
  48 }
  49
  50 impl Step for Docs {
  51     type Output = Option<GeneratedTarball>;
  52     const DEFAULT: bool = true;
  53
  54     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  55         let default = run.builder.config.docs;
  56         run.alias("rust-docs").default_condition(default)
  57     }
  58
  59     fn make_run(run: RunConfig<'_>) {
  60         run.builder.ensure(Docs { host: run.target });
  61     }
  62
  63     /// Builds the `rust-docs` installer component.
  64     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  65         let host = self.host;
  66         builder.default_doc(&[]);
  67
  68         let dest = "share/doc/rust/html";
  69
  70         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  71         tarball.set_product_name("Rust Documentation");
  72         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  73         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  74         Some(tarball.generate())
  75     }
  76 }
  77
  78 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  79 pub struct JsonDocs {
  80     pub host: TargetSelection,
  81 }
  82
  83 impl Step for JsonDocs {
  84     type Output = Option<GeneratedTarball>;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  88         let default = run.builder.config.docs;
  89         run.alias("rust-docs-json").default_condition(default)
  90     }
  91
  92     fn make_run(run: RunConfig<'_>) {
  93         run.builder.ensure(JsonDocs { host: run.target });
  94     }
  95
  96     /// Builds the `rust-docs-json` installer component.
  97     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  98         let host = self.host;
  99         builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
 100
 101         let dest = "share/doc/rust/json";
 102
 103         let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple);
 104         tarball.set_product_name("Rust Documentation In JSON Format");
 105         tarball.is_preview(true);
 106         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 107         Some(tarball.generate())
 108     }
 109 }
 110
 111 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 112 pub struct RustcDocs {
 113     pub host: TargetSelection,
 114 }
 115
 116 impl Step for RustcDocs {
 117     type Output = Option<GeneratedTarball>;
 118     const DEFAULT: bool = true;
 119
 120     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 121         let builder = run.builder;
 122         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 123     }
 124
 125     fn make_run(run: RunConfig<'_>) {
 126         run.builder.ensure(RustcDocs { host: run.target });
 127     }
 128
 129     /// Builds the `rustc-docs` installer component.
 130     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 131         let host = self.host;
 132         builder.default_doc(&[]);
 133
 134         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 135         tarball.set_product_name("Rustc Documentation");
 136         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 137         Some(tarball.generate())
 138     }
 139 }
 140
 141 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 142     let mut found = Vec::with_capacity(files.len());
 143
 144     for file in files {
 145         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 146
 147         if let Some(file_path) = file_path {
 148             found.push(file_path);
 149         } else {
 150             panic!("Could not find '{}' in {:?}", file, path);
 151         }
 152     }
 153
 154     found
 155 }
 156
 157 fn make_win_dist(
 158     rust_root: &Path,
 159     plat_root: &Path,
 160     target: TargetSelection,
 161     builder: &Builder<'_>,
 162 ) {
 163     //Ask gcc where it keeps its stuff
 164     let mut cmd = Command::new(builder.cc(target));
 165     cmd.arg("-print-search-dirs");
 166     let gcc_out = output(&mut cmd);
 167
 168     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 169     let mut lib_path = Vec::new();
 170
 171     for line in gcc_out.lines() {
 172         let idx = line.find(':').unwrap();
 173         let key = &line[..idx];
 174         let trim_chars: &[_] = &[' ', '='];
 175         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 176
 177         if key == "programs" {
 178             bin_path.extend(value);
 179         } else if key == "libraries" {
 180             lib_path.extend(value);
 181         }
 182     }
 183
 184     let compiler = if target == "i686-pc-windows-gnu" {
 185         "i686-w64-mingw32-gcc.exe"
 186     } else if target == "x86_64-pc-windows-gnu" {
 187         "x86_64-w64-mingw32-gcc.exe"
 188     } else {
 189         "gcc.exe"
 190     };
 191     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 192     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 193     if target.starts_with("i686-") {
 194         rustc_dlls.push("libgcc_s_dw2-1.dll");
 195     } else {
 196         rustc_dlls.push("libgcc_s_seh-1.dll");
 197     }
 198
 199     let target_libs = [
 200         //MinGW libs
 201         "libgcc.a",
 202         "libgcc_eh.a",
 203         "libgcc_s.a",
 204         "libm.a",
 205         "libmingw32.a",
 206         "libmingwex.a",
 207         "libstdc++.a",
 208         "libiconv.a",
 209         "libmoldname.a",
 210         "libpthread.a",
 211         //Windows import libs
 212         "libadvapi32.a",
 213         "libbcrypt.a",
 214         "libcomctl32.a",
 215         "libcomdlg32.a",
 216         "libcredui.a",
 217         "libcrypt32.a",
 218         "libdbghelp.a",
 219         "libgdi32.a",
 220         "libimagehlp.a",
 221         "libiphlpapi.a",
 222         "libkernel32.a",
 223         "libmsimg32.a",
 224         "libmsvcrt.a",
 225         "libodbc32.a",
 226         "libole32.a",
 227         "liboleaut32.a",
 228         "libopengl32.a",
 229         "libpsapi.a",
 230         "librpcrt4.a",
 231         "libsecur32.a",
 232         "libsetupapi.a",
 233         "libshell32.a",
 234         "libsynchronization.a",
 235         "libuser32.a",
 236         "libuserenv.a",
 237         "libuuid.a",
 238         "libwinhttp.a",
 239         "libwinmm.a",
 240         "libwinspool.a",
 241         "libws2_32.a",
 242         "libwsock32.a",
 243     ];
 244
 245     //Find mingw artifacts we want to bundle
 246     let target_tools = find_files(&target_tools, &bin_path);
 247     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 248     let target_libs = find_files(&target_libs, &lib_path);
 249
 250     // Copy runtime dlls next to rustc.exe
 251     let dist_bin_dir = rust_root.join("bin/");
 252     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 253     for src in rustc_dlls {
 254         builder.copy_to_folder(&src, &dist_bin_dir);
 255     }
 256
 257     //Copy platform tools to platform-specific bin directory
 258     let target_bin_dir = plat_root
 259         .join("lib")
 260         .join("rustlib")
 261         .join(target.triple)
 262         .join("bin")
 263         .join("self-contained");
 264     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 265     for src in target_tools {
 266         builder.copy_to_folder(&src, &target_bin_dir);
 267     }
 268
 269     // Warn windows-gnu users that the bundled GCC cannot compile C files
 270     builder.create(
 271         &target_bin_dir.join("GCC-WARNING.txt"),
 272         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 273          used as a linker. In order to be able to compile projects containing C code use \
 274          the GCC provided by MinGW or Cygwin.",
 275     );
 276
 277     //Copy platform libs to platform-specific lib directory
 278     let target_lib_dir = plat_root
 279         .join("lib")
 280         .join("rustlib")
 281         .join(target.triple)
 282         .join("lib")
 283         .join("self-contained");
 284     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 285     for src in target_libs {
 286         builder.copy_to_folder(&src, &target_lib_dir);
 287     }
 288 }
 289
 290 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 291 pub struct Mingw {
 292     pub host: TargetSelection,
 293 }
 294
 295 impl Step for Mingw {
 296     type Output = Option<GeneratedTarball>;
 297     const DEFAULT: bool = true;
 298
 299     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 300         run.alias("rust-mingw")
 301     }
 302
 303     fn make_run(run: RunConfig<'_>) {
 304         run.builder.ensure(Mingw { host: run.target });
 305     }
 306
 307     /// Builds the `rust-mingw` installer component.
 308     ///
 309     /// This contains all the bits and pieces to run the MinGW Windows targets
 310     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 311     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 312         let host = self.host;
 313         if !host.ends_with("pc-windows-gnu") {
 314             return None;
 315         }
 316
 317         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 318         tarball.set_product_name("Rust MinGW");
 319
 320         // The first argument is a "temporary directory" which is just
 321         // thrown away (this contains the runtime DLLs included in the rustc package
 322         // above) and the second argument is where to place all the MinGW components
 323         // (which is what we want).
 324         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 325
 326         Some(tarball.generate())
 327     }
 328 }
 329
 330 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 331 pub struct Rustc {
 332     pub compiler: Compiler,
 333 }
 334
 335 impl Step for Rustc {
 336     type Output = GeneratedTarball;
 337     const DEFAULT: bool = true;
 338     const ONLY_HOSTS: bool = true;
 339
 340     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 341         run.alias("rustc")
 342     }
 343
 344     fn make_run(run: RunConfig<'_>) {
 345         run.builder
 346             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 347     }
 348
 349     /// Creates the `rustc` installer component.
 350     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 351         let compiler = self.compiler;
 352         let host = self.compiler.host;
 353
 354         let tarball = Tarball::new(builder, "rustc", &host.triple);
 355
 356         // Prepare the rustc "image", what will actually end up getting installed
 357         prepare_image(builder, compiler, tarball.image_dir());
 358
 359         // On MinGW we've got a few runtime DLL dependencies that we need to
 360         // include. The first argument to this script is where to put these DLLs
 361         // (the image we're creating), and the second argument is a junk directory
 362         // to ignore all other MinGW stuff the script creates.
 363         //
 364         // On 32-bit MinGW we're always including a DLL which needs some extra
 365         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 366         // anything requiring us to distribute a license, but it's likely the
 367         // install will *also* include the rust-mingw package, which also needs
 368         // licenses, so to be safe we just include it here in all MinGW packages.
 369         if host.ends_with("pc-windows-gnu") {
 370             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 371             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 372         }
 373
 374         return tarball.generate();
 375
 376         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 377             let host = compiler.host;
 378             let src = builder.sysroot(compiler);
 379
 380             // Copy rustc/rustdoc binaries
 381             t!(fs::create_dir_all(image.join("bin")));
 382             builder.cp_r(&src.join("bin"), &image.join("bin"));
 383
 384             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 385
 386             let ra_proc_macro_srv = builder
 387                 .ensure(tool::RustAnalyzerProcMacroSrv {
 388                     compiler: builder.compiler_for(
 389                         compiler.stage,
 390                         builder.config.build,
 391                         compiler.host,
 392                     ),
 393                     target: compiler.host,
 394                 })
 395                 .expect("rust-analyzer-proc-macro-server always builds");
 396             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 397
 398             let libdir_relative = builder.libdir_relative(compiler);
 399
 400             // Copy runtime DLLs needed by the compiler
 401             if libdir_relative.to_str() != Some("bin") {
 402                 let libdir = builder.rustc_libdir(compiler);
 403                 for entry in builder.read_dir(&libdir) {
 404                     let name = entry.file_name();
 405                     if let Some(s) = name.to_str() {
 406                         if is_dylib(s) {
 407                             // Don't use custom libdir here because ^lib/ will be resolved again
 408                             // with installer
 409                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 410                         }
 411                     }
 412                 }
 413             }
 414
 415             // Copy over the codegen backends
 416             let backends_src = builder.sysroot_codegen_backends(compiler);
 417             let backends_rel = backends_src
 418                 .strip_prefix(&src)
 419                 .unwrap()
 420                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 421                 .unwrap();
 422             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 423             let backends_dst = image.join("lib").join(&backends_rel);
 424
 425             t!(fs::create_dir_all(&backends_dst));
 426             builder.cp_r(&backends_src, &backends_dst);
 427
 428             // Copy libLLVM.so to the lib dir as well, if needed. While not
 429             // technically needed by rustc itself it's needed by lots of other
 430             // components like the llvm tools and LLD. LLD is included below and
 431             // tools/LLDB come later, so let's just throw it in the rustc
 432             // component for now.
 433             maybe_install_llvm_runtime(builder, host, image);
 434
 435             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 436             t!(fs::create_dir_all(&dst_dir));
 437
 438             // Copy over lld if it's there
 439             if builder.config.lld_enabled {
 440                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 441                 let rust_lld = exe("rust-lld", compiler.host);
 442                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 443                 // for `-Z gcc-ld=lld`
 444                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 445                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 446                 t!(fs::create_dir(&gcc_lld_dst_dir));
 447                 for name in crate::LLD_FILE_NAMES {
 448                     let exe_name = exe(name, compiler.host);
 449                     builder
 450                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 451                 }
 452             }
 453
 454             // Man pages
 455             t!(fs::create_dir_all(image.join("share/man/man1")));
 456             let man_src = builder.src.join("src/doc/man");
 457             let man_dst = image.join("share/man/man1");
 458
 459             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 460             // to hardlink, and we don't want to edit the source templates
 461             for file_entry in builder.read_dir(&man_src) {
 462                 let page_src = file_entry.path();
 463                 let page_dst = man_dst.join(file_entry.file_name());
 464                 let src_text = t!(std::fs::read_to_string(&page_src));
 465                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 466                 t!(std::fs::write(&page_dst, &new_text));
 467                 t!(fs::copy(&page_src, &page_dst));
 468             }
 469
 470             // Debugger scripts
 471             builder
 472                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 473
 474             // Misc license info
 475             let cp = |file: &str| {
 476                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 477             };
 478             cp("COPYRIGHT");
 479             cp("LICENSE-APACHE");
 480             cp("LICENSE-MIT");
 481             cp("README.md");
 482         }
 483     }
 484 }
 485
 486 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 487 pub struct DebuggerScripts {
 488     pub sysroot: Interned<PathBuf>,
 489     pub host: TargetSelection,
 490 }
 491
 492 impl Step for DebuggerScripts {
 493     type Output = ();
 494
 495     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 496         run.never()
 497     }
 498
 499     /// Copies debugger scripts for `target` into the `sysroot` specified.
 500     fn run(self, builder: &Builder<'_>) {
 501         let host = self.host;
 502         let sysroot = self.sysroot;
 503         let dst = sysroot.join("lib/rustlib/etc");
 504         t!(fs::create_dir_all(&dst));
 505         let cp_debugger_script = |file: &str| {
 506             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 507         };
 508         if host.contains("windows-msvc") {
 509             // windbg debugger scripts
 510             builder.install(
 511                 &builder.src.join("src/etc/rust-windbg.cmd"),
 512                 &sysroot.join("bin"),
 513                 0o755,
 514             );
 515
 516             cp_debugger_script("natvis/intrinsic.natvis");
 517             cp_debugger_script("natvis/liballoc.natvis");
 518             cp_debugger_script("natvis/libcore.natvis");
 519             cp_debugger_script("natvis/libstd.natvis");
 520         } else {
 521             cp_debugger_script("rust_types.py");
 522
 523             // gdb debugger scripts
 524             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 525             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 526
 527             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 528             cp_debugger_script("gdb_lookup.py");
 529             cp_debugger_script("gdb_providers.py");
 530
 531             // lldb debugger scripts
 532             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 533
 534             cp_debugger_script("lldb_lookup.py");
 535             cp_debugger_script("lldb_providers.py");
 536             cp_debugger_script("lldb_commands")
 537         }
 538     }
 539 }
 540
 541 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 542     // The only true set of target libraries came from the build triple, so
 543     // let's reduce redundant work by only producing archives from that host.
 544     if compiler.host != builder.config.build {
 545         builder.info("\tskipping, not a build host");
 546         true
 547     } else {
 548         false
 549     }
 550 }
 551
 552 /// Copy stamped files into an image's `target/lib` directory.
 553 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 554     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 555     let self_contained_dst = dst.join("self-contained");
 556     t!(fs::create_dir_all(&dst));
 557     t!(fs::create_dir_all(&self_contained_dst));
 558     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 559         if dependency_type == DependencyType::TargetSelfContained {
 560             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 561         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 562             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 563         }
 564     }
 565 }
 566
 567 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 568 pub struct Std {
 569     pub compiler: Compiler,
 570     pub target: TargetSelection,
 571 }
 572
 573 impl Step for Std {
 574     type Output = Option<GeneratedTarball>;
 575     const DEFAULT: bool = true;
 576
 577     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 578         run.alias("rust-std")
 579     }
 580
 581     fn make_run(run: RunConfig<'_>) {
 582         run.builder.ensure(Std {
 583             compiler: run.builder.compiler_for(
 584                 run.builder.top_stage,
 585                 run.builder.config.build,
 586                 run.target,
 587             ),
 588             target: run.target,
 589         });
 590     }
 591
 592     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 593         let compiler = self.compiler;
 594         let target = self.target;
 595
 596         if skip_host_target_lib(builder, compiler) {
 597             return None;
 598         }
 599
 600         builder.ensure(compile::Std::new(compiler, target));
 601
 602         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 603         tarball.include_target_in_component_name(true);
 604
 605         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 606         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 607         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 608
 609         Some(tarball.generate())
 610     }
 611 }
 612
 613 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 614 pub struct RustcDev {
 615     pub compiler: Compiler,
 616     pub target: TargetSelection,
 617 }
 618
 619 impl Step for RustcDev {
 620     type Output = Option<GeneratedTarball>;
 621     const DEFAULT: bool = true;
 622     const ONLY_HOSTS: bool = true;
 623
 624     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 625         run.alias("rustc-dev")
 626     }
 627
 628     fn make_run(run: RunConfig<'_>) {
 629         run.builder.ensure(RustcDev {
 630             compiler: run.builder.compiler_for(
 631                 run.builder.top_stage,
 632                 run.builder.config.build,
 633                 run.target,
 634             ),
 635             target: run.target,
 636         });
 637     }
 638
 639     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 640         let compiler = self.compiler;
 641         let target = self.target;
 642         if skip_host_target_lib(builder, compiler) {
 643             return None;
 644         }
 645
 646         builder.ensure(compile::Rustc::new(compiler, target));
 647
 648         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 649
 650         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 651         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 652         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 653
 654         let src_files = &["Cargo.lock"];
 655         // This is the reduced set of paths which will become the rustc-dev component
 656         // (essentially the compiler crates and all of their path dependencies).
 657         copy_src_dirs(
 658             builder,
 659             &builder.src,
 660             &["compiler"],
 661             &[],
 662             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 663         );
 664         for file in src_files {
 665             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 666         }
 667
 668         Some(tarball.generate())
 669     }
 670 }
 671
 672 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 673 pub struct Analysis {
 674     pub compiler: Compiler,
 675     pub target: TargetSelection,
 676 }
 677
 678 impl Step for Analysis {
 679     type Output = Option<GeneratedTarball>;
 680     const DEFAULT: bool = true;
 681
 682     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 683         let default = should_build_extended_tool(&run.builder, "analysis");
 684         run.alias("rust-analysis").default_condition(default)
 685     }
 686
 687     fn make_run(run: RunConfig<'_>) {
 688         run.builder.ensure(Analysis {
 689             // Find the actual compiler (handling the full bootstrap option) which
 690             // produced the save-analysis data because that data isn't copied
 691             // through the sysroot uplifting.
 692             compiler: run.builder.compiler_for(
 693                 run.builder.top_stage,
 694                 run.builder.config.build,
 695                 run.target,
 696             ),
 697             target: run.target,
 698         });
 699     }
 700
 701     /// Creates a tarball of save-analysis metadata, if available.
 702     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 703         let compiler = self.compiler;
 704         let target = self.target;
 705         if compiler.host != builder.config.build {
 706             return None;
 707         }
 708
 709         builder.ensure(compile::Std::new(compiler, target));
 710         let src = builder
 711             .stage_out(compiler, Mode::Std)
 712             .join(target.triple)
 713             .join(builder.cargo_dir())
 714             .join("deps")
 715             .join("save-analysis");
 716
 717         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 718         tarball.include_target_in_component_name(true);
 719         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 720         Some(tarball.generate())
 721     }
 722 }
 723
 724 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 725 /// `dst_dir`.
 726 fn copy_src_dirs(
 727     builder: &Builder<'_>,
 728     base: &Path,
 729     src_dirs: &[&str],
 730     exclude_dirs: &[&str],
 731     dst_dir: &Path,
 732 ) {
 733     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 734         let spath = match path.to_str() {
 735             Some(path) => path,
 736             None => return false,
 737         };
 738         if spath.ends_with('~') || spath.ends_with(".pyc") {
 739             return false;
 740         }
 741
 742         const LLVM_PROJECTS: &[&str] = &[
 743             "llvm-project/clang",
 744             "llvm-project\\clang",
 745             "llvm-project/libunwind",
 746             "llvm-project\\libunwind",
 747             "llvm-project/lld",
 748             "llvm-project\\lld",
 749             "llvm-project/lldb",
 750             "llvm-project\\lldb",
 751             "llvm-project/llvm",
 752             "llvm-project\\llvm",
 753             "llvm-project/compiler-rt",
 754             "llvm-project\\compiler-rt",
 755             "llvm-project/cmake",
 756             "llvm-project\\cmake",
 757         ];
 758         if spath.contains("llvm-project")
 759             && !spath.ends_with("llvm-project")
 760             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 761         {
 762             return false;
 763         }
 764
 765         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 766         if LLVM_TEST.iter().any(|path| spath.contains(path))
 767             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 768         {
 769             return false;
 770         }
 771
 772         let full_path = Path::new(dir).join(path);
 773         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 774             return false;
 775         }
 776
 777         let excludes = [
 778             "CVS",
 779             "RCS",
 780             "SCCS",
 781             ".git",
 782             ".gitignore",
 783             ".gitmodules",
 784             ".gitattributes",
 785             ".cvsignore",
 786             ".svn",
 787             ".arch-ids",
 788             "{arch}",
 789             "=RELEASE-ID",
 790             "=meta-update",
 791             "=update",
 792             ".bzr",
 793             ".bzrignore",
 794             ".bzrtags",
 795             ".hg",
 796             ".hgignore",
 797             ".hgrags",
 798             "_darcs",
 799         ];
 800         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 801     }
 802
 803     // Copy the directories using our filter
 804     for item in src_dirs {
 805         let dst = &dst_dir.join(item);
 806         t!(fs::create_dir_all(dst));
 807         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 808     }
 809 }
 810
 811 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 812 pub struct Src;
 813
 814 impl Step for Src {
 815     /// The output path of the src installer tarball
 816     type Output = GeneratedTarball;
 817     const DEFAULT: bool = true;
 818     const ONLY_HOSTS: bool = true;
 819
 820     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 821         run.alias("rust-src")
 822     }
 823
 824     fn make_run(run: RunConfig<'_>) {
 825         run.builder.ensure(Src);
 826     }
 827
 828     /// Creates the `rust-src` installer component
 829     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 830         let tarball = Tarball::new_targetless(builder, "rust-src");
 831
 832         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 833         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 834         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 835         // and fix them...
 836         //
 837         // NOTE: if you update the paths here, you also should update the "virtual" path
 838         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 839         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 840
 841         let src_files = ["Cargo.lock"];
 842         // This is the reduced set of paths which will become the rust-src component
 843         // (essentially libstd and all of its path dependencies).
 844         copy_src_dirs(
 845             builder,
 846             &builder.src,
 847             &["library", "src/llvm-project/libunwind"],
 848             &[
 849                 // not needed and contains symlinks which rustup currently
 850                 // chokes on when unpacking.
 851                 "library/backtrace/crates",
 852                 // these are 30MB combined and aren't necessary for building
 853                 // the standard library.
 854                 "library/stdarch/Cargo.toml",
 855                 "library/stdarch/crates/stdarch-verify",
 856                 "library/stdarch/crates/intrinsic-test",
 857             ],
 858             &dst_src,
 859         );
 860         for file in src_files.iter() {
 861             builder.copy(&builder.src.join(file), &dst_src.join(file));
 862         }
 863
 864         tarball.generate()
 865     }
 866 }
 867
 868 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 869 pub struct PlainSourceTarball;
 870
 871 impl Step for PlainSourceTarball {
 872     /// Produces the location of the tarball generated
 873     type Output = GeneratedTarball;
 874     const DEFAULT: bool = true;
 875     const ONLY_HOSTS: bool = true;
 876
 877     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 878         let builder = run.builder;
 879         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 880     }
 881
 882     fn make_run(run: RunConfig<'_>) {
 883         run.builder.ensure(PlainSourceTarball);
 884     }
 885
 886     /// Creates the plain source tarball
 887     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 888         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 889         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 890         // It also contains symbolic links, unlike other any other dist tarball.
 891         // It's used for distros building rustc from source in a pre-vendored environment.
 892         let mut tarball = Tarball::new(builder, "rustc", "src");
 893         tarball.permit_symlinks(true);
 894         let plain_dst_src = tarball.image_dir();
 895
 896         // This is the set of root paths which will become part of the source package
 897         let src_files = [
 898             "COPYRIGHT",
 899             "LICENSE-APACHE",
 900             "LICENSE-MIT",
 901             "CONTRIBUTING.md",
 902             "README.md",
 903             "RELEASES.md",
 904             "configure",
 905             "x.py",
 906             "config.toml.example",
 907             "Cargo.toml",
 908             "Cargo.lock",
 909         ];
 910         let src_dirs = ["src", "compiler", "library"];
 911
 912         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 913
 914         // Copy the files normally
 915         for item in &src_files {
 916             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 917         }
 918
 919         // Create the version file
 920         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 921         if let Some(sha) = builder.rust_sha() {
 922             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 923         }
 924
 925         // If we're building from git sources, we need to vendor a complete distribution.
 926         if builder.rust_info.is_git() {
 927             // Ensure we have the submodules checked out.
 928             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 929
 930             // Vendor all Cargo dependencies
 931             let mut cmd = Command::new(&builder.initial_cargo);
 932             cmd.arg("vendor")
 933                 .arg("--sync")
 934                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 935                 .arg("--sync")
 936                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 937                 .arg("--sync")
 938                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 939                 .current_dir(&plain_dst_src);
 940
 941             let config = if !builder.config.dry_run {
 942                 t!(String::from_utf8(t!(cmd.output()).stdout))
 943             } else {
 944                 String::new()
 945             };
 946
 947             let cargo_config_dir = plain_dst_src.join(".cargo");
 948             builder.create_dir(&cargo_config_dir);
 949             builder.create(&cargo_config_dir.join("config.toml"), &config);
 950         }
 951
 952         tarball.bare()
 953     }
 954 }
 955
 956 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 957 pub struct Cargo {
 958     pub compiler: Compiler,
 959     pub target: TargetSelection,
 960 }
 961
 962 impl Step for Cargo {
 963     type Output = Option<GeneratedTarball>;
 964     const DEFAULT: bool = true;
 965     const ONLY_HOSTS: bool = true;
 966
 967     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 968         let default = should_build_extended_tool(&run.builder, "cargo");
 969         run.alias("cargo").default_condition(default)
 970     }
 971
 972     fn make_run(run: RunConfig<'_>) {
 973         run.builder.ensure(Cargo {
 974             compiler: run.builder.compiler_for(
 975                 run.builder.top_stage,
 976                 run.builder.config.build,
 977                 run.target,
 978             ),
 979             target: run.target,
 980         });
 981     }
 982
 983     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 984         let compiler = self.compiler;
 985         let target = self.target;
 986
 987         let cargo = builder.ensure(tool::Cargo { compiler, target });
 988         let src = builder.src.join("src/tools/cargo");
 989         let etc = src.join("src/etc");
 990
 991         // Prepare the image directory
 992         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 993         tarball.set_overlay(OverlayKind::Cargo);
 994
 995         tarball.add_file(&cargo, "bin", 0o755);
 996         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 997         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 998         tarball.add_dir(etc.join("man"), "share/man/man1");
 999         tarball.add_legal_and_readme_to("share/doc/cargo");
1000
1001         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1002             let dirent = dirent.expect("read dir entry");
1003             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1004                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1005             }
1006         }
1007
1008         Some(tarball.generate())
1009     }
1010 }
1011
1012 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1013 pub struct Rls {
1014     pub compiler: Compiler,
1015     pub target: TargetSelection,
1016 }
1017
1018 impl Step for Rls {
1019     type Output = Option<GeneratedTarball>;
1020     const ONLY_HOSTS: bool = true;
1021     const DEFAULT: bool = true;
1022
1023     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1024         let default = should_build_extended_tool(&run.builder, "rls");
1025         run.alias("rls").default_condition(default)
1026     }
1027
1028     fn make_run(run: RunConfig<'_>) {
1029         run.builder.ensure(Rls {
1030             compiler: run.builder.compiler_for(
1031                 run.builder.top_stage,
1032                 run.builder.config.build,
1033                 run.target,
1034             ),
1035             target: run.target,
1036         });
1037     }
1038
1039     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1040         let compiler = self.compiler;
1041         let target = self.target;
1042
1043         let rls = builder
1044             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1045             .expect("rls expected to build");
1046
1047         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1048         tarball.set_overlay(OverlayKind::RLS);
1049         tarball.is_preview(true);
1050         tarball.add_file(rls, "bin", 0o755);
1051         tarball.add_legal_and_readme_to("share/doc/rls");
1052         Some(tarball.generate())
1053     }
1054 }
1055
1056 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1057 pub struct RustAnalyzer {
1058     pub compiler: Compiler,
1059     pub target: TargetSelection,
1060 }
1061
1062 impl Step for RustAnalyzer {
1063     type Output = Option<GeneratedTarball>;
1064     const DEFAULT: bool = true;
1065     const ONLY_HOSTS: bool = true;
1066
1067     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1068         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1069         run.alias("rust-analyzer").default_condition(default)
1070     }
1071
1072     fn make_run(run: RunConfig<'_>) {
1073         run.builder.ensure(RustAnalyzer {
1074             compiler: run.builder.compiler_for(
1075                 run.builder.top_stage,
1076                 run.builder.config.build,
1077                 run.target,
1078             ),
1079             target: run.target,
1080         });
1081     }
1082
1083     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1084         let compiler = self.compiler;
1085         let target = self.target;
1086
1087         if target.contains("riscv64") {
1088             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1089             // to build. See #74813 for details.
1090             return None;
1091         }
1092
1093         let rust_analyzer = builder
1094             .ensure(tool::RustAnalyzer { compiler, target })
1095             .expect("rust-analyzer always builds");
1096
1097         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1098         tarball.set_overlay(OverlayKind::RustAnalyzer);
1099         tarball.is_preview(true);
1100         tarball.add_file(rust_analyzer, "bin", 0o755);
1101         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1102         Some(tarball.generate())
1103     }
1104 }
1105
1106 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1107 pub struct Clippy {
1108     pub compiler: Compiler,
1109     pub target: TargetSelection,
1110 }
1111
1112 impl Step for Clippy {
1113     type Output = Option<GeneratedTarball>;
1114     const DEFAULT: bool = true;
1115     const ONLY_HOSTS: bool = true;
1116
1117     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1118         let default = should_build_extended_tool(&run.builder, "clippy");
1119         run.alias("clippy").default_condition(default)
1120     }
1121
1122     fn make_run(run: RunConfig<'_>) {
1123         run.builder.ensure(Clippy {
1124             compiler: run.builder.compiler_for(
1125                 run.builder.top_stage,
1126                 run.builder.config.build,
1127                 run.target,
1128             ),
1129             target: run.target,
1130         });
1131     }
1132
1133     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1134         let compiler = self.compiler;
1135         let target = self.target;
1136
1137         // Prepare the image directory
1138         // We expect clippy to build, because we've exited this step above if tool
1139         // state for clippy isn't testing.
1140         let clippy = builder
1141             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1142             .expect("clippy expected to build - essential tool");
1143         let cargoclippy = builder
1144             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1145             .expect("clippy expected to build - essential tool");
1146
1147         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1148         tarball.set_overlay(OverlayKind::Clippy);
1149         tarball.is_preview(true);
1150         tarball.add_file(clippy, "bin", 0o755);
1151         tarball.add_file(cargoclippy, "bin", 0o755);
1152         tarball.add_legal_and_readme_to("share/doc/clippy");
1153         Some(tarball.generate())
1154     }
1155 }
1156
1157 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1158 pub struct Miri {
1159     pub compiler: Compiler,
1160     pub target: TargetSelection,
1161 }
1162
1163 impl Step for Miri {
1164     type Output = Option<GeneratedTarball>;
1165     const DEFAULT: bool = true;
1166     const ONLY_HOSTS: bool = true;
1167
1168     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1169         let default = should_build_extended_tool(&run.builder, "miri");
1170         run.alias("miri").default_condition(default)
1171     }
1172
1173     fn make_run(run: RunConfig<'_>) {
1174         run.builder.ensure(Miri {
1175             compiler: run.builder.compiler_for(
1176                 run.builder.top_stage,
1177                 run.builder.config.build,
1178                 run.target,
1179             ),
1180             target: run.target,
1181         });
1182     }
1183
1184     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1185         // This prevents miri from being built for "dist" or "install"
1186         // on the stable/beta channels. It is a nightly-only tool and should
1187         // not be included.
1188         if !builder.build.unstable_features() {
1189             return None;
1190         }
1191         let compiler = self.compiler;
1192         let target = self.target;
1193
1194         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1195         let cargomiri =
1196             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1197
1198         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1199         tarball.set_overlay(OverlayKind::Miri);
1200         tarball.is_preview(true);
1201         tarball.add_file(miri, "bin", 0o755);
1202         tarball.add_file(cargomiri, "bin", 0o755);
1203         tarball.add_legal_and_readme_to("share/doc/miri");
1204         Some(tarball.generate())
1205     }
1206 }
1207
1208 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1209 pub struct Rustfmt {
1210     pub compiler: Compiler,
1211     pub target: TargetSelection,
1212 }
1213
1214 impl Step for Rustfmt {
1215     type Output = Option<GeneratedTarball>;
1216     const DEFAULT: bool = true;
1217     const ONLY_HOSTS: bool = true;
1218
1219     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1220         let default = should_build_extended_tool(&run.builder, "rustfmt");
1221         run.alias("rustfmt").default_condition(default)
1222     }
1223
1224     fn make_run(run: RunConfig<'_>) {
1225         run.builder.ensure(Rustfmt {
1226             compiler: run.builder.compiler_for(
1227                 run.builder.top_stage,
1228                 run.builder.config.build,
1229                 run.target,
1230             ),
1231             target: run.target,
1232         });
1233     }
1234
1235     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1236         let compiler = self.compiler;
1237         let target = self.target;
1238
1239         let rustfmt = builder
1240             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1241             .expect("rustfmt expected to build - essential tool");
1242         let cargofmt = builder
1243             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1244             .expect("cargo fmt expected to build - essential tool");
1245         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1246         tarball.set_overlay(OverlayKind::Rustfmt);
1247         tarball.is_preview(true);
1248         tarball.add_file(rustfmt, "bin", 0o755);
1249         tarball.add_file(cargofmt, "bin", 0o755);
1250         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1251         Some(tarball.generate())
1252     }
1253 }
1254
1255 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1256 pub struct RustDemangler {
1257     pub compiler: Compiler,
1258     pub target: TargetSelection,
1259 }
1260
1261 impl Step for RustDemangler {
1262     type Output = Option<GeneratedTarball>;
1263     const DEFAULT: bool = true;
1264     const ONLY_HOSTS: bool = true;
1265
1266     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1267         // While other tools use `should_build_extended_tool` to decide whether to be run by
1268         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1269         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1270         // we run the step by default when only `extended = true`, and decide whether to actually
1271         // run it or not later.
1272         let default = run.builder.config.extended;
1273         run.alias("rust-demangler").default_condition(default)
1274     }
1275
1276     fn make_run(run: RunConfig<'_>) {
1277         run.builder.ensure(RustDemangler {
1278             compiler: run.builder.compiler_for(
1279                 run.builder.top_stage,
1280                 run.builder.config.build,
1281                 run.target,
1282             ),
1283             target: run.target,
1284         });
1285     }
1286
1287     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1288         let compiler = self.compiler;
1289         let target = self.target;
1290
1291         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1292         let condition = should_build_extended_tool(builder, "rust-demangler")
1293             || builder.config.profiler_enabled(target);
1294         if builder.config.extended && !condition {
1295             return None;
1296         }
1297
1298         let rust_demangler = builder
1299             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1300             .expect("rust-demangler expected to build - in-tree tool");
1301
1302         // Prepare the image directory
1303         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1304         tarball.set_overlay(OverlayKind::RustDemangler);
1305         tarball.is_preview(true);
1306         tarball.add_file(&rust_demangler, "bin", 0o755);
1307         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1308         Some(tarball.generate())
1309     }
1310 }
1311
1312 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1313 pub struct Extended {
1314     stage: u32,
1315     host: TargetSelection,
1316     target: TargetSelection,
1317 }
1318
1319 impl Step for Extended {
1320     type Output = ();
1321     const DEFAULT: bool = true;
1322     const ONLY_HOSTS: bool = true;
1323
1324     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1325         let builder = run.builder;
1326         run.alias("extended").default_condition(builder.config.extended)
1327     }
1328
1329     fn make_run(run: RunConfig<'_>) {
1330         run.builder.ensure(Extended {
1331             stage: run.builder.top_stage,
1332             host: run.builder.config.build,
1333             target: run.target,
1334         });
1335     }
1336
1337     /// Creates a combined installer for the specified target in the provided stage.
1338     fn run(self, builder: &Builder<'_>) {
1339         let target = self.target;
1340         let stage = self.stage;
1341         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1342
1343         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1344
1345         let mut tarballs = Vec::new();
1346         let mut built_tools = HashSet::new();
1347         macro_rules! add_component {
1348             ($name:expr => $step:expr) => {
1349                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1350                     tarballs.push(tarball);
1351                     built_tools.insert($name);
1352                 }
1353             };
1354         }
1355
1356         // When rust-std package split from rustc, we needed to ensure that during
1357         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1358         // the std files during uninstall. To do this ensure that rustc comes
1359         // before rust-std in the list below.
1360         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1361         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1362
1363         if target.ends_with("windows-gnu") {
1364             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1365         }
1366
1367         add_component!("rust-docs" => Docs { host: target });
1368         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1369         add_component!("cargo" => Cargo { compiler, target });
1370         add_component!("rustfmt" => Rustfmt { compiler, target });
1371         add_component!("rls" => Rls { compiler, target });
1372         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1373         add_component!("llvm-components" => LlvmTools { target });
1374         add_component!("clippy" => Clippy { compiler, target });
1375         add_component!("miri" => Miri { compiler, target });
1376         add_component!("analysis" => Analysis { compiler, target });
1377
1378         let etc = builder.src.join("src/etc/installer");
1379
1380         // Avoid producing tarballs during a dry run.
1381         if builder.config.dry_run {
1382             return;
1383         }
1384
1385         let tarball = Tarball::new(builder, "rust", &target.triple);
1386         let generated = tarball.combine(&tarballs);
1387
1388         let tmp = tmpdir(builder).join("combined-tarball");
1389         let work = generated.work_dir();
1390
1391         let mut license = String::new();
1392         license += &builder.read(&builder.src.join("COPYRIGHT"));
1393         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1394         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1395         license.push('\n');
1396         license.push('\n');
1397
1398         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1399         let mut rtf = rtf.to_string();
1400         rtf.push('\n');
1401         for line in license.lines() {
1402             rtf.push_str(line);
1403             rtf.push_str("\\line ");
1404         }
1405         rtf.push('}');
1406
1407         fn filter(contents: &str, marker: &str) -> String {
1408             let start = format!("tool-{}-start", marker);
1409             let end = format!("tool-{}-end", marker);
1410             let mut lines = Vec::new();
1411             let mut omitted = false;
1412             for line in contents.lines() {
1413                 if line.contains(&start) {
1414                     omitted = true;
1415                 } else if line.contains(&end) {
1416                     omitted = false;
1417                 } else if !omitted {
1418                     lines.push(line);
1419                 }
1420             }
1421
1422             lines.join("\n")
1423         }
1424
1425         let xform = |p: &Path| {
1426             let mut contents = t!(fs::read_to_string(p));
1427             for tool in &["rust-demangler"] {
1428                 if !built_tools.contains(tool) {
1429                     contents = filter(&contents, tool);
1430                 }
1431             }
1432             let ret = tmp.join(p.file_name().unwrap());
1433             t!(fs::write(&ret, &contents));
1434             ret
1435         };
1436
1437         if target.contains("apple-darwin") {
1438             builder.info("building pkg installer");
1439             let pkg = tmp.join("pkg");
1440             let _ = fs::remove_dir_all(&pkg);
1441
1442             let pkgbuild = |component: &str| {
1443                 let mut cmd = Command::new("pkgbuild");
1444                 cmd.arg("--identifier")
1445                     .arg(format!("org.rust-lang.{}", component))
1446                     .arg("--scripts")
1447                     .arg(pkg.join(component))
1448                     .arg("--nopayload")
1449                     .arg(pkg.join(component).with_extension("pkg"));
1450                 builder.run(&mut cmd);
1451             };
1452
1453             let prepare = |name: &str| {
1454                 builder.create_dir(&pkg.join(name));
1455                 builder.cp_r(
1456                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1457                     &pkg.join(name),
1458                 );
1459                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1460                 pkgbuild(name);
1461             };
1462             prepare("rustc");
1463             prepare("cargo");
1464             prepare("rust-std");
1465             prepare("rust-analysis");
1466             prepare("clippy");
1467             prepare("miri");
1468             prepare("rust-analyzer");
1469             for tool in &["rust-docs", "rust-demangler"] {
1470                 if built_tools.contains(tool) {
1471                     prepare(tool);
1472                 }
1473             }
1474             // create an 'uninstall' package
1475             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1476             pkgbuild("uninstall");
1477
1478             builder.create_dir(&pkg.join("res"));
1479             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1480             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1481             let mut cmd = Command::new("productbuild");
1482             cmd.arg("--distribution")
1483                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1484                 .arg("--resources")
1485                 .arg(pkg.join("res"))
1486                 .arg(distdir(builder).join(format!(
1487                     "{}-{}.pkg",
1488                     pkgname(builder, "rust"),
1489                     target.triple
1490                 )))
1491                 .arg("--package-path")
1492                 .arg(&pkg);
1493             let _time = timeit(builder);
1494             builder.run(&mut cmd);
1495         }
1496
1497         if target.contains("windows") {
1498             let exe = tmp.join("exe");
1499             let _ = fs::remove_dir_all(&exe);
1500
1501             let prepare = |name: &str| {
1502                 builder.create_dir(&exe.join(name));
1503                 let dir = if name == "rust-std" || name == "rust-analysis" {
1504                     format!("{}-{}", name, target.triple)
1505                 } else if name == "rust-analyzer" {
1506                     "rust-analyzer-preview".to_string()
1507                 } else if name == "clippy" {
1508                     "clippy-preview".to_string()
1509                 } else if name == "rust-demangler" {
1510                     "rust-demangler-preview".to_string()
1511                 } else if name == "miri" {
1512                     "miri-preview".to_string()
1513                 } else {
1514                     name.to_string()
1515                 };
1516                 builder.cp_r(
1517                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1518                     &exe.join(name),
1519                 );
1520                 builder.remove(&exe.join(name).join("manifest.in"));
1521             };
1522             prepare("rustc");
1523             prepare("cargo");
1524             prepare("rust-analysis");
1525             prepare("rust-docs");
1526             prepare("rust-std");
1527             prepare("clippy");
1528             prepare("miri");
1529             prepare("rust-analyzer");
1530             for tool in &["rust-demangler"] {
1531                 if built_tools.contains(tool) {
1532                     prepare(tool);
1533                 }
1534             }
1535             if target.ends_with("windows-gnu") {
1536                 prepare("rust-mingw");
1537             }
1538
1539             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1540
1541             // Generate msi installer
1542             let wix_path = env::var_os("WIX")
1543                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1544             let wix = PathBuf::from(wix_path);
1545             let heat = wix.join("bin/heat.exe");
1546             let candle = wix.join("bin/candle.exe");
1547             let light = wix.join("bin/light.exe");
1548
1549             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1550             builder.run(
1551                 Command::new(&heat)
1552                     .current_dir(&exe)
1553                     .arg("dir")
1554                     .arg("rustc")
1555                     .args(&heat_flags)
1556                     .arg("-cg")
1557                     .arg("RustcGroup")
1558                     .arg("-dr")
1559                     .arg("Rustc")
1560                     .arg("-var")
1561                     .arg("var.RustcDir")
1562                     .arg("-out")
1563                     .arg(exe.join("RustcGroup.wxs")),
1564             );
1565             builder.run(
1566                 Command::new(&heat)
1567                     .current_dir(&exe)
1568                     .arg("dir")
1569                     .arg("rust-docs")
1570                     .args(&heat_flags)
1571                     .arg("-cg")
1572                     .arg("DocsGroup")
1573                     .arg("-dr")
1574                     .arg("Docs")
1575                     .arg("-var")
1576                     .arg("var.DocsDir")
1577                     .arg("-out")
1578                     .arg(exe.join("DocsGroup.wxs"))
1579                     .arg("-t")
1580                     .arg(etc.join("msi/squash-components.xsl")),
1581             );
1582             builder.run(
1583                 Command::new(&heat)
1584                     .current_dir(&exe)
1585                     .arg("dir")
1586                     .arg("cargo")
1587                     .args(&heat_flags)
1588                     .arg("-cg")
1589                     .arg("CargoGroup")
1590                     .arg("-dr")
1591                     .arg("Cargo")
1592                     .arg("-var")
1593                     .arg("var.CargoDir")
1594                     .arg("-out")
1595                     .arg(exe.join("CargoGroup.wxs"))
1596                     .arg("-t")
1597                     .arg(etc.join("msi/remove-duplicates.xsl")),
1598             );
1599             builder.run(
1600                 Command::new(&heat)
1601                     .current_dir(&exe)
1602                     .arg("dir")
1603                     .arg("rust-std")
1604                     .args(&heat_flags)
1605                     .arg("-cg")
1606                     .arg("StdGroup")
1607                     .arg("-dr")
1608                     .arg("Std")
1609                     .arg("-var")
1610                     .arg("var.StdDir")
1611                     .arg("-out")
1612                     .arg(exe.join("StdGroup.wxs")),
1613             );
1614             builder.run(
1615                 Command::new(&heat)
1616                     .current_dir(&exe)
1617                     .arg("dir")
1618                     .arg("rust-analyzer")
1619                     .args(&heat_flags)
1620                     .arg("-cg")
1621                     .arg("RustAnalyzerGroup")
1622                     .arg("-dr")
1623                     .arg("RustAnalyzer")
1624                     .arg("-var")
1625                     .arg("var.RustAnalyzerDir")
1626                     .arg("-out")
1627                     .arg(exe.join("RustAnalyzerGroup.wxs"))
1628                     .arg("-t")
1629                     .arg(etc.join("msi/remove-duplicates.xsl")),
1630             );
1631             builder.run(
1632                 Command::new(&heat)
1633                     .current_dir(&exe)
1634                     .arg("dir")
1635                     .arg("clippy")
1636                     .args(&heat_flags)
1637                     .arg("-cg")
1638                     .arg("ClippyGroup")
1639                     .arg("-dr")
1640                     .arg("Clippy")
1641                     .arg("-var")
1642                     .arg("var.ClippyDir")
1643                     .arg("-out")
1644                     .arg(exe.join("ClippyGroup.wxs"))
1645                     .arg("-t")
1646                     .arg(etc.join("msi/remove-duplicates.xsl")),
1647             );
1648             if built_tools.contains("rust-demangler") {
1649                 builder.run(
1650                     Command::new(&heat)
1651                         .current_dir(&exe)
1652                         .arg("dir")
1653                         .arg("rust-demangler")
1654                         .args(&heat_flags)
1655                         .arg("-cg")
1656                         .arg("RustDemanglerGroup")
1657                         .arg("-dr")
1658                         .arg("RustDemangler")
1659                         .arg("-var")
1660                         .arg("var.RustDemanglerDir")
1661                         .arg("-out")
1662                         .arg(exe.join("RustDemanglerGroup.wxs"))
1663                         .arg("-t")
1664                         .arg(etc.join("msi/remove-duplicates.xsl")),
1665                 );
1666             }
1667             builder.run(
1668                 Command::new(&heat)
1669                     .current_dir(&exe)
1670                     .arg("dir")
1671                     .arg("miri")
1672                     .args(&heat_flags)
1673                     .arg("-cg")
1674                     .arg("MiriGroup")
1675                     .arg("-dr")
1676                     .arg("Miri")
1677                     .arg("-var")
1678                     .arg("var.MiriDir")
1679                     .arg("-out")
1680                     .arg(exe.join("MiriGroup.wxs"))
1681                     .arg("-t")
1682                     .arg(etc.join("msi/remove-duplicates.xsl")),
1683             );
1684             builder.run(
1685                 Command::new(&heat)
1686                     .current_dir(&exe)
1687                     .arg("dir")
1688                     .arg("rust-analysis")
1689                     .args(&heat_flags)
1690                     .arg("-cg")
1691                     .arg("AnalysisGroup")
1692                     .arg("-dr")
1693                     .arg("Analysis")
1694                     .arg("-var")
1695                     .arg("var.AnalysisDir")
1696                     .arg("-out")
1697                     .arg(exe.join("AnalysisGroup.wxs"))
1698                     .arg("-t")
1699                     .arg(etc.join("msi/remove-duplicates.xsl")),
1700             );
1701             if target.ends_with("windows-gnu") {
1702                 builder.run(
1703                     Command::new(&heat)
1704                         .current_dir(&exe)
1705                         .arg("dir")
1706                         .arg("rust-mingw")
1707                         .args(&heat_flags)
1708                         .arg("-cg")
1709                         .arg("GccGroup")
1710                         .arg("-dr")
1711                         .arg("Gcc")
1712                         .arg("-var")
1713                         .arg("var.GccDir")
1714                         .arg("-out")
1715                         .arg(exe.join("GccGroup.wxs")),
1716                 );
1717             }
1718
1719             let candle = |input: &Path| {
1720                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1721                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1722                 let mut cmd = Command::new(&candle);
1723                 cmd.current_dir(&exe)
1724                     .arg("-nologo")
1725                     .arg("-dRustcDir=rustc")
1726                     .arg("-dDocsDir=rust-docs")
1727                     .arg("-dCargoDir=cargo")
1728                     .arg("-dStdDir=rust-std")
1729                     .arg("-dAnalysisDir=rust-analysis")
1730                     .arg("-dClippyDir=clippy")
1731                     .arg("-dMiriDir=miri")
1732                     .arg("-arch")
1733                     .arg(&arch)
1734                     .arg("-out")
1735                     .arg(&output)
1736                     .arg(&input);
1737                 add_env(builder, &mut cmd, target);
1738
1739                 if built_tools.contains("rust-demangler") {
1740                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1741                 }
1742                 if built_tools.contains("rust-analyzer") {
1743                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1744                 }
1745                 if target.ends_with("windows-gnu") {
1746                     cmd.arg("-dGccDir=rust-mingw");
1747                 }
1748                 builder.run(&mut cmd);
1749             };
1750             candle(&xform(&etc.join("msi/rust.wxs")));
1751             candle(&etc.join("msi/ui.wxs"));
1752             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1753             candle("RustcGroup.wxs".as_ref());
1754             candle("DocsGroup.wxs".as_ref());
1755             candle("CargoGroup.wxs".as_ref());
1756             candle("StdGroup.wxs".as_ref());
1757             candle("ClippyGroup.wxs".as_ref());
1758             candle("MiriGroup.wxs".as_ref());
1759             if built_tools.contains("rust-demangler") {
1760                 candle("RustDemanglerGroup.wxs".as_ref());
1761             }
1762             if built_tools.contains("rust-analyzer") {
1763                 candle("RustAnalyzerGroup.wxs".as_ref());
1764             }
1765             candle("AnalysisGroup.wxs".as_ref());
1766
1767             if target.ends_with("windows-gnu") {
1768                 candle("GccGroup.wxs".as_ref());
1769             }
1770
1771             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1772             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1773             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1774
1775             builder.info(&format!("building `msi` installer with {:?}", light));
1776             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1777             let mut cmd = Command::new(&light);
1778             cmd.arg("-nologo")
1779                 .arg("-ext")
1780                 .arg("WixUIExtension")
1781                 .arg("-ext")
1782                 .arg("WixUtilExtension")
1783                 .arg("-out")
1784                 .arg(exe.join(&filename))
1785                 .arg("rust.wixobj")
1786                 .arg("ui.wixobj")
1787                 .arg("rustwelcomedlg.wixobj")
1788                 .arg("RustcGroup.wixobj")
1789                 .arg("DocsGroup.wixobj")
1790                 .arg("CargoGroup.wixobj")
1791                 .arg("StdGroup.wixobj")
1792                 .arg("AnalysisGroup.wixobj")
1793                 .arg("ClippyGroup.wixobj")
1794                 .arg("MiriGroup.wixobj")
1795                 .current_dir(&exe);
1796
1797             if built_tools.contains("rust-analyzer") {
1798                 cmd.arg("RustAnalyzerGroup.wixobj");
1799             }
1800             if built_tools.contains("rust-demangler") {
1801                 cmd.arg("RustDemanglerGroup.wixobj");
1802             }
1803
1804             if target.ends_with("windows-gnu") {
1805                 cmd.arg("GccGroup.wixobj");
1806             }
1807             // ICE57 wrongly complains about the shortcuts
1808             cmd.arg("-sice:ICE57");
1809
1810             let _time = timeit(builder);
1811             builder.run(&mut cmd);
1812
1813             if !builder.config.dry_run {
1814                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1815             }
1816         }
1817     }
1818 }
1819
1820 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1821     let mut parts = builder.version.split('.');
1822     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1823         .env("CFG_RELEASE_NUM", &builder.version)
1824         .env("CFG_RELEASE", builder.rust_release())
1825         .env("CFG_VER_MAJOR", parts.next().unwrap())
1826         .env("CFG_VER_MINOR", parts.next().unwrap())
1827         .env("CFG_VER_PATCH", parts.next().unwrap())
1828         .env("CFG_VER_BUILD", "0") // just needed to build
1829         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1830         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1831         .env("CFG_BUILD", target.triple)
1832         .env("CFG_CHANNEL", &builder.config.channel);
1833
1834     if target.contains("windows-gnullvm") {
1835         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1836     } else if target.contains("windows-gnu") {
1837         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1838     } else {
1839         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1840     }
1841 }
1842
1843 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1844 ///
1845 /// Returns whether the files were actually copied.
1846 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1847     if !builder.is_rust_llvm(target) {
1848         // If the LLVM was externally provided, then we don't currently copy
1849         // artifacts into the sysroot. This is not necessarily the right
1850         // choice (in particular, it will require the LLVM dylib to be in
1851         // the linker's load path at runtime), but the common use case for
1852         // external LLVMs is distribution provided LLVMs, and in that case
1853         // they're usually in the standard search path (e.g., /usr/lib) and
1854         // copying them here is going to cause problems as we may end up
1855         // with the wrong files and isn't what distributions want.
1856         //
1857         // This behavior may be revisited in the future though.
1858         //
1859         // If the LLVM is coming from ourselves (just from CI) though, we
1860         // still want to install it, as it otherwise won't be available.
1861         return false;
1862     }
1863
1864     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1865     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1866     // clear why this is the case, though. llvm-config will emit the versioned
1867     // paths and we don't want those in the sysroot (as we're expecting
1868     // unversioned paths).
1869     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1870         let src_libdir = builder.llvm_out(target).join("lib");
1871         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1872         if llvm_dylib_path.exists() {
1873             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1874         }
1875         !builder.config.dry_run
1876     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1877         let mut cmd = Command::new(llvm_config);
1878         cmd.arg("--libfiles");
1879         builder.verbose(&format!("running {:?}", cmd));
1880         let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) };
1881         let build_llvm_out = &builder.llvm_out(builder.config.build);
1882         let target_llvm_out = &builder.llvm_out(target);
1883         for file in files.trim_end().split(' ') {
1884             // If we're not using a custom LLVM, make sure we package for the target.
1885             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1886                 target_llvm_out.join(relative_path)
1887             } else {
1888                 PathBuf::from(file)
1889             };
1890             builder.install(&file, dst_libdir, 0o644);
1891         }
1892         !builder.config.dry_run
1893     } else {
1894         false
1895     }
1896 }
1897
1898 /// Maybe add libLLVM.so to the target lib-dir for linking.
1899 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1900     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1901     // We do not need to copy LLVM files into the sysroot if it is not
1902     // dynamically linked; it is already included into librustc_llvm
1903     // statically.
1904     if builder.llvm_link_shared() {
1905         maybe_install_llvm(builder, target, &dst_libdir);
1906     }
1907 }
1908
1909 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1910 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1911     let dst_libdir =
1912         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1913     // We do not need to copy LLVM files into the sysroot if it is not
1914     // dynamically linked; it is already included into librustc_llvm
1915     // statically.
1916     if builder.llvm_link_shared() {
1917         maybe_install_llvm(builder, target, &dst_libdir);
1918     }
1919 }
1920
1921 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1922 pub struct LlvmTools {
1923     pub target: TargetSelection,
1924 }
1925
1926 impl Step for LlvmTools {
1927     type Output = Option<GeneratedTarball>;
1928     const ONLY_HOSTS: bool = true;
1929     const DEFAULT: bool = true;
1930
1931     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1932         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1933         // FIXME: allow using the names of the tools themselves?
1934         run.alias("llvm-tools").default_condition(default)
1935     }
1936
1937     fn make_run(run: RunConfig<'_>) {
1938         run.builder.ensure(LlvmTools { target: run.target });
1939     }
1940
1941     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1942         let target = self.target;
1943
1944         /* run only if llvm-config isn't used */
1945         if let Some(config) = builder.config.target_config.get(&target) {
1946             if let Some(ref _s) = config.llvm_config {
1947                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1948                 return None;
1949             }
1950         }
1951
1952         builder.ensure(crate::native::Llvm { target });
1953
1954         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1955         tarball.set_overlay(OverlayKind::LLVM);
1956         tarball.is_preview(true);
1957
1958         // Prepare the image directory
1959         let src_bindir = builder.llvm_out(target).join("bin");
1960         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1961         for tool in LLVM_TOOLS {
1962             let exe = src_bindir.join(exe(tool, target));
1963             tarball.add_file(&exe, &dst_bindir, 0o755);
1964         }
1965
1966         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1967         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1968         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1969         // compiler libraries.
1970         maybe_install_llvm_target(builder, target, tarball.image_dir());
1971
1972         Some(tarball.generate())
1973     }
1974 }
1975
1976 // Tarball intended for internal consumption to ease rustc/std development.
1977 //
1978 // Should not be considered stable by end users.
1979 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1980 pub struct RustDev {
1981     pub target: TargetSelection,
1982 }
1983
1984 impl Step for RustDev {
1985     type Output = Option<GeneratedTarball>;
1986     const DEFAULT: bool = true;
1987     const ONLY_HOSTS: bool = true;
1988
1989     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1990         run.alias("rust-dev")
1991     }
1992
1993     fn make_run(run: RunConfig<'_>) {
1994         run.builder.ensure(RustDev { target: run.target });
1995     }
1996
1997     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1998         let target = self.target;
1999
2000         /* run only if llvm-config isn't used */
2001         if let Some(config) = builder.config.target_config.get(&target) {
2002             if let Some(ref _s) = config.llvm_config {
2003                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2004                 return None;
2005             }
2006         }
2007
2008         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2009         tarball.set_overlay(OverlayKind::LLVM);
2010
2011         builder.ensure(crate::native::Llvm { target });
2012
2013         let src_bindir = builder.llvm_out(target).join("bin");
2014         // If updating this list, you likely want to change
2015         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2016         // will not pick up the extra file until LLVM gets bumped.
2017         for bin in &[
2018             "llvm-config",
2019             "llvm-ar",
2020             "llvm-objdump",
2021             "llvm-profdata",
2022             "llvm-bcanalyzer",
2023             "llvm-cov",
2024             "llvm-dwp",
2025             "llvm-nm",
2026             "llvm-dwarfdump",
2027             "llvm-dis",
2028         ] {
2029             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2030         }
2031
2032         // We don't build LLD on some platforms, so only add it if it exists
2033         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2034         if lld_path.exists() {
2035             tarball.add_file(lld_path, "bin", 0o755);
2036         }
2037
2038         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2039
2040         // Copy the include directory as well; needed mostly to build
2041         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2042         // just broadly useful to be able to link against the bundled LLVM.
2043         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2044
2045         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2046         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2047         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2048         // compiler libraries.
2049         let dst_libdir = tarball.image_dir().join("lib");
2050         maybe_install_llvm(builder, target, &dst_libdir);
2051         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2052         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2053
2054         Some(tarball.generate())
2055     }
2056 }
2057
2058 // Tarball intended for internal consumption to ease rustc/std development.
2059 //
2060 // Should not be considered stable by end users.
2061 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2062 pub struct Bootstrap {
2063     pub target: TargetSelection,
2064 }
2065
2066 impl Step for Bootstrap {
2067     type Output = Option<GeneratedTarball>;
2068     const DEFAULT: bool = false;
2069     const ONLY_HOSTS: bool = true;
2070
2071     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2072         run.alias("bootstrap")
2073     }
2074
2075     fn make_run(run: RunConfig<'_>) {
2076         run.builder.ensure(Bootstrap { target: run.target });
2077     }
2078
2079     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2080         let target = self.target;
2081
2082         let tarball = Tarball::new(builder, "bootstrap", &target.triple);
2083
2084         let bootstrap_outdir = &builder.bootstrap_out;
2085         for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
2086             tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
2087         }
2088
2089         Some(tarball.generate())
2090     }
2091 }
2092
2093 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2094 /// release process to avoid cloning the monorepo and building stuff.
2095 ///
2096 /// Should not be considered stable by end users.
2097 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2098 pub struct BuildManifest {
2099     pub target: TargetSelection,
2100 }
2101
2102 impl Step for BuildManifest {
2103     type Output = GeneratedTarball;
2104     const DEFAULT: bool = false;
2105     const ONLY_HOSTS: bool = true;
2106
2107     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2108         run.alias("build-manifest")
2109     }
2110
2111     fn make_run(run: RunConfig<'_>) {
2112         run.builder.ensure(BuildManifest { target: run.target });
2113     }
2114
2115     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2116         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2117
2118         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2119         tarball.add_file(&build_manifest, "bin", 0o755);
2120         tarball.generate()
2121     }
2122 }
2123
2124 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2125 ///
2126 /// Currently this is the PGO profile data.
2127 ///
2128 /// Should not be considered stable by end users.
2129 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2130 pub struct ReproducibleArtifacts {
2131     pub target: TargetSelection,
2132 }
2133
2134 impl Step for ReproducibleArtifacts {
2135     type Output = Option<GeneratedTarball>;
2136     const DEFAULT: bool = true;
2137     const ONLY_HOSTS: bool = true;
2138
2139     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2140         run.alias("reproducible-artifacts")
2141     }
2142
2143     fn make_run(run: RunConfig<'_>) {
2144         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2145     }
2146
2147     fn run(self, builder: &Builder<'_>) -> Self::Output {
2148         let mut added_anything = false;
2149         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2150         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2151             tarball.add_file(path, ".", 0o644);
2152             added_anything = true;
2153         }
2154         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2155             tarball.add_file(path, ".", 0o644);
2156             added_anything = true;
2157         }
2158         if added_anything { Some(tarball.generate()) } else { None }
2159     }
2160 }