src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
  43         panic!(
  44             "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
  45             tool_name, help
  46         )
  47     }
  48 }
  49
  50 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  51     if !builder.config.extended {
  52         return false;
  53     }
  54     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  55 }
  56
  57 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  58 pub struct Docs {
  59     pub host: TargetSelection,
  60 }
  61
  62 impl Step for Docs {
  63     type Output = Option<GeneratedTarball>;
  64     const DEFAULT: bool = true;
  65
  66     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  67         let default = run.builder.config.docs;
  68         run.alias("rust-docs").default_condition(default)
  69     }
  70
  71     fn make_run(run: RunConfig<'_>) {
  72         run.builder.ensure(Docs { host: run.target });
  73     }
  74
  75     /// Builds the `rust-docs` installer component.
  76     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  77         let host = self.host;
  78         builder.default_doc(&[]);
  79
  80         let dest = "share/doc/rust/html";
  81
  82         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  83         tarball.set_product_name("Rust Documentation");
  84         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  85         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  86         Some(tarball.generate())
  87     }
  88 }
  89
  90 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  91 pub struct RustcDocs {
  92     pub host: TargetSelection,
  93 }
  94
  95 impl Step for RustcDocs {
  96     type Output = Option<GeneratedTarball>;
  97     const DEFAULT: bool = true;
  98
  99     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 100         let builder = run.builder;
 101         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 102     }
 103
 104     fn make_run(run: RunConfig<'_>) {
 105         run.builder.ensure(RustcDocs { host: run.target });
 106     }
 107
 108     /// Builds the `rustc-docs` installer component.
 109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 110         let host = self.host;
 111         builder.default_doc(&[]);
 112
 113         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 114         tarball.set_product_name("Rustc Documentation");
 115         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 116         Some(tarball.generate())
 117     }
 118 }
 119
 120 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 121     let mut found = Vec::with_capacity(files.len());
 122
 123     for file in files {
 124         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 125
 126         if let Some(file_path) = file_path {
 127             found.push(file_path);
 128         } else {
 129             panic!("Could not find '{}' in {:?}", file, path);
 130         }
 131     }
 132
 133     found
 134 }
 135
 136 fn make_win_dist(
 137     rust_root: &Path,
 138     plat_root: &Path,
 139     target: TargetSelection,
 140     builder: &Builder<'_>,
 141 ) {
 142     //Ask gcc where it keeps its stuff
 143     let mut cmd = Command::new(builder.cc(target));
 144     cmd.arg("-print-search-dirs");
 145     let gcc_out = output(&mut cmd);
 146
 147     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 148     let mut lib_path = Vec::new();
 149
 150     for line in gcc_out.lines() {
 151         let idx = line.find(':').unwrap();
 152         let key = &line[..idx];
 153         let trim_chars: &[_] = &[' ', '='];
 154         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 155
 156         if key == "programs" {
 157             bin_path.extend(value);
 158         } else if key == "libraries" {
 159             lib_path.extend(value);
 160         }
 161     }
 162
 163     let compiler = if target == "i686-pc-windows-gnu" {
 164         "i686-w64-mingw32-gcc.exe"
 165     } else if target == "x86_64-pc-windows-gnu" {
 166         "x86_64-w64-mingw32-gcc.exe"
 167     } else {
 168         "gcc.exe"
 169     };
 170     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 171     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 172     if target.starts_with("i686-") {
 173         rustc_dlls.push("libgcc_s_dw2-1.dll");
 174     } else {
 175         rustc_dlls.push("libgcc_s_seh-1.dll");
 176     }
 177
 178     let target_libs = [
 179         //MinGW libs
 180         "libgcc.a",
 181         "libgcc_eh.a",
 182         "libgcc_s.a",
 183         "libm.a",
 184         "libmingw32.a",
 185         "libmingwex.a",
 186         "libstdc++.a",
 187         "libiconv.a",
 188         "libmoldname.a",
 189         "libpthread.a",
 190         //Windows import libs
 191         "libadvapi32.a",
 192         "libbcrypt.a",
 193         "libcomctl32.a",
 194         "libcomdlg32.a",
 195         "libcredui.a",
 196         "libcrypt32.a",
 197         "libdbghelp.a",
 198         "libgdi32.a",
 199         "libimagehlp.a",
 200         "libiphlpapi.a",
 201         "libkernel32.a",
 202         "libmsimg32.a",
 203         "libmsvcrt.a",
 204         "libodbc32.a",
 205         "libole32.a",
 206         "liboleaut32.a",
 207         "libopengl32.a",
 208         "libpsapi.a",
 209         "librpcrt4.a",
 210         "libsecur32.a",
 211         "libsetupapi.a",
 212         "libshell32.a",
 213         "libsynchronization.a",
 214         "libuser32.a",
 215         "libuserenv.a",
 216         "libuuid.a",
 217         "libwinhttp.a",
 218         "libwinmm.a",
 219         "libwinspool.a",
 220         "libws2_32.a",
 221         "libwsock32.a",
 222     ];
 223
 224     //Find mingw artifacts we want to bundle
 225     let target_tools = find_files(&target_tools, &bin_path);
 226     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 227     let target_libs = find_files(&target_libs, &lib_path);
 228
 229     // Copy runtime dlls next to rustc.exe
 230     let dist_bin_dir = rust_root.join("bin/");
 231     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 232     for src in rustc_dlls {
 233         builder.copy_to_folder(&src, &dist_bin_dir);
 234     }
 235
 236     //Copy platform tools to platform-specific bin directory
 237     let target_bin_dir = plat_root
 238         .join("lib")
 239         .join("rustlib")
 240         .join(target.triple)
 241         .join("bin")
 242         .join("self-contained");
 243     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 244     for src in target_tools {
 245         builder.copy_to_folder(&src, &target_bin_dir);
 246     }
 247
 248     // Warn windows-gnu users that the bundled GCC cannot compile C files
 249     builder.create(
 250         &target_bin_dir.join("GCC-WARNING.txt"),
 251         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 252          used as a linker. In order to be able to compile projects containing C code use \
 253          the GCC provided by MinGW or Cygwin.",
 254     );
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root
 258         .join("lib")
 259         .join("rustlib")
 260         .join(target.triple)
 261         .join("lib")
 262         .join("self-contained");
 263     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 264     for src in target_libs {
 265         builder.copy_to_folder(&src, &target_lib_dir);
 266     }
 267 }
 268
 269 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 270 pub struct Mingw {
 271     pub host: TargetSelection,
 272 }
 273
 274 impl Step for Mingw {
 275     type Output = Option<GeneratedTarball>;
 276     const DEFAULT: bool = true;
 277
 278     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 279         run.alias("rust-mingw")
 280     }
 281
 282     fn make_run(run: RunConfig<'_>) {
 283         run.builder.ensure(Mingw { host: run.target });
 284     }
 285
 286     /// Builds the `rust-mingw` installer component.
 287     ///
 288     /// This contains all the bits and pieces to run the MinGW Windows targets
 289     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 290     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 291         let host = self.host;
 292         if !host.ends_with("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 297         tarball.set_product_name("Rust MinGW");
 298
 299         // The first argument is a "temporary directory" which is just
 300         // thrown away (this contains the runtime DLLs included in the rustc package
 301         // above) and the second argument is where to place all the MinGW components
 302         // (which is what we want).
 303         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 304
 305         Some(tarball.generate())
 306     }
 307 }
 308
 309 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 310 pub struct Rustc {
 311     pub compiler: Compiler,
 312 }
 313
 314 impl Step for Rustc {
 315     type Output = GeneratedTarball;
 316     const DEFAULT: bool = true;
 317     const ONLY_HOSTS: bool = true;
 318
 319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 320         run.alias("rustc")
 321     }
 322
 323     fn make_run(run: RunConfig<'_>) {
 324         run.builder
 325             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 326     }
 327
 328     /// Creates the `rustc` installer component.
 329     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 330         let compiler = self.compiler;
 331         let host = self.compiler.host;
 332
 333         let tarball = Tarball::new(builder, "rustc", &host.triple);
 334
 335         // Prepare the rustc "image", what will actually end up getting installed
 336         prepare_image(builder, compiler, tarball.image_dir());
 337
 338         // On MinGW we've got a few runtime DLL dependencies that we need to
 339         // include. The first argument to this script is where to put these DLLs
 340         // (the image we're creating), and the second argument is a junk directory
 341         // to ignore all other MinGW stuff the script creates.
 342         //
 343         // On 32-bit MinGW we're always including a DLL which needs some extra
 344         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 345         // anything requiring us to distribute a license, but it's likely the
 346         // install will *also* include the rust-mingw package, which also needs
 347         // licenses, so to be safe we just include it here in all MinGW packages.
 348         if host.ends_with("pc-windows-gnu") {
 349             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 350             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 351         }
 352
 353         return tarball.generate();
 354
 355         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 356             let host = compiler.host;
 357             let src = builder.sysroot(compiler);
 358
 359             // Copy rustc/rustdoc binaries
 360             t!(fs::create_dir_all(image.join("bin")));
 361             builder.cp_r(&src.join("bin"), &image.join("bin"));
 362
 363             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 364
 365             let ra_proc_macro_srv = builder
 366                 .ensure(tool::RustAnalyzerProcMacroSrv {
 367                     compiler: builder.compiler_for(
 368                         compiler.stage,
 369                         builder.config.build,
 370                         compiler.host,
 371                     ),
 372                     target: compiler.host,
 373                 })
 374                 .expect("rust-analyzer-proc-macro-server always builds");
 375             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 376
 377             let libdir_relative = builder.libdir_relative(compiler);
 378
 379             // Copy runtime DLLs needed by the compiler
 380             if libdir_relative.to_str() != Some("bin") {
 381                 let libdir = builder.rustc_libdir(compiler);
 382                 for entry in builder.read_dir(&libdir) {
 383                     let name = entry.file_name();
 384                     if let Some(s) = name.to_str() {
 385                         if is_dylib(s) {
 386                             // Don't use custom libdir here because ^lib/ will be resolved again
 387                             // with installer
 388                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 389                         }
 390                     }
 391                 }
 392             }
 393
 394             // Copy over the codegen backends
 395             let backends_src = builder.sysroot_codegen_backends(compiler);
 396             let backends_rel = backends_src
 397                 .strip_prefix(&src)
 398                 .unwrap()
 399                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 400                 .unwrap();
 401             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 402             let backends_dst = image.join("lib").join(&backends_rel);
 403
 404             t!(fs::create_dir_all(&backends_dst));
 405             builder.cp_r(&backends_src, &backends_dst);
 406
 407             // Copy libLLVM.so to the lib dir as well, if needed. While not
 408             // technically needed by rustc itself it's needed by lots of other
 409             // components like the llvm tools and LLD. LLD is included below and
 410             // tools/LLDB come later, so let's just throw it in the rustc
 411             // component for now.
 412             maybe_install_llvm_runtime(builder, host, image);
 413
 414             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 415             t!(fs::create_dir_all(&dst_dir));
 416
 417             // Copy over lld if it's there
 418             if builder.config.lld_enabled {
 419                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 420                 let rust_lld = exe("rust-lld", compiler.host);
 421                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 422                 // for `-Z gcc-ld=lld`
 423                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 424                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 425                 t!(fs::create_dir(&gcc_lld_dst_dir));
 426                 let exe_name = exe("ld", compiler.host);
 427                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 428             }
 429
 430             // Man pages
 431             t!(fs::create_dir_all(image.join("share/man/man1")));
 432             let man_src = builder.src.join("src/doc/man");
 433             let man_dst = image.join("share/man/man1");
 434
 435             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 436             // to hardlink, and we don't want to edit the source templates
 437             for file_entry in builder.read_dir(&man_src) {
 438                 let page_src = file_entry.path();
 439                 let page_dst = man_dst.join(file_entry.file_name());
 440                 let src_text = t!(std::fs::read_to_string(&page_src));
 441                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 442                 t!(std::fs::write(&page_dst, &new_text));
 443                 t!(fs::copy(&page_src, &page_dst));
 444             }
 445
 446             // Debugger scripts
 447             builder
 448                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 449
 450             // Misc license info
 451             let cp = |file: &str| {
 452                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 453             };
 454             cp("COPYRIGHT");
 455             cp("LICENSE-APACHE");
 456             cp("LICENSE-MIT");
 457             cp("README.md");
 458         }
 459     }
 460 }
 461
 462 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 463 pub struct DebuggerScripts {
 464     pub sysroot: Interned<PathBuf>,
 465     pub host: TargetSelection,
 466 }
 467
 468 impl Step for DebuggerScripts {
 469     type Output = ();
 470
 471     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 472         run.never()
 473     }
 474
 475     /// Copies debugger scripts for `target` into the `sysroot` specified.
 476     fn run(self, builder: &Builder<'_>) {
 477         let host = self.host;
 478         let sysroot = self.sysroot;
 479         let dst = sysroot.join("lib/rustlib/etc");
 480         t!(fs::create_dir_all(&dst));
 481         let cp_debugger_script = |file: &str| {
 482             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 483         };
 484         if host.contains("windows-msvc") {
 485             // windbg debugger scripts
 486             builder.install(
 487                 &builder.src.join("src/etc/rust-windbg.cmd"),
 488                 &sysroot.join("bin"),
 489                 0o755,
 490             );
 491
 492             cp_debugger_script("natvis/intrinsic.natvis");
 493             cp_debugger_script("natvis/liballoc.natvis");
 494             cp_debugger_script("natvis/libcore.natvis");
 495             cp_debugger_script("natvis/libstd.natvis");
 496         } else {
 497             cp_debugger_script("rust_types.py");
 498
 499             // gdb debugger scripts
 500             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 501             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 502
 503             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 504             cp_debugger_script("gdb_lookup.py");
 505             cp_debugger_script("gdb_providers.py");
 506
 507             // lldb debugger scripts
 508             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 509
 510             cp_debugger_script("lldb_lookup.py");
 511             cp_debugger_script("lldb_providers.py");
 512             cp_debugger_script("lldb_commands")
 513         }
 514     }
 515 }
 516
 517 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 518     // The only true set of target libraries came from the build triple, so
 519     // let's reduce redundant work by only producing archives from that host.
 520     if compiler.host != builder.config.build {
 521         builder.info("\tskipping, not a build host");
 522         true
 523     } else {
 524         false
 525     }
 526 }
 527
 528 /// Copy stamped files into an image's `target/lib` directory.
 529 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 530     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 531     let self_contained_dst = dst.join("self-contained");
 532     t!(fs::create_dir_all(&dst));
 533     t!(fs::create_dir_all(&self_contained_dst));
 534     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 535         if dependency_type == DependencyType::TargetSelfContained {
 536             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 537         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 538             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 539         }
 540     }
 541 }
 542
 543 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 544 pub struct Std {
 545     pub compiler: Compiler,
 546     pub target: TargetSelection,
 547 }
 548
 549 impl Step for Std {
 550     type Output = Option<GeneratedTarball>;
 551     const DEFAULT: bool = true;
 552
 553     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 554         run.alias("rust-std")
 555     }
 556
 557     fn make_run(run: RunConfig<'_>) {
 558         run.builder.ensure(Std {
 559             compiler: run.builder.compiler_for(
 560                 run.builder.top_stage,
 561                 run.builder.config.build,
 562                 run.target,
 563             ),
 564             target: run.target,
 565         });
 566     }
 567
 568     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 569         let compiler = self.compiler;
 570         let target = self.target;
 571
 572         if skip_host_target_lib(builder, compiler) {
 573             return None;
 574         }
 575
 576         builder.ensure(compile::Std::new(compiler, target));
 577
 578         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 579         tarball.include_target_in_component_name(true);
 580
 581         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 582         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 583         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 584
 585         Some(tarball.generate())
 586     }
 587 }
 588
 589 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 590 pub struct RustcDev {
 591     pub compiler: Compiler,
 592     pub target: TargetSelection,
 593 }
 594
 595 impl Step for RustcDev {
 596     type Output = Option<GeneratedTarball>;
 597     const DEFAULT: bool = true;
 598     const ONLY_HOSTS: bool = true;
 599
 600     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 601         run.alias("rustc-dev")
 602     }
 603
 604     fn make_run(run: RunConfig<'_>) {
 605         run.builder.ensure(RustcDev {
 606             compiler: run.builder.compiler_for(
 607                 run.builder.top_stage,
 608                 run.builder.config.build,
 609                 run.target,
 610             ),
 611             target: run.target,
 612         });
 613     }
 614
 615     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 616         let compiler = self.compiler;
 617         let target = self.target;
 618         if skip_host_target_lib(builder, compiler) {
 619             return None;
 620         }
 621
 622         builder.ensure(compile::Rustc::new(compiler, target));
 623
 624         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 625
 626         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 627         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 628         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 629
 630         let src_files = &["Cargo.lock"];
 631         // This is the reduced set of paths which will become the rustc-dev component
 632         // (essentially the compiler crates and all of their path dependencies).
 633         copy_src_dirs(
 634             builder,
 635             &builder.src,
 636             &["compiler"],
 637             &[],
 638             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 639         );
 640         for file in src_files {
 641             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 642         }
 643
 644         Some(tarball.generate())
 645     }
 646 }
 647
 648 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 649 pub struct Analysis {
 650     pub compiler: Compiler,
 651     pub target: TargetSelection,
 652 }
 653
 654 impl Step for Analysis {
 655     type Output = Option<GeneratedTarball>;
 656     const DEFAULT: bool = true;
 657
 658     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 659         let default = should_build_extended_tool(&run.builder, "analysis");
 660         run.alias("rust-analysis").default_condition(default)
 661     }
 662
 663     fn make_run(run: RunConfig<'_>) {
 664         run.builder.ensure(Analysis {
 665             // Find the actual compiler (handling the full bootstrap option) which
 666             // produced the save-analysis data because that data isn't copied
 667             // through the sysroot uplifting.
 668             compiler: run.builder.compiler_for(
 669                 run.builder.top_stage,
 670                 run.builder.config.build,
 671                 run.target,
 672             ),
 673             target: run.target,
 674         });
 675     }
 676
 677     /// Creates a tarball of save-analysis metadata, if available.
 678     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 679         let compiler = self.compiler;
 680         let target = self.target;
 681         if compiler.host != builder.config.build {
 682             return None;
 683         }
 684
 685         builder.ensure(compile::Std::new(compiler, target));
 686         let src = builder
 687             .stage_out(compiler, Mode::Std)
 688             .join(target.triple)
 689             .join(builder.cargo_dir())
 690             .join("deps")
 691             .join("save-analysis");
 692
 693         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 694         tarball.include_target_in_component_name(true);
 695         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 696         Some(tarball.generate())
 697     }
 698 }
 699
 700 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 701 /// `dst_dir`.
 702 fn copy_src_dirs(
 703     builder: &Builder<'_>,
 704     base: &Path,
 705     src_dirs: &[&str],
 706     exclude_dirs: &[&str],
 707     dst_dir: &Path,
 708 ) {
 709     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 710         let spath = match path.to_str() {
 711             Some(path) => path,
 712             None => return false,
 713         };
 714         if spath.ends_with('~') || spath.ends_with(".pyc") {
 715             return false;
 716         }
 717
 718         const LLVM_PROJECTS: &[&str] = &[
 719             "llvm-project/clang",
 720             "llvm-project\\clang",
 721             "llvm-project/libunwind",
 722             "llvm-project\\libunwind",
 723             "llvm-project/lld",
 724             "llvm-project\\lld",
 725             "llvm-project/lldb",
 726             "llvm-project\\lldb",
 727             "llvm-project/llvm",
 728             "llvm-project\\llvm",
 729             "llvm-project/compiler-rt",
 730             "llvm-project\\compiler-rt",
 731             "llvm-project/cmake",
 732             "llvm-project\\cmake",
 733         ];
 734         if spath.contains("llvm-project")
 735             && !spath.ends_with("llvm-project")
 736             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 737         {
 738             return false;
 739         }
 740
 741         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 742         if LLVM_TEST.iter().any(|path| spath.contains(path))
 743             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 744         {
 745             return false;
 746         }
 747
 748         let full_path = Path::new(dir).join(path);
 749         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 750             return false;
 751         }
 752
 753         let excludes = [
 754             "CVS",
 755             "RCS",
 756             "SCCS",
 757             ".git",
 758             ".gitignore",
 759             ".gitmodules",
 760             ".gitattributes",
 761             ".cvsignore",
 762             ".svn",
 763             ".arch-ids",
 764             "{arch}",
 765             "=RELEASE-ID",
 766             "=meta-update",
 767             "=update",
 768             ".bzr",
 769             ".bzrignore",
 770             ".bzrtags",
 771             ".hg",
 772             ".hgignore",
 773             ".hgrags",
 774             "_darcs",
 775         ];
 776         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 777     }
 778
 779     // Copy the directories using our filter
 780     for item in src_dirs {
 781         let dst = &dst_dir.join(item);
 782         t!(fs::create_dir_all(dst));
 783         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 784     }
 785 }
 786
 787 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 788 pub struct Src;
 789
 790 impl Step for Src {
 791     /// The output path of the src installer tarball
 792     type Output = GeneratedTarball;
 793     const DEFAULT: bool = true;
 794     const ONLY_HOSTS: bool = true;
 795
 796     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 797         run.alias("rust-src")
 798     }
 799
 800     fn make_run(run: RunConfig<'_>) {
 801         run.builder.ensure(Src);
 802     }
 803
 804     /// Creates the `rust-src` installer component
 805     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 806         let tarball = Tarball::new_targetless(builder, "rust-src");
 807
 808         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 809         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 810         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 811         // and fix them...
 812         //
 813         // NOTE: if you update the paths here, you also should update the "virtual" path
 814         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 815         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 816
 817         let src_files = ["Cargo.lock"];
 818         // This is the reduced set of paths which will become the rust-src component
 819         // (essentially libstd and all of its path dependencies).
 820         copy_src_dirs(
 821             builder,
 822             &builder.src,
 823             &["library", "src/llvm-project/libunwind"],
 824             &[
 825                 // not needed and contains symlinks which rustup currently
 826                 // chokes on when unpacking.
 827                 "library/backtrace/crates",
 828                 // these are 30MB combined and aren't necessary for building
 829                 // the standard library.
 830                 "library/stdarch/Cargo.toml",
 831                 "library/stdarch/crates/stdarch-verify",
 832                 "library/stdarch/crates/intrinsic-test",
 833             ],
 834             &dst_src,
 835         );
 836         for file in src_files.iter() {
 837             builder.copy(&builder.src.join(file), &dst_src.join(file));
 838         }
 839
 840         tarball.generate()
 841     }
 842 }
 843
 844 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 845 pub struct PlainSourceTarball;
 846
 847 impl Step for PlainSourceTarball {
 848     /// Produces the location of the tarball generated
 849     type Output = GeneratedTarball;
 850     const DEFAULT: bool = true;
 851     const ONLY_HOSTS: bool = true;
 852
 853     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 854         let builder = run.builder;
 855         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 856     }
 857
 858     fn make_run(run: RunConfig<'_>) {
 859         run.builder.ensure(PlainSourceTarball);
 860     }
 861
 862     /// Creates the plain source tarball
 863     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 864         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 865         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 866         // It also contains symbolic links, unlike other any other dist tarball.
 867         // It's used for distros building rustc from source in a pre-vendored environment.
 868         let mut tarball = Tarball::new(builder, "rustc", "src");
 869         tarball.permit_symlinks(true);
 870         let plain_dst_src = tarball.image_dir();
 871
 872         // This is the set of root paths which will become part of the source package
 873         let src_files = [
 874             "COPYRIGHT",
 875             "LICENSE-APACHE",
 876             "LICENSE-MIT",
 877             "CONTRIBUTING.md",
 878             "README.md",
 879             "RELEASES.md",
 880             "configure",
 881             "x.py",
 882             "config.toml.example",
 883             "Cargo.toml",
 884             "Cargo.lock",
 885         ];
 886         let src_dirs = ["src", "compiler", "library"];
 887
 888         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 889
 890         // Copy the files normally
 891         for item in &src_files {
 892             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 893         }
 894
 895         // Create the version file
 896         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 897         if let Some(sha) = builder.rust_sha() {
 898             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 899         }
 900
 901         // If we're building from git sources, we need to vendor a complete distribution.
 902         if builder.rust_info.is_git() {
 903             // Ensure we have the submodules checked out.
 904             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 905
 906             // Vendor all Cargo dependencies
 907             let mut cmd = Command::new(&builder.initial_cargo);
 908             cmd.arg("vendor")
 909                 .arg("--sync")
 910                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 911                 .arg("--sync")
 912                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 913                 .arg("--sync")
 914                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 915                 .current_dir(&plain_dst_src);
 916
 917             let config = if !builder.config.dry_run {
 918                 t!(String::from_utf8(t!(cmd.output()).stdout))
 919             } else {
 920                 String::new()
 921             };
 922
 923             let cargo_config_dir = plain_dst_src.join(".cargo");
 924             builder.create_dir(&cargo_config_dir);
 925             builder.create(&cargo_config_dir.join("config.toml"), &config);
 926         }
 927
 928         tarball.bare()
 929     }
 930 }
 931
 932 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 933 pub struct Cargo {
 934     pub compiler: Compiler,
 935     pub target: TargetSelection,
 936 }
 937
 938 impl Step for Cargo {
 939     type Output = Option<GeneratedTarball>;
 940     const DEFAULT: bool = true;
 941     const ONLY_HOSTS: bool = true;
 942
 943     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 944         let default = should_build_extended_tool(&run.builder, "cargo");
 945         run.alias("cargo").default_condition(default)
 946     }
 947
 948     fn make_run(run: RunConfig<'_>) {
 949         run.builder.ensure(Cargo {
 950             compiler: run.builder.compiler_for(
 951                 run.builder.top_stage,
 952                 run.builder.config.build,
 953                 run.target,
 954             ),
 955             target: run.target,
 956         });
 957     }
 958
 959     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 960         let compiler = self.compiler;
 961         let target = self.target;
 962
 963         let cargo = builder.ensure(tool::Cargo { compiler, target });
 964         let src = builder.src.join("src/tools/cargo");
 965         let etc = src.join("src/etc");
 966
 967         // Prepare the image directory
 968         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 969         tarball.set_overlay(OverlayKind::Cargo);
 970
 971         tarball.add_file(&cargo, "bin", 0o755);
 972         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 973         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 974         tarball.add_dir(etc.join("man"), "share/man/man1");
 975         tarball.add_legal_and_readme_to("share/doc/cargo");
 976
 977         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 978             let dirent = dirent.expect("read dir entry");
 979             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 980                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 981             }
 982         }
 983
 984         Some(tarball.generate())
 985     }
 986 }
 987
 988 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 989 pub struct Rls {
 990     pub compiler: Compiler,
 991     pub target: TargetSelection,
 992 }
 993
 994 impl Step for Rls {
 995     type Output = Option<GeneratedTarball>;
 996     const ONLY_HOSTS: bool = true;
 997     const DEFAULT: bool = true;
 998
 999     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1000         let default = should_build_extended_tool(&run.builder, "rls");
1001         run.alias("rls").default_condition(default)
1002     }
1003
1004     fn make_run(run: RunConfig<'_>) {
1005         run.builder.ensure(Rls {
1006             compiler: run.builder.compiler_for(
1007                 run.builder.top_stage,
1008                 run.builder.config.build,
1009                 run.target,
1010             ),
1011             target: run.target,
1012         });
1013     }
1014
1015     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1016         let compiler = self.compiler;
1017         let target = self.target;
1018
1019         let rls = builder
1020             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1021             .or_else(|| {
1022                 missing_tool("RLS", builder.build.config.missing_tools);
1023                 None
1024             })?;
1025
1026         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1027         tarball.set_overlay(OverlayKind::RLS);
1028         tarball.is_preview(true);
1029         tarball.add_file(rls, "bin", 0o755);
1030         tarball.add_legal_and_readme_to("share/doc/rls");
1031         Some(tarball.generate())
1032     }
1033 }
1034
1035 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1036 pub struct RustAnalyzer {
1037     pub compiler: Compiler,
1038     pub target: TargetSelection,
1039 }
1040
1041 impl Step for RustAnalyzer {
1042     type Output = Option<GeneratedTarball>;
1043     const DEFAULT: bool = true;
1044     const ONLY_HOSTS: bool = true;
1045
1046     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1047         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1048         run.alias("rust-analyzer").default_condition(default)
1049     }
1050
1051     fn make_run(run: RunConfig<'_>) {
1052         run.builder.ensure(RustAnalyzer {
1053             compiler: run.builder.compiler_for(
1054                 run.builder.top_stage,
1055                 run.builder.config.build,
1056                 run.target,
1057             ),
1058             target: run.target,
1059         });
1060     }
1061
1062     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1063         let compiler = self.compiler;
1064         let target = self.target;
1065
1066         if target.contains("riscv64") {
1067             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1068             // to build. See #74813 for details.
1069             return None;
1070         }
1071
1072         let rust_analyzer = builder
1073             .ensure(tool::RustAnalyzer { compiler, target })
1074             .expect("rust-analyzer always builds");
1075
1076         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1077         tarball.set_overlay(OverlayKind::RustAnalyzer);
1078         tarball.is_preview(true);
1079         tarball.add_file(rust_analyzer, "bin", 0o755);
1080         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1081         Some(tarball.generate())
1082     }
1083 }
1084
1085 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1086 pub struct Clippy {
1087     pub compiler: Compiler,
1088     pub target: TargetSelection,
1089 }
1090
1091 impl Step for Clippy {
1092     type Output = Option<GeneratedTarball>;
1093     const DEFAULT: bool = true;
1094     const ONLY_HOSTS: bool = true;
1095
1096     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1097         let default = should_build_extended_tool(&run.builder, "clippy");
1098         run.alias("clippy").default_condition(default)
1099     }
1100
1101     fn make_run(run: RunConfig<'_>) {
1102         run.builder.ensure(Clippy {
1103             compiler: run.builder.compiler_for(
1104                 run.builder.top_stage,
1105                 run.builder.config.build,
1106                 run.target,
1107             ),
1108             target: run.target,
1109         });
1110     }
1111
1112     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1113         let compiler = self.compiler;
1114         let target = self.target;
1115
1116         // Prepare the image directory
1117         // We expect clippy to build, because we've exited this step above if tool
1118         // state for clippy isn't testing.
1119         let clippy = builder
1120             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1121             .expect("clippy expected to build - essential tool");
1122         let cargoclippy = builder
1123             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1124             .expect("clippy expected to build - essential tool");
1125
1126         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1127         tarball.set_overlay(OverlayKind::Clippy);
1128         tarball.is_preview(true);
1129         tarball.add_file(clippy, "bin", 0o755);
1130         tarball.add_file(cargoclippy, "bin", 0o755);
1131         tarball.add_legal_and_readme_to("share/doc/clippy");
1132         Some(tarball.generate())
1133     }
1134 }
1135
1136 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1137 pub struct Miri {
1138     pub compiler: Compiler,
1139     pub target: TargetSelection,
1140 }
1141
1142 impl Step for Miri {
1143     type Output = Option<GeneratedTarball>;
1144     const DEFAULT: bool = true;
1145     const ONLY_HOSTS: bool = true;
1146
1147     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1148         let default = should_build_extended_tool(&run.builder, "miri");
1149         run.alias("miri").default_condition(default)
1150     }
1151
1152     fn make_run(run: RunConfig<'_>) {
1153         run.builder.ensure(Miri {
1154             compiler: run.builder.compiler_for(
1155                 run.builder.top_stage,
1156                 run.builder.config.build,
1157                 run.target,
1158             ),
1159             target: run.target,
1160         });
1161     }
1162
1163     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1164         // This prevents miri from being built for "dist" or "install"
1165         // on the stable/beta channels. It is a nightly-only tool and should
1166         // not be included.
1167         if !builder.build.unstable_features() {
1168             return None;
1169         }
1170         let compiler = self.compiler;
1171         let target = self.target;
1172
1173         let miri = builder
1174             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1175             .or_else(|| {
1176                 missing_tool("miri", builder.build.config.missing_tools);
1177                 None
1178             })?;
1179         let cargomiri = builder
1180             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1181             .or_else(|| {
1182                 missing_tool("cargo miri", builder.build.config.missing_tools);
1183                 None
1184             })?;
1185
1186         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1187         tarball.set_overlay(OverlayKind::Miri);
1188         tarball.is_preview(true);
1189         tarball.add_file(miri, "bin", 0o755);
1190         tarball.add_file(cargomiri, "bin", 0o755);
1191         tarball.add_legal_and_readme_to("share/doc/miri");
1192         Some(tarball.generate())
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Rustfmt {
1198     pub compiler: Compiler,
1199     pub target: TargetSelection,
1200 }
1201
1202 impl Step for Rustfmt {
1203     type Output = Option<GeneratedTarball>;
1204     const DEFAULT: bool = true;
1205     const ONLY_HOSTS: bool = true;
1206
1207     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1208         let default = should_build_extended_tool(&run.builder, "rustfmt");
1209         run.alias("rustfmt").default_condition(default)
1210     }
1211
1212     fn make_run(run: RunConfig<'_>) {
1213         run.builder.ensure(Rustfmt {
1214             compiler: run.builder.compiler_for(
1215                 run.builder.top_stage,
1216                 run.builder.config.build,
1217                 run.target,
1218             ),
1219             target: run.target,
1220         });
1221     }
1222
1223     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1224         let compiler = self.compiler;
1225         let target = self.target;
1226
1227         let rustfmt = builder
1228             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1229             .or_else(|| {
1230                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1231                 None
1232             })?;
1233         let cargofmt = builder
1234             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1235             .or_else(|| {
1236                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1237                 None
1238             })?;
1239
1240         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1241         tarball.set_overlay(OverlayKind::Rustfmt);
1242         tarball.is_preview(true);
1243         tarball.add_file(rustfmt, "bin", 0o755);
1244         tarball.add_file(cargofmt, "bin", 0o755);
1245         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1246         Some(tarball.generate())
1247     }
1248 }
1249
1250 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1251 pub struct RustDemangler {
1252     pub compiler: Compiler,
1253     pub target: TargetSelection,
1254 }
1255
1256 impl Step for RustDemangler {
1257     type Output = Option<GeneratedTarball>;
1258     const DEFAULT: bool = true;
1259     const ONLY_HOSTS: bool = true;
1260
1261     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1262         // While other tools use `should_build_extended_tool` to decide whether to be run by
1263         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1264         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1265         // we run the step by default when only `extended = true`, and decide whether to actually
1266         // run it or not later.
1267         let default = run.builder.config.extended;
1268         run.alias("rust-demangler").default_condition(default)
1269     }
1270
1271     fn make_run(run: RunConfig<'_>) {
1272         run.builder.ensure(RustDemangler {
1273             compiler: run.builder.compiler_for(
1274                 run.builder.top_stage,
1275                 run.builder.config.build,
1276                 run.target,
1277             ),
1278             target: run.target,
1279         });
1280     }
1281
1282     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1283         let compiler = self.compiler;
1284         let target = self.target;
1285
1286         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1287         let condition = should_build_extended_tool(builder, "rust-demangler")
1288             || builder.config.profiler_enabled(target);
1289         if builder.config.extended && !condition {
1290             return None;
1291         }
1292
1293         let rust_demangler = builder
1294             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1295             .expect("rust-demangler expected to build - in-tree tool");
1296
1297         // Prepare the image directory
1298         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1299         tarball.set_overlay(OverlayKind::RustDemangler);
1300         tarball.is_preview(true);
1301         tarball.add_file(&rust_demangler, "bin", 0o755);
1302         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1303         Some(tarball.generate())
1304     }
1305 }
1306
1307 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1308 pub struct Extended {
1309     stage: u32,
1310     host: TargetSelection,
1311     target: TargetSelection,
1312 }
1313
1314 impl Step for Extended {
1315     type Output = ();
1316     const DEFAULT: bool = true;
1317     const ONLY_HOSTS: bool = true;
1318
1319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1320         let builder = run.builder;
1321         run.alias("extended").default_condition(builder.config.extended)
1322     }
1323
1324     fn make_run(run: RunConfig<'_>) {
1325         run.builder.ensure(Extended {
1326             stage: run.builder.top_stage,
1327             host: run.builder.config.build,
1328             target: run.target,
1329         });
1330     }
1331
1332     /// Creates a combined installer for the specified target in the provided stage.
1333     fn run(self, builder: &Builder<'_>) {
1334         let target = self.target;
1335         let stage = self.stage;
1336         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1337
1338         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1339
1340         let mut tarballs = Vec::new();
1341         let mut built_tools = HashSet::new();
1342         macro_rules! add_component {
1343             ($name:expr => $step:expr) => {
1344                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1345                     tarballs.push(tarball);
1346                     built_tools.insert($name);
1347                 }
1348             };
1349         }
1350
1351         // When rust-std package split from rustc, we needed to ensure that during
1352         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1353         // the std files during uninstall. To do this ensure that rustc comes
1354         // before rust-std in the list below.
1355         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1356         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1357
1358         if target.ends_with("windows-gnu") {
1359             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1360         }
1361
1362         add_component!("rust-docs" => Docs { host: target });
1363         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1364         add_component!("cargo" => Cargo { compiler, target });
1365         add_component!("rustfmt" => Rustfmt { compiler, target });
1366         add_component!("rls" => Rls { compiler, target });
1367         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1368         add_component!("llvm-components" => LlvmTools { target });
1369         add_component!("clippy" => Clippy { compiler, target });
1370         add_component!("miri" => Miri { compiler, target });
1371         add_component!("analysis" => Analysis { compiler, target });
1372
1373         let etc = builder.src.join("src/etc/installer");
1374
1375         // Avoid producing tarballs during a dry run.
1376         if builder.config.dry_run {
1377             return;
1378         }
1379
1380         let tarball = Tarball::new(builder, "rust", &target.triple);
1381         let generated = tarball.combine(&tarballs);
1382
1383         let tmp = tmpdir(builder).join("combined-tarball");
1384         let work = generated.work_dir();
1385
1386         let mut license = String::new();
1387         license += &builder.read(&builder.src.join("COPYRIGHT"));
1388         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1389         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1390         license.push('\n');
1391         license.push('\n');
1392
1393         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1394         let mut rtf = rtf.to_string();
1395         rtf.push('\n');
1396         for line in license.lines() {
1397             rtf.push_str(line);
1398             rtf.push_str("\\line ");
1399         }
1400         rtf.push('}');
1401
1402         fn filter(contents: &str, marker: &str) -> String {
1403             let start = format!("tool-{}-start", marker);
1404             let end = format!("tool-{}-end", marker);
1405             let mut lines = Vec::new();
1406             let mut omitted = false;
1407             for line in contents.lines() {
1408                 if line.contains(&start) {
1409                     omitted = true;
1410                 } else if line.contains(&end) {
1411                     omitted = false;
1412                 } else if !omitted {
1413                     lines.push(line);
1414                 }
1415             }
1416
1417             lines.join("\n")
1418         }
1419
1420         let xform = |p: &Path| {
1421             let mut contents = t!(fs::read_to_string(p));
1422             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1423                 if !built_tools.contains(tool) {
1424                     contents = filter(&contents, tool);
1425                 }
1426             }
1427             let ret = tmp.join(p.file_name().unwrap());
1428             t!(fs::write(&ret, &contents));
1429             ret
1430         };
1431
1432         if target.contains("apple-darwin") {
1433             builder.info("building pkg installer");
1434             let pkg = tmp.join("pkg");
1435             let _ = fs::remove_dir_all(&pkg);
1436
1437             let pkgbuild = |component: &str| {
1438                 let mut cmd = Command::new("pkgbuild");
1439                 cmd.arg("--identifier")
1440                     .arg(format!("org.rust-lang.{}", component))
1441                     .arg("--scripts")
1442                     .arg(pkg.join(component))
1443                     .arg("--nopayload")
1444                     .arg(pkg.join(component).with_extension("pkg"));
1445                 builder.run(&mut cmd);
1446             };
1447
1448             let prepare = |name: &str| {
1449                 builder.create_dir(&pkg.join(name));
1450                 builder.cp_r(
1451                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1452                     &pkg.join(name),
1453                 );
1454                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1455                 pkgbuild(name);
1456             };
1457             prepare("rustc");
1458             prepare("cargo");
1459             prepare("rust-std");
1460             prepare("rust-analysis");
1461             prepare("clippy");
1462             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1463                 if built_tools.contains(tool) {
1464                     prepare(tool);
1465                 }
1466             }
1467             // create an 'uninstall' package
1468             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1469             pkgbuild("uninstall");
1470
1471             builder.create_dir(&pkg.join("res"));
1472             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1473             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1474             let mut cmd = Command::new("productbuild");
1475             cmd.arg("--distribution")
1476                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1477                 .arg("--resources")
1478                 .arg(pkg.join("res"))
1479                 .arg(distdir(builder).join(format!(
1480                     "{}-{}.pkg",
1481                     pkgname(builder, "rust"),
1482                     target.triple
1483                 )))
1484                 .arg("--package-path")
1485                 .arg(&pkg);
1486             let _time = timeit(builder);
1487             builder.run(&mut cmd);
1488         }
1489
1490         if target.contains("windows") {
1491             let exe = tmp.join("exe");
1492             let _ = fs::remove_dir_all(&exe);
1493
1494             let prepare = |name: &str| {
1495                 builder.create_dir(&exe.join(name));
1496                 let dir = if name == "rust-std" || name == "rust-analysis" {
1497                     format!("{}-{}", name, target.triple)
1498                 } else if name == "rls" {
1499                     "rls-preview".to_string()
1500                 } else if name == "rust-analyzer" {
1501                     "rust-analyzer-preview".to_string()
1502                 } else if name == "clippy" {
1503                     "clippy-preview".to_string()
1504                 } else if name == "rust-demangler" {
1505                     "rust-demangler-preview".to_string()
1506                 } else if name == "miri" {
1507                     "miri-preview".to_string()
1508                 } else {
1509                     name.to_string()
1510                 };
1511                 builder.cp_r(
1512                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1513                     &exe.join(name),
1514                 );
1515                 builder.remove(&exe.join(name).join("manifest.in"));
1516             };
1517             prepare("rustc");
1518             prepare("cargo");
1519             prepare("rust-analysis");
1520             prepare("rust-docs");
1521             prepare("rust-std");
1522             prepare("clippy");
1523             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1524                 if built_tools.contains(tool) {
1525                     prepare(tool);
1526                 }
1527             }
1528             if target.ends_with("windows-gnu") {
1529                 prepare("rust-mingw");
1530             }
1531
1532             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1533
1534             // Generate msi installer
1535             let wix_path = env::var_os("WIX")
1536                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1537             let wix = PathBuf::from(wix_path);
1538             let heat = wix.join("bin/heat.exe");
1539             let candle = wix.join("bin/candle.exe");
1540             let light = wix.join("bin/light.exe");
1541
1542             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1543             builder.run(
1544                 Command::new(&heat)
1545                     .current_dir(&exe)
1546                     .arg("dir")
1547                     .arg("rustc")
1548                     .args(&heat_flags)
1549                     .arg("-cg")
1550                     .arg("RustcGroup")
1551                     .arg("-dr")
1552                     .arg("Rustc")
1553                     .arg("-var")
1554                     .arg("var.RustcDir")
1555                     .arg("-out")
1556                     .arg(exe.join("RustcGroup.wxs")),
1557             );
1558             builder.run(
1559                 Command::new(&heat)
1560                     .current_dir(&exe)
1561                     .arg("dir")
1562                     .arg("rust-docs")
1563                     .args(&heat_flags)
1564                     .arg("-cg")
1565                     .arg("DocsGroup")
1566                     .arg("-dr")
1567                     .arg("Docs")
1568                     .arg("-var")
1569                     .arg("var.DocsDir")
1570                     .arg("-out")
1571                     .arg(exe.join("DocsGroup.wxs"))
1572                     .arg("-t")
1573                     .arg(etc.join("msi/squash-components.xsl")),
1574             );
1575             builder.run(
1576                 Command::new(&heat)
1577                     .current_dir(&exe)
1578                     .arg("dir")
1579                     .arg("cargo")
1580                     .args(&heat_flags)
1581                     .arg("-cg")
1582                     .arg("CargoGroup")
1583                     .arg("-dr")
1584                     .arg("Cargo")
1585                     .arg("-var")
1586                     .arg("var.CargoDir")
1587                     .arg("-out")
1588                     .arg(exe.join("CargoGroup.wxs"))
1589                     .arg("-t")
1590                     .arg(etc.join("msi/remove-duplicates.xsl")),
1591             );
1592             builder.run(
1593                 Command::new(&heat)
1594                     .current_dir(&exe)
1595                     .arg("dir")
1596                     .arg("rust-std")
1597                     .args(&heat_flags)
1598                     .arg("-cg")
1599                     .arg("StdGroup")
1600                     .arg("-dr")
1601                     .arg("Std")
1602                     .arg("-var")
1603                     .arg("var.StdDir")
1604                     .arg("-out")
1605                     .arg(exe.join("StdGroup.wxs")),
1606             );
1607             if built_tools.contains("rls") {
1608                 builder.run(
1609                     Command::new(&heat)
1610                         .current_dir(&exe)
1611                         .arg("dir")
1612                         .arg("rls")
1613                         .args(&heat_flags)
1614                         .arg("-cg")
1615                         .arg("RlsGroup")
1616                         .arg("-dr")
1617                         .arg("Rls")
1618                         .arg("-var")
1619                         .arg("var.RlsDir")
1620                         .arg("-out")
1621                         .arg(exe.join("RlsGroup.wxs"))
1622                         .arg("-t")
1623                         .arg(etc.join("msi/remove-duplicates.xsl")),
1624                 );
1625             }
1626             if built_tools.contains("rust-analyzer") {
1627                 builder.run(
1628                     Command::new(&heat)
1629                         .current_dir(&exe)
1630                         .arg("dir")
1631                         .arg("rust-analyzer")
1632                         .args(&heat_flags)
1633                         .arg("-cg")
1634                         .arg("RustAnalyzerGroup")
1635                         .arg("-dr")
1636                         .arg("RustAnalyzer")
1637                         .arg("-var")
1638                         .arg("var.RustAnalyzerDir")
1639                         .arg("-out")
1640                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1641                         .arg("-t")
1642                         .arg(etc.join("msi/remove-duplicates.xsl")),
1643                 );
1644             }
1645             builder.run(
1646                 Command::new(&heat)
1647                     .current_dir(&exe)
1648                     .arg("dir")
1649                     .arg("clippy")
1650                     .args(&heat_flags)
1651                     .arg("-cg")
1652                     .arg("ClippyGroup")
1653                     .arg("-dr")
1654                     .arg("Clippy")
1655                     .arg("-var")
1656                     .arg("var.ClippyDir")
1657                     .arg("-out")
1658                     .arg(exe.join("ClippyGroup.wxs"))
1659                     .arg("-t")
1660                     .arg(etc.join("msi/remove-duplicates.xsl")),
1661             );
1662             if built_tools.contains("rust-demangler") {
1663                 builder.run(
1664                     Command::new(&heat)
1665                         .current_dir(&exe)
1666                         .arg("dir")
1667                         .arg("rust-demangler")
1668                         .args(&heat_flags)
1669                         .arg("-cg")
1670                         .arg("RustDemanglerGroup")
1671                         .arg("-dr")
1672                         .arg("RustDemangler")
1673                         .arg("-var")
1674                         .arg("var.RustDemanglerDir")
1675                         .arg("-out")
1676                         .arg(exe.join("RustDemanglerGroup.wxs"))
1677                         .arg("-t")
1678                         .arg(etc.join("msi/remove-duplicates.xsl")),
1679                 );
1680             }
1681             if built_tools.contains("miri") {
1682                 builder.run(
1683                     Command::new(&heat)
1684                         .current_dir(&exe)
1685                         .arg("dir")
1686                         .arg("miri")
1687                         .args(&heat_flags)
1688                         .arg("-cg")
1689                         .arg("MiriGroup")
1690                         .arg("-dr")
1691                         .arg("Miri")
1692                         .arg("-var")
1693                         .arg("var.MiriDir")
1694                         .arg("-out")
1695                         .arg(exe.join("MiriGroup.wxs"))
1696                         .arg("-t")
1697                         .arg(etc.join("msi/remove-duplicates.xsl")),
1698                 );
1699             }
1700             builder.run(
1701                 Command::new(&heat)
1702                     .current_dir(&exe)
1703                     .arg("dir")
1704                     .arg("rust-analysis")
1705                     .args(&heat_flags)
1706                     .arg("-cg")
1707                     .arg("AnalysisGroup")
1708                     .arg("-dr")
1709                     .arg("Analysis")
1710                     .arg("-var")
1711                     .arg("var.AnalysisDir")
1712                     .arg("-out")
1713                     .arg(exe.join("AnalysisGroup.wxs"))
1714                     .arg("-t")
1715                     .arg(etc.join("msi/remove-duplicates.xsl")),
1716             );
1717             if target.ends_with("windows-gnu") {
1718                 builder.run(
1719                     Command::new(&heat)
1720                         .current_dir(&exe)
1721                         .arg("dir")
1722                         .arg("rust-mingw")
1723                         .args(&heat_flags)
1724                         .arg("-cg")
1725                         .arg("GccGroup")
1726                         .arg("-dr")
1727                         .arg("Gcc")
1728                         .arg("-var")
1729                         .arg("var.GccDir")
1730                         .arg("-out")
1731                         .arg(exe.join("GccGroup.wxs")),
1732                 );
1733             }
1734
1735             let candle = |input: &Path| {
1736                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1737                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1738                 let mut cmd = Command::new(&candle);
1739                 cmd.current_dir(&exe)
1740                     .arg("-nologo")
1741                     .arg("-dRustcDir=rustc")
1742                     .arg("-dDocsDir=rust-docs")
1743                     .arg("-dCargoDir=cargo")
1744                     .arg("-dStdDir=rust-std")
1745                     .arg("-dAnalysisDir=rust-analysis")
1746                     .arg("-dClippyDir=clippy")
1747                     .arg("-arch")
1748                     .arg(&arch)
1749                     .arg("-out")
1750                     .arg(&output)
1751                     .arg(&input);
1752                 add_env(builder, &mut cmd, target);
1753
1754                 if built_tools.contains("rust-demangler") {
1755                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1756                 }
1757                 if built_tools.contains("rls") {
1758                     cmd.arg("-dRlsDir=rls");
1759                 }
1760                 if built_tools.contains("rust-analyzer") {
1761                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1762                 }
1763                 if built_tools.contains("miri") {
1764                     cmd.arg("-dMiriDir=miri");
1765                 }
1766                 if target.ends_with("windows-gnu") {
1767                     cmd.arg("-dGccDir=rust-mingw");
1768                 }
1769                 builder.run(&mut cmd);
1770             };
1771             candle(&xform(&etc.join("msi/rust.wxs")));
1772             candle(&etc.join("msi/ui.wxs"));
1773             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1774             candle("RustcGroup.wxs".as_ref());
1775             candle("DocsGroup.wxs".as_ref());
1776             candle("CargoGroup.wxs".as_ref());
1777             candle("StdGroup.wxs".as_ref());
1778             candle("ClippyGroup.wxs".as_ref());
1779             if built_tools.contains("rust-demangler") {
1780                 candle("RustDemanglerGroup.wxs".as_ref());
1781             }
1782             if built_tools.contains("rls") {
1783                 candle("RlsGroup.wxs".as_ref());
1784             }
1785             if built_tools.contains("rust-analyzer") {
1786                 candle("RustAnalyzerGroup.wxs".as_ref());
1787             }
1788             if built_tools.contains("miri") {
1789                 candle("MiriGroup.wxs".as_ref());
1790             }
1791             candle("AnalysisGroup.wxs".as_ref());
1792
1793             if target.ends_with("windows-gnu") {
1794                 candle("GccGroup.wxs".as_ref());
1795             }
1796
1797             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1798             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1799             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1800
1801             builder.info(&format!("building `msi` installer with {:?}", light));
1802             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1803             let mut cmd = Command::new(&light);
1804             cmd.arg("-nologo")
1805                 .arg("-ext")
1806                 .arg("WixUIExtension")
1807                 .arg("-ext")
1808                 .arg("WixUtilExtension")
1809                 .arg("-out")
1810                 .arg(exe.join(&filename))
1811                 .arg("rust.wixobj")
1812                 .arg("ui.wixobj")
1813                 .arg("rustwelcomedlg.wixobj")
1814                 .arg("RustcGroup.wixobj")
1815                 .arg("DocsGroup.wixobj")
1816                 .arg("CargoGroup.wixobj")
1817                 .arg("StdGroup.wixobj")
1818                 .arg("AnalysisGroup.wixobj")
1819                 .arg("ClippyGroup.wixobj")
1820                 .current_dir(&exe);
1821
1822             if built_tools.contains("rls") {
1823                 cmd.arg("RlsGroup.wixobj");
1824             }
1825             if built_tools.contains("rust-analyzer") {
1826                 cmd.arg("RustAnalyzerGroup.wixobj");
1827             }
1828             if built_tools.contains("rust-demangler") {
1829                 cmd.arg("RustDemanglerGroup.wixobj");
1830             }
1831             if built_tools.contains("miri") {
1832                 cmd.arg("MiriGroup.wixobj");
1833             }
1834
1835             if target.ends_with("windows-gnu") {
1836                 cmd.arg("GccGroup.wixobj");
1837             }
1838             // ICE57 wrongly complains about the shortcuts
1839             cmd.arg("-sice:ICE57");
1840
1841             let _time = timeit(builder);
1842             builder.run(&mut cmd);
1843
1844             if !builder.config.dry_run {
1845                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1846             }
1847         }
1848     }
1849 }
1850
1851 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1852     let mut parts = builder.version.split('.');
1853     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1854         .env("CFG_RELEASE_NUM", &builder.version)
1855         .env("CFG_RELEASE", builder.rust_release())
1856         .env("CFG_VER_MAJOR", parts.next().unwrap())
1857         .env("CFG_VER_MINOR", parts.next().unwrap())
1858         .env("CFG_VER_PATCH", parts.next().unwrap())
1859         .env("CFG_VER_BUILD", "0") // just needed to build
1860         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1861         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1862         .env("CFG_BUILD", target.triple)
1863         .env("CFG_CHANNEL", &builder.config.channel);
1864
1865     if target.contains("windows-gnullvm") {
1866         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1867     } else if target.contains("windows-gnu") {
1868         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1869     } else {
1870         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1871     }
1872 }
1873
1874 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1875 ///
1876 /// Returns whether the files were actually copied.
1877 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1878     if let Some(config) = builder.config.target_config.get(&target) {
1879         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1880             // If the LLVM was externally provided, then we don't currently copy
1881             // artifacts into the sysroot. This is not necessarily the right
1882             // choice (in particular, it will require the LLVM dylib to be in
1883             // the linker's load path at runtime), but the common use case for
1884             // external LLVMs is distribution provided LLVMs, and in that case
1885             // they're usually in the standard search path (e.g., /usr/lib) and
1886             // copying them here is going to cause problems as we may end up
1887             // with the wrong files and isn't what distributions want.
1888             //
1889             // This behavior may be revisited in the future though.
1890             //
1891             // If the LLVM is coming from ourselves (just from CI) though, we
1892             // still want to install it, as it otherwise won't be available.
1893             return false;
1894         }
1895     }
1896
1897     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1898     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1899     // clear why this is the case, though. llvm-config will emit the versioned
1900     // paths and we don't want those in the sysroot (as we're expecting
1901     // unversioned paths).
1902     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1903         let src_libdir = builder.llvm_out(target).join("lib");
1904         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1905         if llvm_dylib_path.exists() {
1906             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1907         }
1908         !builder.config.dry_run
1909     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1910         let mut cmd = Command::new(llvm_config);
1911         cmd.arg("--libfiles");
1912         builder.verbose(&format!("running {:?}", cmd));
1913         let files = output(&mut cmd);
1914         let build_llvm_out = &builder.llvm_out(builder.config.build);
1915         let target_llvm_out = &builder.llvm_out(target);
1916         for file in files.trim_end().split(' ') {
1917             // If we're not using a custom LLVM, make sure we package for the target.
1918             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1919                 target_llvm_out.join(relative_path)
1920             } else {
1921                 PathBuf::from(file)
1922             };
1923             builder.install(&file, dst_libdir, 0o644);
1924         }
1925         !builder.config.dry_run
1926     } else {
1927         false
1928     }
1929 }
1930
1931 /// Maybe add libLLVM.so to the target lib-dir for linking.
1932 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1933     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1934     // We do not need to copy LLVM files into the sysroot if it is not
1935     // dynamically linked; it is already included into librustc_llvm
1936     // statically.
1937     if builder.llvm_link_shared() {
1938         maybe_install_llvm(builder, target, &dst_libdir);
1939     }
1940 }
1941
1942 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1943 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1944     let dst_libdir =
1945         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1946     // We do not need to copy LLVM files into the sysroot if it is not
1947     // dynamically linked; it is already included into librustc_llvm
1948     // statically.
1949     if builder.llvm_link_shared() {
1950         maybe_install_llvm(builder, target, &dst_libdir);
1951     }
1952 }
1953
1954 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1955 pub struct LlvmTools {
1956     pub target: TargetSelection,
1957 }
1958
1959 impl Step for LlvmTools {
1960     type Output = Option<GeneratedTarball>;
1961     const ONLY_HOSTS: bool = true;
1962     const DEFAULT: bool = true;
1963
1964     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1965         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1966         // FIXME: allow using the names of the tools themselves?
1967         run.alias("llvm-tools").default_condition(default)
1968     }
1969
1970     fn make_run(run: RunConfig<'_>) {
1971         run.builder.ensure(LlvmTools { target: run.target });
1972     }
1973
1974     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1975         let target = self.target;
1976
1977         /* run only if llvm-config isn't used */
1978         if let Some(config) = builder.config.target_config.get(&target) {
1979             if let Some(ref _s) = config.llvm_config {
1980                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1981                 return None;
1982             }
1983         }
1984
1985         builder.ensure(crate::native::Llvm { target });
1986
1987         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1988         tarball.set_overlay(OverlayKind::LLVM);
1989         tarball.is_preview(true);
1990
1991         // Prepare the image directory
1992         let src_bindir = builder.llvm_out(target).join("bin");
1993         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1994         for tool in LLVM_TOOLS {
1995             let exe = src_bindir.join(exe(tool, target));
1996             tarball.add_file(&exe, &dst_bindir, 0o755);
1997         }
1998
1999         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2000         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2001         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2002         // compiler libraries.
2003         maybe_install_llvm_target(builder, target, tarball.image_dir());
2004
2005         Some(tarball.generate())
2006     }
2007 }
2008
2009 // Tarball intended for internal consumption to ease rustc/std development.
2010 //
2011 // Should not be considered stable by end users.
2012 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2013 pub struct RustDev {
2014     pub target: TargetSelection,
2015 }
2016
2017 impl Step for RustDev {
2018     type Output = Option<GeneratedTarball>;
2019     const DEFAULT: bool = true;
2020     const ONLY_HOSTS: bool = true;
2021
2022     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2023         run.alias("rust-dev")
2024     }
2025
2026     fn make_run(run: RunConfig<'_>) {
2027         run.builder.ensure(RustDev { target: run.target });
2028     }
2029
2030     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2031         let target = self.target;
2032
2033         /* run only if llvm-config isn't used */
2034         if let Some(config) = builder.config.target_config.get(&target) {
2035             if let Some(ref _s) = config.llvm_config {
2036                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2037                 return None;
2038             }
2039         }
2040
2041         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2042         tarball.set_overlay(OverlayKind::LLVM);
2043
2044         builder.ensure(crate::native::Llvm { target });
2045
2046         let src_bindir = builder.llvm_out(target).join("bin");
2047         // If updating this list, you likely want to change
2048         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2049         // will not pick up the extra file until LLVM gets bumped.
2050         for bin in &[
2051             "llvm-config",
2052             "llvm-ar",
2053             "llvm-objdump",
2054             "llvm-profdata",
2055             "llvm-bcanalyzer",
2056             "llvm-cov",
2057             "llvm-dwp",
2058             "llvm-nm",
2059             "llvm-dwarfdump",
2060         ] {
2061             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2062         }
2063
2064         // We don't build LLD on some platforms, so only add it if it exists
2065         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2066         if lld_path.exists() {
2067             tarball.add_file(lld_path, "bin", 0o755);
2068         }
2069
2070         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2071
2072         // Copy the include directory as well; needed mostly to build
2073         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2074         // just broadly useful to be able to link against the bundled LLVM.
2075         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2076
2077         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2078         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2079         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2080         // compiler libraries.
2081         let dst_libdir = tarball.image_dir().join("lib");
2082         maybe_install_llvm(builder, target, &dst_libdir);
2083         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2084         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2085
2086         Some(tarball.generate())
2087     }
2088 }
2089
2090 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2091 /// release process to avoid cloning the monorepo and building stuff.
2092 ///
2093 /// Should not be considered stable by end users.
2094 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2095 pub struct BuildManifest {
2096     pub target: TargetSelection,
2097 }
2098
2099 impl Step for BuildManifest {
2100     type Output = GeneratedTarball;
2101     const DEFAULT: bool = false;
2102     const ONLY_HOSTS: bool = true;
2103
2104     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2105         run.alias("build-manifest")
2106     }
2107
2108     fn make_run(run: RunConfig<'_>) {
2109         run.builder.ensure(BuildManifest { target: run.target });
2110     }
2111
2112     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2113         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2114
2115         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2116         tarball.add_file(&build_manifest, "bin", 0o755);
2117         tarball.generate()
2118     }
2119 }
2120
2121 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2122 ///
2123 /// Currently this is the PGO profile data.
2124 ///
2125 /// Should not be considered stable by end users.
2126 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2127 pub struct ReproducibleArtifacts {
2128     pub target: TargetSelection,
2129 }
2130
2131 impl Step for ReproducibleArtifacts {
2132     type Output = Option<GeneratedTarball>;
2133     const DEFAULT: bool = true;
2134     const ONLY_HOSTS: bool = true;
2135
2136     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2137         run.alias("reproducible-artifacts")
2138     }
2139
2140     fn make_run(run: RunConfig<'_>) {
2141         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2142     }
2143
2144     fn run(self, builder: &Builder<'_>) -> Self::Output {
2145         let mut added_anything = false;
2146         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2147         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2148             tarball.add_file(path, ".", 0o644);
2149             added_anything = true;
2150         }
2151         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2152             tarball.add_file(path, ".", 0o644);
2153             added_anything = true;
2154         }
2155         if added_anything { Some(tarball.generate()) } else { None }
2156     }
2157 }