src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
  43         panic!(
  44             "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
  45             tool_name, help
  46         )
  47     }
  48 }
  49
  50 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  51     if !builder.config.extended {
  52         return false;
  53     }
  54     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  55 }
  56
  57 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  58 pub struct Docs {
  59     pub host: TargetSelection,
  60 }
  61
  62 impl Step for Docs {
  63     type Output = Option<GeneratedTarball>;
  64     const DEFAULT: bool = true;
  65
  66     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  67         let default = run.builder.config.docs;
  68         run.alias("rust-docs").default_condition(default)
  69     }
  70
  71     fn make_run(run: RunConfig<'_>) {
  72         run.builder.ensure(Docs { host: run.target });
  73     }
  74
  75     /// Builds the `rust-docs` installer component.
  76     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  77         let host = self.host;
  78         builder.default_doc(&[]);
  79
  80         let dest = "share/doc/rust/html";
  81
  82         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  83         tarball.set_product_name("Rust Documentation");
  84         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  85         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  86         Some(tarball.generate())
  87     }
  88 }
  89
  90 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  91 pub struct RustcDocs {
  92     pub host: TargetSelection,
  93 }
  94
  95 impl Step for RustcDocs {
  96     type Output = Option<GeneratedTarball>;
  97     const DEFAULT: bool = true;
  98
  99     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 100         let builder = run.builder;
 101         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 102     }
 103
 104     fn make_run(run: RunConfig<'_>) {
 105         run.builder.ensure(RustcDocs { host: run.target });
 106     }
 107
 108     /// Builds the `rustc-docs` installer component.
 109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 110         let host = self.host;
 111         builder.default_doc(&[]);
 112
 113         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 114         tarball.set_product_name("Rustc Documentation");
 115         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 116         Some(tarball.generate())
 117     }
 118 }
 119
 120 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 121     let mut found = Vec::with_capacity(files.len());
 122
 123     for file in files {
 124         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 125
 126         if let Some(file_path) = file_path {
 127             found.push(file_path);
 128         } else {
 129             panic!("Could not find '{}' in {:?}", file, path);
 130         }
 131     }
 132
 133     found
 134 }
 135
 136 fn make_win_dist(
 137     rust_root: &Path,
 138     plat_root: &Path,
 139     target: TargetSelection,
 140     builder: &Builder<'_>,
 141 ) {
 142     //Ask gcc where it keeps its stuff
 143     let mut cmd = Command::new(builder.cc(target));
 144     cmd.arg("-print-search-dirs");
 145     let gcc_out = output(&mut cmd);
 146
 147     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 148     let mut lib_path = Vec::new();
 149
 150     for line in gcc_out.lines() {
 151         let idx = line.find(':').unwrap();
 152         let key = &line[..idx];
 153         let trim_chars: &[_] = &[' ', '='];
 154         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 155
 156         if key == "programs" {
 157             bin_path.extend(value);
 158         } else if key == "libraries" {
 159             lib_path.extend(value);
 160         }
 161     }
 162
 163     let compiler = if target == "i686-pc-windows-gnu" {
 164         "i686-w64-mingw32-gcc.exe"
 165     } else if target == "x86_64-pc-windows-gnu" {
 166         "x86_64-w64-mingw32-gcc.exe"
 167     } else {
 168         "gcc.exe"
 169     };
 170     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 171     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 172     if target.starts_with("i686-") {
 173         rustc_dlls.push("libgcc_s_dw2-1.dll");
 174     } else {
 175         rustc_dlls.push("libgcc_s_seh-1.dll");
 176     }
 177
 178     let target_libs = [
 179         //MinGW libs
 180         "libgcc.a",
 181         "libgcc_eh.a",
 182         "libgcc_s.a",
 183         "libm.a",
 184         "libmingw32.a",
 185         "libmingwex.a",
 186         "libstdc++.a",
 187         "libiconv.a",
 188         "libmoldname.a",
 189         "libpthread.a",
 190         //Windows import libs
 191         "libadvapi32.a",
 192         "libbcrypt.a",
 193         "libcomctl32.a",
 194         "libcomdlg32.a",
 195         "libcredui.a",
 196         "libcrypt32.a",
 197         "libdbghelp.a",
 198         "libgdi32.a",
 199         "libimagehlp.a",
 200         "libiphlpapi.a",
 201         "libkernel32.a",
 202         "libmsimg32.a",
 203         "libmsvcrt.a",
 204         "libodbc32.a",
 205         "libole32.a",
 206         "liboleaut32.a",
 207         "libopengl32.a",
 208         "libpsapi.a",
 209         "librpcrt4.a",
 210         "libsecur32.a",
 211         "libsetupapi.a",
 212         "libshell32.a",
 213         "libsynchronization.a",
 214         "libuser32.a",
 215         "libuserenv.a",
 216         "libuuid.a",
 217         "libwinhttp.a",
 218         "libwinmm.a",
 219         "libwinspool.a",
 220         "libws2_32.a",
 221         "libwsock32.a",
 222     ];
 223
 224     //Find mingw artifacts we want to bundle
 225     let target_tools = find_files(&target_tools, &bin_path);
 226     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 227     let target_libs = find_files(&target_libs, &lib_path);
 228
 229     // Copy runtime dlls next to rustc.exe
 230     let dist_bin_dir = rust_root.join("bin/");
 231     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 232     for src in rustc_dlls {
 233         builder.copy_to_folder(&src, &dist_bin_dir);
 234     }
 235
 236     //Copy platform tools to platform-specific bin directory
 237     let target_bin_dir = plat_root
 238         .join("lib")
 239         .join("rustlib")
 240         .join(target.triple)
 241         .join("bin")
 242         .join("self-contained");
 243     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 244     for src in target_tools {
 245         builder.copy_to_folder(&src, &target_bin_dir);
 246     }
 247
 248     // Warn windows-gnu users that the bundled GCC cannot compile C files
 249     builder.create(
 250         &target_bin_dir.join("GCC-WARNING.txt"),
 251         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 252          used as a linker. In order to be able to compile projects containing C code use \
 253          the GCC provided by MinGW or Cygwin.",
 254     );
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root
 258         .join("lib")
 259         .join("rustlib")
 260         .join(target.triple)
 261         .join("lib")
 262         .join("self-contained");
 263     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 264     for src in target_libs {
 265         builder.copy_to_folder(&src, &target_lib_dir);
 266     }
 267 }
 268
 269 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 270 pub struct Mingw {
 271     pub host: TargetSelection,
 272 }
 273
 274 impl Step for Mingw {
 275     type Output = Option<GeneratedTarball>;
 276     const DEFAULT: bool = true;
 277
 278     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 279         run.alias("rust-mingw")
 280     }
 281
 282     fn make_run(run: RunConfig<'_>) {
 283         run.builder.ensure(Mingw { host: run.target });
 284     }
 285
 286     /// Builds the `rust-mingw` installer component.
 287     ///
 288     /// This contains all the bits and pieces to run the MinGW Windows targets
 289     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 290     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 291         let host = self.host;
 292         if !host.ends_with("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 297         tarball.set_product_name("Rust MinGW");
 298
 299         // The first argument is a "temporary directory" which is just
 300         // thrown away (this contains the runtime DLLs included in the rustc package
 301         // above) and the second argument is where to place all the MinGW components
 302         // (which is what we want).
 303         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 304
 305         Some(tarball.generate())
 306     }
 307 }
 308
 309 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 310 pub struct Rustc {
 311     pub compiler: Compiler,
 312 }
 313
 314 impl Step for Rustc {
 315     type Output = GeneratedTarball;
 316     const DEFAULT: bool = true;
 317     const ONLY_HOSTS: bool = true;
 318
 319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 320         run.alias("rustc")
 321     }
 322
 323     fn make_run(run: RunConfig<'_>) {
 324         run.builder
 325             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 326     }
 327
 328     /// Creates the `rustc` installer component.
 329     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 330         let compiler = self.compiler;
 331         let host = self.compiler.host;
 332
 333         let tarball = Tarball::new(builder, "rustc", &host.triple);
 334
 335         // Prepare the rustc "image", what will actually end up getting installed
 336         prepare_image(builder, compiler, tarball.image_dir());
 337
 338         // On MinGW we've got a few runtime DLL dependencies that we need to
 339         // include. The first argument to this script is where to put these DLLs
 340         // (the image we're creating), and the second argument is a junk directory
 341         // to ignore all other MinGW stuff the script creates.
 342         //
 343         // On 32-bit MinGW we're always including a DLL which needs some extra
 344         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 345         // anything requiring us to distribute a license, but it's likely the
 346         // install will *also* include the rust-mingw package, which also needs
 347         // licenses, so to be safe we just include it here in all MinGW packages.
 348         if host.ends_with("pc-windows-gnu") {
 349             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 350             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 351         }
 352
 353         return tarball.generate();
 354
 355         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 356             let host = compiler.host;
 357             let src = builder.sysroot(compiler);
 358
 359             // Copy rustc/rustdoc binaries
 360             t!(fs::create_dir_all(image.join("bin")));
 361             builder.cp_r(&src.join("bin"), &image.join("bin"));
 362
 363             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 364
 365             let ra_proc_macro_srv = builder
 366                 .ensure(tool::RustAnalyzerProcMacroSrv {
 367                     compiler: builder.compiler_for(
 368                         compiler.stage,
 369                         builder.config.build,
 370                         compiler.host,
 371                     ),
 372                     target: compiler.host,
 373                 })
 374                 .expect("rust-analyzer-proc-macro-server always builds");
 375             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 376
 377             let libdir_relative = builder.libdir_relative(compiler);
 378
 379             // Copy runtime DLLs needed by the compiler
 380             if libdir_relative.to_str() != Some("bin") {
 381                 let libdir = builder.rustc_libdir(compiler);
 382                 for entry in builder.read_dir(&libdir) {
 383                     let name = entry.file_name();
 384                     if let Some(s) = name.to_str() {
 385                         if is_dylib(s) {
 386                             // Don't use custom libdir here because ^lib/ will be resolved again
 387                             // with installer
 388                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 389                         }
 390                     }
 391                 }
 392             }
 393
 394             // Copy over the codegen backends
 395             let backends_src = builder.sysroot_codegen_backends(compiler);
 396             let backends_rel = backends_src
 397                 .strip_prefix(&src)
 398                 .unwrap()
 399                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 400                 .unwrap();
 401             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 402             let backends_dst = image.join("lib").join(&backends_rel);
 403
 404             t!(fs::create_dir_all(&backends_dst));
 405             builder.cp_r(&backends_src, &backends_dst);
 406
 407             // Copy libLLVM.so to the lib dir as well, if needed. While not
 408             // technically needed by rustc itself it's needed by lots of other
 409             // components like the llvm tools and LLD. LLD is included below and
 410             // tools/LLDB come later, so let's just throw it in the rustc
 411             // component for now.
 412             maybe_install_llvm_runtime(builder, host, image);
 413
 414             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 415             t!(fs::create_dir_all(&dst_dir));
 416
 417             // Copy over lld if it's there
 418             if builder.config.lld_enabled {
 419                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 420                 let rust_lld = exe("rust-lld", compiler.host);
 421                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 422                 // for `-Z gcc-ld=lld`
 423                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 424                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 425                 t!(fs::create_dir(&gcc_lld_dst_dir));
 426                 let exe_name = exe("ld", compiler.host);
 427                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 428             }
 429
 430             // Man pages
 431             t!(fs::create_dir_all(image.join("share/man/man1")));
 432             let man_src = builder.src.join("src/doc/man");
 433             let man_dst = image.join("share/man/man1");
 434
 435             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 436             // to hardlink, and we don't want to edit the source templates
 437             for file_entry in builder.read_dir(&man_src) {
 438                 let page_src = file_entry.path();
 439                 let page_dst = man_dst.join(file_entry.file_name());
 440                 let src_text = t!(std::fs::read_to_string(&page_src));
 441                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 442                 t!(std::fs::write(&page_dst, &new_text));
 443                 t!(fs::copy(&page_src, &page_dst));
 444             }
 445
 446             // Debugger scripts
 447             builder
 448                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 449
 450             // Misc license info
 451             let cp = |file: &str| {
 452                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 453             };
 454             cp("COPYRIGHT");
 455             cp("LICENSE-APACHE");
 456             cp("LICENSE-MIT");
 457             cp("README.md");
 458         }
 459     }
 460 }
 461
 462 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 463 pub struct DebuggerScripts {
 464     pub sysroot: Interned<PathBuf>,
 465     pub host: TargetSelection,
 466 }
 467
 468 impl Step for DebuggerScripts {
 469     type Output = ();
 470
 471     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 472         run.never()
 473     }
 474
 475     /// Copies debugger scripts for `target` into the `sysroot` specified.
 476     fn run(self, builder: &Builder<'_>) {
 477         let host = self.host;
 478         let sysroot = self.sysroot;
 479         let dst = sysroot.join("lib/rustlib/etc");
 480         t!(fs::create_dir_all(&dst));
 481         let cp_debugger_script = |file: &str| {
 482             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 483         };
 484         if host.contains("windows-msvc") {
 485             // windbg debugger scripts
 486             builder.install(
 487                 &builder.src.join("src/etc/rust-windbg.cmd"),
 488                 &sysroot.join("bin"),
 489                 0o755,
 490             );
 491
 492             cp_debugger_script("natvis/intrinsic.natvis");
 493             cp_debugger_script("natvis/liballoc.natvis");
 494             cp_debugger_script("natvis/libcore.natvis");
 495             cp_debugger_script("natvis/libstd.natvis");
 496         } else {
 497             cp_debugger_script("rust_types.py");
 498
 499             // gdb debugger scripts
 500             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 501             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 502
 503             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 504             cp_debugger_script("gdb_lookup.py");
 505             cp_debugger_script("gdb_providers.py");
 506
 507             // lldb debugger scripts
 508             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 509
 510             cp_debugger_script("lldb_lookup.py");
 511             cp_debugger_script("lldb_providers.py");
 512             cp_debugger_script("lldb_commands")
 513         }
 514     }
 515 }
 516
 517 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 518     // The only true set of target libraries came from the build triple, so
 519     // let's reduce redundant work by only producing archives from that host.
 520     if compiler.host != builder.config.build {
 521         builder.info("\tskipping, not a build host");
 522         true
 523     } else {
 524         false
 525     }
 526 }
 527
 528 /// Copy stamped files into an image's `target/lib` directory.
 529 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 530     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 531     let self_contained_dst = dst.join("self-contained");
 532     t!(fs::create_dir_all(&dst));
 533     t!(fs::create_dir_all(&self_contained_dst));
 534     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 535         if dependency_type == DependencyType::TargetSelfContained {
 536             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 537         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 538             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 539         }
 540     }
 541 }
 542
 543 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 544 pub struct Std {
 545     pub compiler: Compiler,
 546     pub target: TargetSelection,
 547 }
 548
 549 impl Step for Std {
 550     type Output = Option<GeneratedTarball>;
 551     const DEFAULT: bool = true;
 552
 553     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 554         run.alias("rust-std")
 555     }
 556
 557     fn make_run(run: RunConfig<'_>) {
 558         run.builder.ensure(Std {
 559             compiler: run.builder.compiler_for(
 560                 run.builder.top_stage,
 561                 run.builder.config.build,
 562                 run.target,
 563             ),
 564             target: run.target,
 565         });
 566     }
 567
 568     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 569         let compiler = self.compiler;
 570         let target = self.target;
 571
 572         if skip_host_target_lib(builder, compiler) {
 573             return None;
 574         }
 575
 576         builder.ensure(compile::Std::new(compiler, target));
 577
 578         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 579         tarball.include_target_in_component_name(true);
 580
 581         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 582         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 583         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 584
 585         Some(tarball.generate())
 586     }
 587 }
 588
 589 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 590 pub struct RustcDev {
 591     pub compiler: Compiler,
 592     pub target: TargetSelection,
 593 }
 594
 595 impl Step for RustcDev {
 596     type Output = Option<GeneratedTarball>;
 597     const DEFAULT: bool = true;
 598     const ONLY_HOSTS: bool = true;
 599
 600     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 601         run.alias("rustc-dev")
 602     }
 603
 604     fn make_run(run: RunConfig<'_>) {
 605         run.builder.ensure(RustcDev {
 606             compiler: run.builder.compiler_for(
 607                 run.builder.top_stage,
 608                 run.builder.config.build,
 609                 run.target,
 610             ),
 611             target: run.target,
 612         });
 613     }
 614
 615     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 616         let compiler = self.compiler;
 617         let target = self.target;
 618         if skip_host_target_lib(builder, compiler) {
 619             return None;
 620         }
 621
 622         builder.ensure(compile::Rustc::new(compiler, target));
 623
 624         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 625
 626         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 627         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 628         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 629
 630         let src_files = &["Cargo.lock"];
 631         // This is the reduced set of paths which will become the rustc-dev component
 632         // (essentially the compiler crates and all of their path dependencies).
 633         copy_src_dirs(
 634             builder,
 635             &builder.src,
 636             &["compiler"],
 637             &[],
 638             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 639         );
 640         for file in src_files {
 641             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 642         }
 643
 644         Some(tarball.generate())
 645     }
 646 }
 647
 648 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 649 pub struct Analysis {
 650     pub compiler: Compiler,
 651     pub target: TargetSelection,
 652 }
 653
 654 impl Step for Analysis {
 655     type Output = Option<GeneratedTarball>;
 656     const DEFAULT: bool = true;
 657
 658     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 659         let default = should_build_extended_tool(&run.builder, "analysis");
 660         run.alias("rust-analysis").default_condition(default)
 661     }
 662
 663     fn make_run(run: RunConfig<'_>) {
 664         run.builder.ensure(Analysis {
 665             // Find the actual compiler (handling the full bootstrap option) which
 666             // produced the save-analysis data because that data isn't copied
 667             // through the sysroot uplifting.
 668             compiler: run.builder.compiler_for(
 669                 run.builder.top_stage,
 670                 run.builder.config.build,
 671                 run.target,
 672             ),
 673             target: run.target,
 674         });
 675     }
 676
 677     /// Creates a tarball of save-analysis metadata, if available.
 678     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 679         let compiler = self.compiler;
 680         let target = self.target;
 681         if compiler.host != builder.config.build {
 682             return None;
 683         }
 684
 685         builder.ensure(compile::Std::new(compiler, target));
 686         let src = builder
 687             .stage_out(compiler, Mode::Std)
 688             .join(target.triple)
 689             .join(builder.cargo_dir())
 690             .join("deps")
 691             .join("save-analysis");
 692
 693         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 694         tarball.include_target_in_component_name(true);
 695         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 696         Some(tarball.generate())
 697     }
 698 }
 699
 700 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 701 /// `dst_dir`.
 702 fn copy_src_dirs(
 703     builder: &Builder<'_>,
 704     base: &Path,
 705     src_dirs: &[&str],
 706     exclude_dirs: &[&str],
 707     dst_dir: &Path,
 708 ) {
 709     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 710         let spath = match path.to_str() {
 711             Some(path) => path,
 712             None => return false,
 713         };
 714         if spath.ends_with('~') || spath.ends_with(".pyc") {
 715             return false;
 716         }
 717
 718         const LLVM_PROJECTS: &[&str] = &[
 719             "llvm-project/clang",
 720             "llvm-project\\clang",
 721             "llvm-project/libunwind",
 722             "llvm-project\\libunwind",
 723             "llvm-project/lld",
 724             "llvm-project\\lld",
 725             "llvm-project/lldb",
 726             "llvm-project\\lldb",
 727             "llvm-project/llvm",
 728             "llvm-project\\llvm",
 729             "llvm-project/compiler-rt",
 730             "llvm-project\\compiler-rt",
 731             "llvm-project/cmake",
 732             "llvm-project\\cmake",
 733         ];
 734         if spath.contains("llvm-project")
 735             && !spath.ends_with("llvm-project")
 736             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 737         {
 738             return false;
 739         }
 740
 741         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 742         if LLVM_TEST.iter().any(|path| spath.contains(path))
 743             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 744         {
 745             return false;
 746         }
 747
 748         let full_path = Path::new(dir).join(path);
 749         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 750             return false;
 751         }
 752
 753         let excludes = [
 754             "CVS",
 755             "RCS",
 756             "SCCS",
 757             ".git",
 758             ".gitignore",
 759             ".gitmodules",
 760             ".gitattributes",
 761             ".cvsignore",
 762             ".svn",
 763             ".arch-ids",
 764             "{arch}",
 765             "=RELEASE-ID",
 766             "=meta-update",
 767             "=update",
 768             ".bzr",
 769             ".bzrignore",
 770             ".bzrtags",
 771             ".hg",
 772             ".hgignore",
 773             ".hgrags",
 774             "_darcs",
 775         ];
 776         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 777     }
 778
 779     // Copy the directories using our filter
 780     for item in src_dirs {
 781         let dst = &dst_dir.join(item);
 782         t!(fs::create_dir_all(dst));
 783         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 784     }
 785 }
 786
 787 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 788 pub struct Src;
 789
 790 impl Step for Src {
 791     /// The output path of the src installer tarball
 792     type Output = GeneratedTarball;
 793     const DEFAULT: bool = true;
 794     const ONLY_HOSTS: bool = true;
 795
 796     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 797         run.alias("rust-src")
 798     }
 799
 800     fn make_run(run: RunConfig<'_>) {
 801         run.builder.ensure(Src);
 802     }
 803
 804     /// Creates the `rust-src` installer component
 805     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 806         let tarball = Tarball::new_targetless(builder, "rust-src");
 807
 808         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 809         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 810         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 811         // and fix them...
 812         //
 813         // NOTE: if you update the paths here, you also should update the "virtual" path
 814         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 815         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 816
 817         let src_files = ["Cargo.lock"];
 818         // This is the reduced set of paths which will become the rust-src component
 819         // (essentially libstd and all of its path dependencies).
 820         copy_src_dirs(
 821             builder,
 822             &builder.src,
 823             &["library", "src/llvm-project/libunwind"],
 824             &[
 825                 // not needed and contains symlinks which rustup currently
 826                 // chokes on when unpacking.
 827                 "library/backtrace/crates",
 828                 // these are 30MB combined and aren't necessary for building
 829                 // the standard library.
 830                 "library/stdarch/Cargo.toml",
 831                 "library/stdarch/crates/stdarch-verify",
 832                 "library/stdarch/crates/intrinsic-test",
 833             ],
 834             &dst_src,
 835         );
 836         for file in src_files.iter() {
 837             builder.copy(&builder.src.join(file), &dst_src.join(file));
 838         }
 839
 840         tarball.generate()
 841     }
 842 }
 843
 844 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 845 pub struct PlainSourceTarball;
 846
 847 impl Step for PlainSourceTarball {
 848     /// Produces the location of the tarball generated
 849     type Output = GeneratedTarball;
 850     const DEFAULT: bool = true;
 851     const ONLY_HOSTS: bool = true;
 852
 853     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 854         let builder = run.builder;
 855         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 856     }
 857
 858     fn make_run(run: RunConfig<'_>) {
 859         run.builder.ensure(PlainSourceTarball);
 860     }
 861
 862     /// Creates the plain source tarball
 863     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 864         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 865         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 866         // It also contains symbolic links, unlike other any other dist tarball.
 867         // It's used for distros building rustc from source in a pre-vendored environment.
 868         let mut tarball = Tarball::new(builder, "rustc", "src");
 869         tarball.permit_symlinks(true);
 870         let plain_dst_src = tarball.image_dir();
 871
 872         // This is the set of root paths which will become part of the source package
 873         let src_files = [
 874             "COPYRIGHT",
 875             "LICENSE-APACHE",
 876             "LICENSE-MIT",
 877             "CONTRIBUTING.md",
 878             "README.md",
 879             "RELEASES.md",
 880             "configure",
 881             "x.py",
 882             "config.toml.example",
 883             "Cargo.toml",
 884             "Cargo.lock",
 885         ];
 886         let src_dirs = ["src", "compiler", "library"];
 887
 888         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 889
 890         // Copy the files normally
 891         for item in &src_files {
 892             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 893         }
 894
 895         // Create the version file
 896         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 897         if let Some(sha) = builder.rust_sha() {
 898             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 899         }
 900
 901         // If we're building from git sources, we need to vendor a complete distribution.
 902         if builder.rust_info.is_git() {
 903             // Ensure we have the submodules checked out.
 904             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 905
 906             // Vendor all Cargo dependencies
 907             let mut cmd = Command::new(&builder.initial_cargo);
 908             cmd.arg("vendor")
 909                 .arg("--sync")
 910                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 911                 .arg("--sync")
 912                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 913                 .arg("--sync")
 914                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 915                 .current_dir(&plain_dst_src);
 916
 917             let config = if !builder.config.dry_run {
 918                 t!(String::from_utf8(t!(cmd.output()).stdout))
 919             } else {
 920                 String::new()
 921             };
 922
 923             let cargo_config_dir = plain_dst_src.join(".cargo");
 924             builder.create_dir(&cargo_config_dir);
 925             builder.create(&cargo_config_dir.join("config.toml"), &config);
 926         }
 927
 928         tarball.bare()
 929     }
 930 }
 931
 932 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 933 pub struct Cargo {
 934     pub compiler: Compiler,
 935     pub target: TargetSelection,
 936 }
 937
 938 impl Step for Cargo {
 939     type Output = Option<GeneratedTarball>;
 940     const DEFAULT: bool = true;
 941     const ONLY_HOSTS: bool = true;
 942
 943     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 944         let default = should_build_extended_tool(&run.builder, "cargo");
 945         run.alias("cargo").default_condition(default)
 946     }
 947
 948     fn make_run(run: RunConfig<'_>) {
 949         run.builder.ensure(Cargo {
 950             compiler: run.builder.compiler_for(
 951                 run.builder.top_stage,
 952                 run.builder.config.build,
 953                 run.target,
 954             ),
 955             target: run.target,
 956         });
 957     }
 958
 959     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 960         let compiler = self.compiler;
 961         let target = self.target;
 962
 963         let cargo = builder.ensure(tool::Cargo { compiler, target });
 964         let src = builder.src.join("src/tools/cargo");
 965         let etc = src.join("src/etc");
 966
 967         // Prepare the image directory
 968         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 969         tarball.set_overlay(OverlayKind::Cargo);
 970
 971         tarball.add_file(&cargo, "bin", 0o755);
 972         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 973         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 974         tarball.add_dir(etc.join("man"), "share/man/man1");
 975         tarball.add_legal_and_readme_to("share/doc/cargo");
 976
 977         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 978             let dirent = dirent.expect("read dir entry");
 979             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 980                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 981             }
 982         }
 983
 984         Some(tarball.generate())
 985     }
 986 }
 987
 988 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 989 pub struct Rls {
 990     pub compiler: Compiler,
 991     pub target: TargetSelection,
 992 }
 993
 994 impl Step for Rls {
 995     type Output = Option<GeneratedTarball>;
 996     const ONLY_HOSTS: bool = true;
 997     const DEFAULT: bool = true;
 998
 999     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1000         let default = should_build_extended_tool(&run.builder, "rls");
1001         run.alias("rls").default_condition(default)
1002     }
1003
1004     fn make_run(run: RunConfig<'_>) {
1005         run.builder.ensure(Rls {
1006             compiler: run.builder.compiler_for(
1007                 run.builder.top_stage,
1008                 run.builder.config.build,
1009                 run.target,
1010             ),
1011             target: run.target,
1012         });
1013     }
1014
1015     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1016         let compiler = self.compiler;
1017         let target = self.target;
1018
1019         let rls = builder
1020             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1021             .expect("rls expected to build");
1022
1023         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1024         tarball.set_overlay(OverlayKind::RLS);
1025         tarball.is_preview(true);
1026         tarball.add_file(rls, "bin", 0o755);
1027         tarball.add_legal_and_readme_to("share/doc/rls");
1028         Some(tarball.generate())
1029     }
1030 }
1031
1032 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1033 pub struct RustAnalyzer {
1034     pub compiler: Compiler,
1035     pub target: TargetSelection,
1036 }
1037
1038 impl Step for RustAnalyzer {
1039     type Output = Option<GeneratedTarball>;
1040     const DEFAULT: bool = true;
1041     const ONLY_HOSTS: bool = true;
1042
1043     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1044         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1045         run.alias("rust-analyzer").default_condition(default)
1046     }
1047
1048     fn make_run(run: RunConfig<'_>) {
1049         run.builder.ensure(RustAnalyzer {
1050             compiler: run.builder.compiler_for(
1051                 run.builder.top_stage,
1052                 run.builder.config.build,
1053                 run.target,
1054             ),
1055             target: run.target,
1056         });
1057     }
1058
1059     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1060         let compiler = self.compiler;
1061         let target = self.target;
1062
1063         if target.contains("riscv64") {
1064             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1065             // to build. See #74813 for details.
1066             return None;
1067         }
1068
1069         let rust_analyzer = builder
1070             .ensure(tool::RustAnalyzer { compiler, target })
1071             .expect("rust-analyzer always builds");
1072
1073         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1074         tarball.set_overlay(OverlayKind::RustAnalyzer);
1075         tarball.is_preview(true);
1076         tarball.add_file(rust_analyzer, "bin", 0o755);
1077         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1078         Some(tarball.generate())
1079     }
1080 }
1081
1082 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1083 pub struct Clippy {
1084     pub compiler: Compiler,
1085     pub target: TargetSelection,
1086 }
1087
1088 impl Step for Clippy {
1089     type Output = Option<GeneratedTarball>;
1090     const DEFAULT: bool = true;
1091     const ONLY_HOSTS: bool = true;
1092
1093     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1094         let default = should_build_extended_tool(&run.builder, "clippy");
1095         run.alias("clippy").default_condition(default)
1096     }
1097
1098     fn make_run(run: RunConfig<'_>) {
1099         run.builder.ensure(Clippy {
1100             compiler: run.builder.compiler_for(
1101                 run.builder.top_stage,
1102                 run.builder.config.build,
1103                 run.target,
1104             ),
1105             target: run.target,
1106         });
1107     }
1108
1109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1110         let compiler = self.compiler;
1111         let target = self.target;
1112
1113         // Prepare the image directory
1114         // We expect clippy to build, because we've exited this step above if tool
1115         // state for clippy isn't testing.
1116         let clippy = builder
1117             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1118             .expect("clippy expected to build - essential tool");
1119         let cargoclippy = builder
1120             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1121             .expect("clippy expected to build - essential tool");
1122
1123         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1124         tarball.set_overlay(OverlayKind::Clippy);
1125         tarball.is_preview(true);
1126         tarball.add_file(clippy, "bin", 0o755);
1127         tarball.add_file(cargoclippy, "bin", 0o755);
1128         tarball.add_legal_and_readme_to("share/doc/clippy");
1129         Some(tarball.generate())
1130     }
1131 }
1132
1133 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1134 pub struct Miri {
1135     pub compiler: Compiler,
1136     pub target: TargetSelection,
1137 }
1138
1139 impl Step for Miri {
1140     type Output = Option<GeneratedTarball>;
1141     const DEFAULT: bool = true;
1142     const ONLY_HOSTS: bool = true;
1143
1144     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1145         let default = should_build_extended_tool(&run.builder, "miri");
1146         run.alias("miri").default_condition(default)
1147     }
1148
1149     fn make_run(run: RunConfig<'_>) {
1150         run.builder.ensure(Miri {
1151             compiler: run.builder.compiler_for(
1152                 run.builder.top_stage,
1153                 run.builder.config.build,
1154                 run.target,
1155             ),
1156             target: run.target,
1157         });
1158     }
1159
1160     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1161         // This prevents miri from being built for "dist" or "install"
1162         // on the stable/beta channels. It is a nightly-only tool and should
1163         // not be included.
1164         if !builder.build.unstable_features() {
1165             return None;
1166         }
1167         let compiler = self.compiler;
1168         let target = self.target;
1169
1170         let miri = builder
1171             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1172             .or_else(|| {
1173                 missing_tool("miri", builder.build.config.missing_tools);
1174                 None
1175             })?;
1176         let cargomiri = builder
1177             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1178             .or_else(|| {
1179                 missing_tool("cargo miri", builder.build.config.missing_tools);
1180                 None
1181             })?;
1182
1183         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1184         tarball.set_overlay(OverlayKind::Miri);
1185         tarball.is_preview(true);
1186         tarball.add_file(miri, "bin", 0o755);
1187         tarball.add_file(cargomiri, "bin", 0o755);
1188         tarball.add_legal_and_readme_to("share/doc/miri");
1189         Some(tarball.generate())
1190     }
1191 }
1192
1193 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1194 pub struct Rustfmt {
1195     pub compiler: Compiler,
1196     pub target: TargetSelection,
1197 }
1198
1199 impl Step for Rustfmt {
1200     type Output = Option<GeneratedTarball>;
1201     const DEFAULT: bool = true;
1202     const ONLY_HOSTS: bool = true;
1203
1204     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1205         let default = should_build_extended_tool(&run.builder, "rustfmt");
1206         run.alias("rustfmt").default_condition(default)
1207     }
1208
1209     fn make_run(run: RunConfig<'_>) {
1210         run.builder.ensure(Rustfmt {
1211             compiler: run.builder.compiler_for(
1212                 run.builder.top_stage,
1213                 run.builder.config.build,
1214                 run.target,
1215             ),
1216             target: run.target,
1217         });
1218     }
1219
1220     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1221         let compiler = self.compiler;
1222         let target = self.target;
1223
1224         let rustfmt = builder
1225             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1226             .expect("rustfmt expected to build - essential tool");
1227         let cargofmt = builder
1228             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1229             .expect("cargo fmt expected to build - essential tool");
1230         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1231         tarball.set_overlay(OverlayKind::Rustfmt);
1232         tarball.is_preview(true);
1233         tarball.add_file(rustfmt, "bin", 0o755);
1234         tarball.add_file(cargofmt, "bin", 0o755);
1235         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1236         Some(tarball.generate())
1237     }
1238 }
1239
1240 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1241 pub struct RustDemangler {
1242     pub compiler: Compiler,
1243     pub target: TargetSelection,
1244 }
1245
1246 impl Step for RustDemangler {
1247     type Output = Option<GeneratedTarball>;
1248     const DEFAULT: bool = true;
1249     const ONLY_HOSTS: bool = true;
1250
1251     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1252         // While other tools use `should_build_extended_tool` to decide whether to be run by
1253         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1254         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1255         // we run the step by default when only `extended = true`, and decide whether to actually
1256         // run it or not later.
1257         let default = run.builder.config.extended;
1258         run.alias("rust-demangler").default_condition(default)
1259     }
1260
1261     fn make_run(run: RunConfig<'_>) {
1262         run.builder.ensure(RustDemangler {
1263             compiler: run.builder.compiler_for(
1264                 run.builder.top_stage,
1265                 run.builder.config.build,
1266                 run.target,
1267             ),
1268             target: run.target,
1269         });
1270     }
1271
1272     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1273         let compiler = self.compiler;
1274         let target = self.target;
1275
1276         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1277         let condition = should_build_extended_tool(builder, "rust-demangler")
1278             || builder.config.profiler_enabled(target);
1279         if builder.config.extended && !condition {
1280             return None;
1281         }
1282
1283         let rust_demangler = builder
1284             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1285             .expect("rust-demangler expected to build - in-tree tool");
1286
1287         // Prepare the image directory
1288         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1289         tarball.set_overlay(OverlayKind::RustDemangler);
1290         tarball.is_preview(true);
1291         tarball.add_file(&rust_demangler, "bin", 0o755);
1292         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1293         Some(tarball.generate())
1294     }
1295 }
1296
1297 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1298 pub struct Extended {
1299     stage: u32,
1300     host: TargetSelection,
1301     target: TargetSelection,
1302 }
1303
1304 impl Step for Extended {
1305     type Output = ();
1306     const DEFAULT: bool = true;
1307     const ONLY_HOSTS: bool = true;
1308
1309     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1310         let builder = run.builder;
1311         run.alias("extended").default_condition(builder.config.extended)
1312     }
1313
1314     fn make_run(run: RunConfig<'_>) {
1315         run.builder.ensure(Extended {
1316             stage: run.builder.top_stage,
1317             host: run.builder.config.build,
1318             target: run.target,
1319         });
1320     }
1321
1322     /// Creates a combined installer for the specified target in the provided stage.
1323     fn run(self, builder: &Builder<'_>) {
1324         let target = self.target;
1325         let stage = self.stage;
1326         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1327
1328         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1329
1330         let mut tarballs = Vec::new();
1331         let mut built_tools = HashSet::new();
1332         macro_rules! add_component {
1333             ($name:expr => $step:expr) => {
1334                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1335                     tarballs.push(tarball);
1336                     built_tools.insert($name);
1337                 }
1338             };
1339         }
1340
1341         // When rust-std package split from rustc, we needed to ensure that during
1342         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1343         // the std files during uninstall. To do this ensure that rustc comes
1344         // before rust-std in the list below.
1345         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1346         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1347
1348         if target.ends_with("windows-gnu") {
1349             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1350         }
1351
1352         add_component!("rust-docs" => Docs { host: target });
1353         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1354         add_component!("cargo" => Cargo { compiler, target });
1355         add_component!("rustfmt" => Rustfmt { compiler, target });
1356         add_component!("rls" => Rls { compiler, target });
1357         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1358         add_component!("llvm-components" => LlvmTools { target });
1359         add_component!("clippy" => Clippy { compiler, target });
1360         add_component!("miri" => Miri { compiler, target });
1361         add_component!("analysis" => Analysis { compiler, target });
1362
1363         let etc = builder.src.join("src/etc/installer");
1364
1365         // Avoid producing tarballs during a dry run.
1366         if builder.config.dry_run {
1367             return;
1368         }
1369
1370         let tarball = Tarball::new(builder, "rust", &target.triple);
1371         let generated = tarball.combine(&tarballs);
1372
1373         let tmp = tmpdir(builder).join("combined-tarball");
1374         let work = generated.work_dir();
1375
1376         let mut license = String::new();
1377         license += &builder.read(&builder.src.join("COPYRIGHT"));
1378         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1379         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1380         license.push('\n');
1381         license.push('\n');
1382
1383         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1384         let mut rtf = rtf.to_string();
1385         rtf.push('\n');
1386         for line in license.lines() {
1387             rtf.push_str(line);
1388             rtf.push_str("\\line ");
1389         }
1390         rtf.push('}');
1391
1392         fn filter(contents: &str, marker: &str) -> String {
1393             let start = format!("tool-{}-start", marker);
1394             let end = format!("tool-{}-end", marker);
1395             let mut lines = Vec::new();
1396             let mut omitted = false;
1397             for line in contents.lines() {
1398                 if line.contains(&start) {
1399                     omitted = true;
1400                 } else if line.contains(&end) {
1401                     omitted = false;
1402                 } else if !omitted {
1403                     lines.push(line);
1404                 }
1405             }
1406
1407             lines.join("\n")
1408         }
1409
1410         let xform = |p: &Path| {
1411             let mut contents = t!(fs::read_to_string(p));
1412             for tool in &["rust-demangler", "rust-analyzer", "miri", "rustfmt"] {
1413                 if !built_tools.contains(tool) {
1414                     contents = filter(&contents, tool);
1415                 }
1416             }
1417             let ret = tmp.join(p.file_name().unwrap());
1418             t!(fs::write(&ret, &contents));
1419             ret
1420         };
1421
1422         if target.contains("apple-darwin") {
1423             builder.info("building pkg installer");
1424             let pkg = tmp.join("pkg");
1425             let _ = fs::remove_dir_all(&pkg);
1426
1427             let pkgbuild = |component: &str| {
1428                 let mut cmd = Command::new("pkgbuild");
1429                 cmd.arg("--identifier")
1430                     .arg(format!("org.rust-lang.{}", component))
1431                     .arg("--scripts")
1432                     .arg(pkg.join(component))
1433                     .arg("--nopayload")
1434                     .arg(pkg.join(component).with_extension("pkg"));
1435                 builder.run(&mut cmd);
1436             };
1437
1438             let prepare = |name: &str| {
1439                 builder.create_dir(&pkg.join(name));
1440                 builder.cp_r(
1441                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1442                     &pkg.join(name),
1443                 );
1444                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1445                 pkgbuild(name);
1446             };
1447             prepare("rustc");
1448             prepare("cargo");
1449             prepare("rust-std");
1450             prepare("rust-analysis");
1451             prepare("clippy");
1452             for tool in &["rust-docs", "rust-demangler", "rust-analyzer", "miri"] {
1453                 if built_tools.contains(tool) {
1454                     prepare(tool);
1455                 }
1456             }
1457             // create an 'uninstall' package
1458             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1459             pkgbuild("uninstall");
1460
1461             builder.create_dir(&pkg.join("res"));
1462             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1463             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1464             let mut cmd = Command::new("productbuild");
1465             cmd.arg("--distribution")
1466                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1467                 .arg("--resources")
1468                 .arg(pkg.join("res"))
1469                 .arg(distdir(builder).join(format!(
1470                     "{}-{}.pkg",
1471                     pkgname(builder, "rust"),
1472                     target.triple
1473                 )))
1474                 .arg("--package-path")
1475                 .arg(&pkg);
1476             let _time = timeit(builder);
1477             builder.run(&mut cmd);
1478         }
1479
1480         if target.contains("windows") {
1481             let exe = tmp.join("exe");
1482             let _ = fs::remove_dir_all(&exe);
1483
1484             let prepare = |name: &str| {
1485                 builder.create_dir(&exe.join(name));
1486                 let dir = if name == "rust-std" || name == "rust-analysis" {
1487                     format!("{}-{}", name, target.triple)
1488                 } else if name == "rust-analyzer" {
1489                     "rust-analyzer-preview".to_string()
1490                 } else if name == "clippy" {
1491                     "clippy-preview".to_string()
1492                 } else if name == "rust-demangler" {
1493                     "rust-demangler-preview".to_string()
1494                 } else if name == "miri" {
1495                     "miri-preview".to_string()
1496                 } else {
1497                     name.to_string()
1498                 };
1499                 builder.cp_r(
1500                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1501                     &exe.join(name),
1502                 );
1503                 builder.remove(&exe.join(name).join("manifest.in"));
1504             };
1505             prepare("rustc");
1506             prepare("cargo");
1507             prepare("rust-analysis");
1508             prepare("rust-docs");
1509             prepare("rust-std");
1510             prepare("clippy");
1511             for tool in &["rust-demangler", "rust-analyzer", "miri"] {
1512                 if built_tools.contains(tool) {
1513                     prepare(tool);
1514                 }
1515             }
1516             if target.ends_with("windows-gnu") {
1517                 prepare("rust-mingw");
1518             }
1519
1520             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1521
1522             // Generate msi installer
1523             let wix_path = env::var_os("WIX")
1524                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1525             let wix = PathBuf::from(wix_path);
1526             let heat = wix.join("bin/heat.exe");
1527             let candle = wix.join("bin/candle.exe");
1528             let light = wix.join("bin/light.exe");
1529
1530             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1531             builder.run(
1532                 Command::new(&heat)
1533                     .current_dir(&exe)
1534                     .arg("dir")
1535                     .arg("rustc")
1536                     .args(&heat_flags)
1537                     .arg("-cg")
1538                     .arg("RustcGroup")
1539                     .arg("-dr")
1540                     .arg("Rustc")
1541                     .arg("-var")
1542                     .arg("var.RustcDir")
1543                     .arg("-out")
1544                     .arg(exe.join("RustcGroup.wxs")),
1545             );
1546             builder.run(
1547                 Command::new(&heat)
1548                     .current_dir(&exe)
1549                     .arg("dir")
1550                     .arg("rust-docs")
1551                     .args(&heat_flags)
1552                     .arg("-cg")
1553                     .arg("DocsGroup")
1554                     .arg("-dr")
1555                     .arg("Docs")
1556                     .arg("-var")
1557                     .arg("var.DocsDir")
1558                     .arg("-out")
1559                     .arg(exe.join("DocsGroup.wxs"))
1560                     .arg("-t")
1561                     .arg(etc.join("msi/squash-components.xsl")),
1562             );
1563             builder.run(
1564                 Command::new(&heat)
1565                     .current_dir(&exe)
1566                     .arg("dir")
1567                     .arg("cargo")
1568                     .args(&heat_flags)
1569                     .arg("-cg")
1570                     .arg("CargoGroup")
1571                     .arg("-dr")
1572                     .arg("Cargo")
1573                     .arg("-var")
1574                     .arg("var.CargoDir")
1575                     .arg("-out")
1576                     .arg(exe.join("CargoGroup.wxs"))
1577                     .arg("-t")
1578                     .arg(etc.join("msi/remove-duplicates.xsl")),
1579             );
1580             builder.run(
1581                 Command::new(&heat)
1582                     .current_dir(&exe)
1583                     .arg("dir")
1584                     .arg("rust-std")
1585                     .args(&heat_flags)
1586                     .arg("-cg")
1587                     .arg("StdGroup")
1588                     .arg("-dr")
1589                     .arg("Std")
1590                     .arg("-var")
1591                     .arg("var.StdDir")
1592                     .arg("-out")
1593                     .arg(exe.join("StdGroup.wxs")),
1594             );
1595             if built_tools.contains("rust-analyzer") {
1596                 builder.run(
1597                     Command::new(&heat)
1598                         .current_dir(&exe)
1599                         .arg("dir")
1600                         .arg("rust-analyzer")
1601                         .args(&heat_flags)
1602                         .arg("-cg")
1603                         .arg("RustAnalyzerGroup")
1604                         .arg("-dr")
1605                         .arg("RustAnalyzer")
1606                         .arg("-var")
1607                         .arg("var.RustAnalyzerDir")
1608                         .arg("-out")
1609                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1610                         .arg("-t")
1611                         .arg(etc.join("msi/remove-duplicates.xsl")),
1612                 );
1613             }
1614             builder.run(
1615                 Command::new(&heat)
1616                     .current_dir(&exe)
1617                     .arg("dir")
1618                     .arg("clippy")
1619                     .args(&heat_flags)
1620                     .arg("-cg")
1621                     .arg("ClippyGroup")
1622                     .arg("-dr")
1623                     .arg("Clippy")
1624                     .arg("-var")
1625                     .arg("var.ClippyDir")
1626                     .arg("-out")
1627                     .arg(exe.join("ClippyGroup.wxs"))
1628                     .arg("-t")
1629                     .arg(etc.join("msi/remove-duplicates.xsl")),
1630             );
1631             if built_tools.contains("rust-demangler") {
1632                 builder.run(
1633                     Command::new(&heat)
1634                         .current_dir(&exe)
1635                         .arg("dir")
1636                         .arg("rust-demangler")
1637                         .args(&heat_flags)
1638                         .arg("-cg")
1639                         .arg("RustDemanglerGroup")
1640                         .arg("-dr")
1641                         .arg("RustDemangler")
1642                         .arg("-var")
1643                         .arg("var.RustDemanglerDir")
1644                         .arg("-out")
1645                         .arg(exe.join("RustDemanglerGroup.wxs"))
1646                         .arg("-t")
1647                         .arg(etc.join("msi/remove-duplicates.xsl")),
1648                 );
1649             }
1650             if built_tools.contains("miri") {
1651                 builder.run(
1652                     Command::new(&heat)
1653                         .current_dir(&exe)
1654                         .arg("dir")
1655                         .arg("miri")
1656                         .args(&heat_flags)
1657                         .arg("-cg")
1658                         .arg("MiriGroup")
1659                         .arg("-dr")
1660                         .arg("Miri")
1661                         .arg("-var")
1662                         .arg("var.MiriDir")
1663                         .arg("-out")
1664                         .arg(exe.join("MiriGroup.wxs"))
1665                         .arg("-t")
1666                         .arg(etc.join("msi/remove-duplicates.xsl")),
1667                 );
1668             }
1669             builder.run(
1670                 Command::new(&heat)
1671                     .current_dir(&exe)
1672                     .arg("dir")
1673                     .arg("rust-analysis")
1674                     .args(&heat_flags)
1675                     .arg("-cg")
1676                     .arg("AnalysisGroup")
1677                     .arg("-dr")
1678                     .arg("Analysis")
1679                     .arg("-var")
1680                     .arg("var.AnalysisDir")
1681                     .arg("-out")
1682                     .arg(exe.join("AnalysisGroup.wxs"))
1683                     .arg("-t")
1684                     .arg(etc.join("msi/remove-duplicates.xsl")),
1685             );
1686             if target.ends_with("windows-gnu") {
1687                 builder.run(
1688                     Command::new(&heat)
1689                         .current_dir(&exe)
1690                         .arg("dir")
1691                         .arg("rust-mingw")
1692                         .args(&heat_flags)
1693                         .arg("-cg")
1694                         .arg("GccGroup")
1695                         .arg("-dr")
1696                         .arg("Gcc")
1697                         .arg("-var")
1698                         .arg("var.GccDir")
1699                         .arg("-out")
1700                         .arg(exe.join("GccGroup.wxs")),
1701                 );
1702             }
1703
1704             let candle = |input: &Path| {
1705                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1706                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1707                 let mut cmd = Command::new(&candle);
1708                 cmd.current_dir(&exe)
1709                     .arg("-nologo")
1710                     .arg("-dRustcDir=rustc")
1711                     .arg("-dDocsDir=rust-docs")
1712                     .arg("-dCargoDir=cargo")
1713                     .arg("-dStdDir=rust-std")
1714                     .arg("-dAnalysisDir=rust-analysis")
1715                     .arg("-dClippyDir=clippy")
1716                     .arg("-arch")
1717                     .arg(&arch)
1718                     .arg("-out")
1719                     .arg(&output)
1720                     .arg(&input);
1721                 add_env(builder, &mut cmd, target);
1722
1723                 if built_tools.contains("rust-demangler") {
1724                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1725                 }
1726                 if built_tools.contains("rust-analyzer") {
1727                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1728                 }
1729                 if built_tools.contains("miri") {
1730                     cmd.arg("-dMiriDir=miri");
1731                 }
1732                 if target.ends_with("windows-gnu") {
1733                     cmd.arg("-dGccDir=rust-mingw");
1734                 }
1735                 builder.run(&mut cmd);
1736             };
1737             candle(&xform(&etc.join("msi/rust.wxs")));
1738             candle(&etc.join("msi/ui.wxs"));
1739             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1740             candle("RustcGroup.wxs".as_ref());
1741             candle("DocsGroup.wxs".as_ref());
1742             candle("CargoGroup.wxs".as_ref());
1743             candle("StdGroup.wxs".as_ref());
1744             candle("ClippyGroup.wxs".as_ref());
1745             if built_tools.contains("rust-demangler") {
1746                 candle("RustDemanglerGroup.wxs".as_ref());
1747             }
1748             if built_tools.contains("rust-analyzer") {
1749                 candle("RustAnalyzerGroup.wxs".as_ref());
1750             }
1751             if built_tools.contains("miri") {
1752                 candle("MiriGroup.wxs".as_ref());
1753             }
1754             candle("AnalysisGroup.wxs".as_ref());
1755
1756             if target.ends_with("windows-gnu") {
1757                 candle("GccGroup.wxs".as_ref());
1758             }
1759
1760             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1761             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1762             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1763
1764             builder.info(&format!("building `msi` installer with {:?}", light));
1765             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1766             let mut cmd = Command::new(&light);
1767             cmd.arg("-nologo")
1768                 .arg("-ext")
1769                 .arg("WixUIExtension")
1770                 .arg("-ext")
1771                 .arg("WixUtilExtension")
1772                 .arg("-out")
1773                 .arg(exe.join(&filename))
1774                 .arg("rust.wixobj")
1775                 .arg("ui.wixobj")
1776                 .arg("rustwelcomedlg.wixobj")
1777                 .arg("RustcGroup.wixobj")
1778                 .arg("DocsGroup.wixobj")
1779                 .arg("CargoGroup.wixobj")
1780                 .arg("StdGroup.wixobj")
1781                 .arg("AnalysisGroup.wixobj")
1782                 .arg("ClippyGroup.wixobj")
1783                 .current_dir(&exe);
1784
1785             if built_tools.contains("rust-analyzer") {
1786                 cmd.arg("RustAnalyzerGroup.wixobj");
1787             }
1788             if built_tools.contains("rust-demangler") {
1789                 cmd.arg("RustDemanglerGroup.wixobj");
1790             }
1791             if built_tools.contains("miri") {
1792                 cmd.arg("MiriGroup.wixobj");
1793             }
1794
1795             if target.ends_with("windows-gnu") {
1796                 cmd.arg("GccGroup.wixobj");
1797             }
1798             // ICE57 wrongly complains about the shortcuts
1799             cmd.arg("-sice:ICE57");
1800
1801             let _time = timeit(builder);
1802             builder.run(&mut cmd);
1803
1804             if !builder.config.dry_run {
1805                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1806             }
1807         }
1808     }
1809 }
1810
1811 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1812     let mut parts = builder.version.split('.');
1813     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1814         .env("CFG_RELEASE_NUM", &builder.version)
1815         .env("CFG_RELEASE", builder.rust_release())
1816         .env("CFG_VER_MAJOR", parts.next().unwrap())
1817         .env("CFG_VER_MINOR", parts.next().unwrap())
1818         .env("CFG_VER_PATCH", parts.next().unwrap())
1819         .env("CFG_VER_BUILD", "0") // just needed to build
1820         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1821         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1822         .env("CFG_BUILD", target.triple)
1823         .env("CFG_CHANNEL", &builder.config.channel);
1824
1825     if target.contains("windows-gnullvm") {
1826         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1827     } else if target.contains("windows-gnu") {
1828         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1829     } else {
1830         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1831     }
1832 }
1833
1834 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1835 ///
1836 /// Returns whether the files were actually copied.
1837 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1838     if let Some(config) = builder.config.target_config.get(&target) {
1839         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1840             // If the LLVM was externally provided, then we don't currently copy
1841             // artifacts into the sysroot. This is not necessarily the right
1842             // choice (in particular, it will require the LLVM dylib to be in
1843             // the linker's load path at runtime), but the common use case for
1844             // external LLVMs is distribution provided LLVMs, and in that case
1845             // they're usually in the standard search path (e.g., /usr/lib) and
1846             // copying them here is going to cause problems as we may end up
1847             // with the wrong files and isn't what distributions want.
1848             //
1849             // This behavior may be revisited in the future though.
1850             //
1851             // If the LLVM is coming from ourselves (just from CI) though, we
1852             // still want to install it, as it otherwise won't be available.
1853             return false;
1854         }
1855     }
1856
1857     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1858     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1859     // clear why this is the case, though. llvm-config will emit the versioned
1860     // paths and we don't want those in the sysroot (as we're expecting
1861     // unversioned paths).
1862     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1863         let src_libdir = builder.llvm_out(target).join("lib");
1864         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1865         if llvm_dylib_path.exists() {
1866             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1867         }
1868         !builder.config.dry_run
1869     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1870         let mut cmd = Command::new(llvm_config);
1871         cmd.arg("--libfiles");
1872         builder.verbose(&format!("running {:?}", cmd));
1873         let files = output(&mut cmd);
1874         let build_llvm_out = &builder.llvm_out(builder.config.build);
1875         let target_llvm_out = &builder.llvm_out(target);
1876         for file in files.trim_end().split(' ') {
1877             // If we're not using a custom LLVM, make sure we package for the target.
1878             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1879                 target_llvm_out.join(relative_path)
1880             } else {
1881                 PathBuf::from(file)
1882             };
1883             builder.install(&file, dst_libdir, 0o644);
1884         }
1885         !builder.config.dry_run
1886     } else {
1887         false
1888     }
1889 }
1890
1891 /// Maybe add libLLVM.so to the target lib-dir for linking.
1892 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1893     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1894     // We do not need to copy LLVM files into the sysroot if it is not
1895     // dynamically linked; it is already included into librustc_llvm
1896     // statically.
1897     if builder.llvm_link_shared() {
1898         maybe_install_llvm(builder, target, &dst_libdir);
1899     }
1900 }
1901
1902 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1903 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1904     let dst_libdir =
1905         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1906     // We do not need to copy LLVM files into the sysroot if it is not
1907     // dynamically linked; it is already included into librustc_llvm
1908     // statically.
1909     if builder.llvm_link_shared() {
1910         maybe_install_llvm(builder, target, &dst_libdir);
1911     }
1912 }
1913
1914 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1915 pub struct LlvmTools {
1916     pub target: TargetSelection,
1917 }
1918
1919 impl Step for LlvmTools {
1920     type Output = Option<GeneratedTarball>;
1921     const ONLY_HOSTS: bool = true;
1922     const DEFAULT: bool = true;
1923
1924     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1925         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1926         // FIXME: allow using the names of the tools themselves?
1927         run.alias("llvm-tools").default_condition(default)
1928     }
1929
1930     fn make_run(run: RunConfig<'_>) {
1931         run.builder.ensure(LlvmTools { target: run.target });
1932     }
1933
1934     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1935         let target = self.target;
1936
1937         /* run only if llvm-config isn't used */
1938         if let Some(config) = builder.config.target_config.get(&target) {
1939             if let Some(ref _s) = config.llvm_config {
1940                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1941                 return None;
1942             }
1943         }
1944
1945         builder.ensure(crate::native::Llvm { target });
1946
1947         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1948         tarball.set_overlay(OverlayKind::LLVM);
1949         tarball.is_preview(true);
1950
1951         // Prepare the image directory
1952         let src_bindir = builder.llvm_out(target).join("bin");
1953         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1954         for tool in LLVM_TOOLS {
1955             let exe = src_bindir.join(exe(tool, target));
1956             tarball.add_file(&exe, &dst_bindir, 0o755);
1957         }
1958
1959         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1960         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1961         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1962         // compiler libraries.
1963         maybe_install_llvm_target(builder, target, tarball.image_dir());
1964
1965         Some(tarball.generate())
1966     }
1967 }
1968
1969 // Tarball intended for internal consumption to ease rustc/std development.
1970 //
1971 // Should not be considered stable by end users.
1972 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1973 pub struct RustDev {
1974     pub target: TargetSelection,
1975 }
1976
1977 impl Step for RustDev {
1978     type Output = Option<GeneratedTarball>;
1979     const DEFAULT: bool = true;
1980     const ONLY_HOSTS: bool = true;
1981
1982     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1983         run.alias("rust-dev")
1984     }
1985
1986     fn make_run(run: RunConfig<'_>) {
1987         run.builder.ensure(RustDev { target: run.target });
1988     }
1989
1990     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1991         let target = self.target;
1992
1993         /* run only if llvm-config isn't used */
1994         if let Some(config) = builder.config.target_config.get(&target) {
1995             if let Some(ref _s) = config.llvm_config {
1996                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
1997                 return None;
1998             }
1999         }
2000
2001         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2002         tarball.set_overlay(OverlayKind::LLVM);
2003
2004         builder.ensure(crate::native::Llvm { target });
2005
2006         let src_bindir = builder.llvm_out(target).join("bin");
2007         // If updating this list, you likely want to change
2008         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2009         // will not pick up the extra file until LLVM gets bumped.
2010         for bin in &[
2011             "llvm-config",
2012             "llvm-ar",
2013             "llvm-objdump",
2014             "llvm-profdata",
2015             "llvm-bcanalyzer",
2016             "llvm-cov",
2017             "llvm-dwp",
2018             "llvm-nm",
2019             "llvm-dwarfdump",
2020         ] {
2021             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2022         }
2023
2024         // We don't build LLD on some platforms, so only add it if it exists
2025         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2026         if lld_path.exists() {
2027             tarball.add_file(lld_path, "bin", 0o755);
2028         }
2029
2030         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2031
2032         // Copy the include directory as well; needed mostly to build
2033         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2034         // just broadly useful to be able to link against the bundled LLVM.
2035         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2036
2037         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2038         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2039         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2040         // compiler libraries.
2041         let dst_libdir = tarball.image_dir().join("lib");
2042         maybe_install_llvm(builder, target, &dst_libdir);
2043         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2044         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2045
2046         Some(tarball.generate())
2047     }
2048 }
2049
2050 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2051 /// release process to avoid cloning the monorepo and building stuff.
2052 ///
2053 /// Should not be considered stable by end users.
2054 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2055 pub struct BuildManifest {
2056     pub target: TargetSelection,
2057 }
2058
2059 impl Step for BuildManifest {
2060     type Output = GeneratedTarball;
2061     const DEFAULT: bool = false;
2062     const ONLY_HOSTS: bool = true;
2063
2064     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2065         run.alias("build-manifest")
2066     }
2067
2068     fn make_run(run: RunConfig<'_>) {
2069         run.builder.ensure(BuildManifest { target: run.target });
2070     }
2071
2072     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2073         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2074
2075         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2076         tarball.add_file(&build_manifest, "bin", 0o755);
2077         tarball.generate()
2078     }
2079 }
2080
2081 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2082 ///
2083 /// Currently this is the PGO profile data.
2084 ///
2085 /// Should not be considered stable by end users.
2086 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2087 pub struct ReproducibleArtifacts {
2088     pub target: TargetSelection,
2089 }
2090
2091 impl Step for ReproducibleArtifacts {
2092     type Output = Option<GeneratedTarball>;
2093     const DEFAULT: bool = true;
2094     const ONLY_HOSTS: bool = true;
2095
2096     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2097         run.alias("reproducible-artifacts")
2098     }
2099
2100     fn make_run(run: RunConfig<'_>) {
2101         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2102     }
2103
2104     fn run(self, builder: &Builder<'_>) -> Self::Output {
2105         let mut added_anything = false;
2106         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2107         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2108             tarball.add_file(path, ".", 0o644);
2109             added_anything = true;
2110         }
2111         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2112             tarball.add_file(path, ".", 0o644);
2113             added_anything = true;
2114         }
2115         if added_anything { Some(tarball.generate()) } else { None }
2116     }
2117 }