src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.alias("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.alias("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.ends_with("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.alias("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.ends_with("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 let exe_name = exe("ld", compiler.host);
 411                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 412             }
 413
 414             // Man pages
 415             t!(fs::create_dir_all(image.join("share/man/man1")));
 416             let man_src = builder.src.join("src/doc/man");
 417             let man_dst = image.join("share/man/man1");
 418
 419             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 420             // to hardlink, and we don't want to edit the source templates
 421             for file_entry in builder.read_dir(&man_src) {
 422                 let page_src = file_entry.path();
 423                 let page_dst = man_dst.join(file_entry.file_name());
 424                 let src_text = t!(std::fs::read_to_string(&page_src));
 425                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 426                 t!(std::fs::write(&page_dst, &new_text));
 427                 t!(fs::copy(&page_src, &page_dst));
 428             }
 429
 430             // Debugger scripts
 431             builder
 432                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 433
 434             // Misc license info
 435             let cp = |file: &str| {
 436                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 437             };
 438             cp("COPYRIGHT");
 439             cp("LICENSE-APACHE");
 440             cp("LICENSE-MIT");
 441             cp("README.md");
 442         }
 443     }
 444 }
 445
 446 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 447 pub struct DebuggerScripts {
 448     pub sysroot: Interned<PathBuf>,
 449     pub host: TargetSelection,
 450 }
 451
 452 impl Step for DebuggerScripts {
 453     type Output = ();
 454
 455     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 456         run.never()
 457     }
 458
 459     /// Copies debugger scripts for `target` into the `sysroot` specified.
 460     fn run(self, builder: &Builder<'_>) {
 461         let host = self.host;
 462         let sysroot = self.sysroot;
 463         let dst = sysroot.join("lib/rustlib/etc");
 464         t!(fs::create_dir_all(&dst));
 465         let cp_debugger_script = |file: &str| {
 466             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 467         };
 468         if host.contains("windows-msvc") {
 469             // windbg debugger scripts
 470             builder.install(
 471                 &builder.src.join("src/etc/rust-windbg.cmd"),
 472                 &sysroot.join("bin"),
 473                 0o755,
 474             );
 475
 476             cp_debugger_script("natvis/intrinsic.natvis");
 477             cp_debugger_script("natvis/liballoc.natvis");
 478             cp_debugger_script("natvis/libcore.natvis");
 479             cp_debugger_script("natvis/libstd.natvis");
 480         } else {
 481             cp_debugger_script("rust_types.py");
 482
 483             // gdb debugger scripts
 484             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 485             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 486
 487             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 488             cp_debugger_script("gdb_lookup.py");
 489             cp_debugger_script("gdb_providers.py");
 490
 491             // lldb debugger scripts
 492             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 493
 494             cp_debugger_script("lldb_lookup.py");
 495             cp_debugger_script("lldb_providers.py");
 496             cp_debugger_script("lldb_commands")
 497         }
 498     }
 499 }
 500
 501 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 502     // The only true set of target libraries came from the build triple, so
 503     // let's reduce redundant work by only producing archives from that host.
 504     if compiler.host != builder.config.build {
 505         builder.info("\tskipping, not a build host");
 506         true
 507     } else {
 508         false
 509     }
 510 }
 511
 512 /// Copy stamped files into an image's `target/lib` directory.
 513 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 514     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 515     let self_contained_dst = dst.join("self-contained");
 516     t!(fs::create_dir_all(&dst));
 517     t!(fs::create_dir_all(&self_contained_dst));
 518     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 519         if dependency_type == DependencyType::TargetSelfContained {
 520             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 521         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 522             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 523         }
 524     }
 525 }
 526
 527 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 528 pub struct Std {
 529     pub compiler: Compiler,
 530     pub target: TargetSelection,
 531 }
 532
 533 impl Step for Std {
 534     type Output = Option<GeneratedTarball>;
 535     const DEFAULT: bool = true;
 536
 537     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 538         run.alias("rust-std")
 539     }
 540
 541     fn make_run(run: RunConfig<'_>) {
 542         run.builder.ensure(Std {
 543             compiler: run.builder.compiler_for(
 544                 run.builder.top_stage,
 545                 run.builder.config.build,
 546                 run.target,
 547             ),
 548             target: run.target,
 549         });
 550     }
 551
 552     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 553         let compiler = self.compiler;
 554         let target = self.target;
 555
 556         if skip_host_target_lib(builder, compiler) {
 557             return None;
 558         }
 559
 560         builder.ensure(compile::Std { compiler, target });
 561
 562         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 563         tarball.include_target_in_component_name(true);
 564
 565         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 566         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 567         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 568
 569         Some(tarball.generate())
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct RustcDev {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for RustcDev {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582     const ONLY_HOSTS: bool = true;
 583
 584     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 585         run.alias("rustc-dev")
 586     }
 587
 588     fn make_run(run: RunConfig<'_>) {
 589         run.builder.ensure(RustcDev {
 590             compiler: run.builder.compiler_for(
 591                 run.builder.top_stage,
 592                 run.builder.config.build,
 593                 run.target,
 594             ),
 595             target: run.target,
 596         });
 597     }
 598
 599     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 600         let compiler = self.compiler;
 601         let target = self.target;
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Rustc { compiler, target });
 607
 608         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 609
 610         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 611         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 612         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 613
 614         let src_files = &["Cargo.lock"];
 615         // This is the reduced set of paths which will become the rustc-dev component
 616         // (essentially the compiler crates and all of their path dependencies).
 617         copy_src_dirs(
 618             builder,
 619             &builder.src,
 620             &["compiler"],
 621             &[],
 622             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 623         );
 624         for file in src_files {
 625             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 626         }
 627
 628         Some(tarball.generate())
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: TargetSelection,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = Option<GeneratedTarball>;
 640     const DEFAULT: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         let default = should_build_extended_tool(&run.builder, "analysis");
 644         run.alias("rust-analysis").default_condition(default)
 645     }
 646
 647     fn make_run(run: RunConfig<'_>) {
 648         run.builder.ensure(Analysis {
 649             // Find the actual compiler (handling the full bootstrap option) which
 650             // produced the save-analysis data because that data isn't copied
 651             // through the sysroot uplifting.
 652             compiler: run.builder.compiler_for(
 653                 run.builder.top_stage,
 654                 run.builder.config.build,
 655                 run.target,
 656             ),
 657             target: run.target,
 658         });
 659     }
 660
 661     /// Creates a tarball of save-analysis metadata, if available.
 662     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 663         let compiler = self.compiler;
 664         let target = self.target;
 665         if compiler.host != builder.config.build {
 666             return None;
 667         }
 668
 669         builder.ensure(compile::Std { compiler, target });
 670         let src = builder
 671             .stage_out(compiler, Mode::Std)
 672             .join(target.triple)
 673             .join(builder.cargo_dir())
 674             .join("deps")
 675             .join("save-analysis");
 676
 677         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 678         tarball.include_target_in_component_name(true);
 679         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 680         Some(tarball.generate())
 681     }
 682 }
 683
 684 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 685 /// `dst_dir`.
 686 fn copy_src_dirs(
 687     builder: &Builder<'_>,
 688     base: &Path,
 689     src_dirs: &[&str],
 690     exclude_dirs: &[&str],
 691     dst_dir: &Path,
 692 ) {
 693     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 694         let spath = match path.to_str() {
 695             Some(path) => path,
 696             None => return false,
 697         };
 698         if spath.ends_with('~') || spath.ends_with(".pyc") {
 699             return false;
 700         }
 701
 702         const LLVM_PROJECTS: &[&str] = &[
 703             "llvm-project/clang",
 704             "llvm-project\\clang",
 705             "llvm-project/libunwind",
 706             "llvm-project\\libunwind",
 707             "llvm-project/lld",
 708             "llvm-project\\lld",
 709             "llvm-project/lldb",
 710             "llvm-project\\lldb",
 711             "llvm-project/llvm",
 712             "llvm-project\\llvm",
 713             "llvm-project/compiler-rt",
 714             "llvm-project\\compiler-rt",
 715             "llvm-project/cmake",
 716             "llvm-project\\cmake",
 717         ];
 718         if spath.contains("llvm-project")
 719             && !spath.ends_with("llvm-project")
 720             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 721         {
 722             return false;
 723         }
 724
 725         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 726         if LLVM_TEST.iter().any(|path| spath.contains(path))
 727             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 728         {
 729             return false;
 730         }
 731
 732         let full_path = Path::new(dir).join(path);
 733         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 734             return false;
 735         }
 736
 737         let excludes = [
 738             "CVS",
 739             "RCS",
 740             "SCCS",
 741             ".git",
 742             ".gitignore",
 743             ".gitmodules",
 744             ".gitattributes",
 745             ".cvsignore",
 746             ".svn",
 747             ".arch-ids",
 748             "{arch}",
 749             "=RELEASE-ID",
 750             "=meta-update",
 751             "=update",
 752             ".bzr",
 753             ".bzrignore",
 754             ".bzrtags",
 755             ".hg",
 756             ".hgignore",
 757             ".hgrags",
 758             "_darcs",
 759         ];
 760         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 761     }
 762
 763     // Copy the directories using our filter
 764     for item in src_dirs {
 765         let dst = &dst_dir.join(item);
 766         t!(fs::create_dir_all(dst));
 767         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 768     }
 769 }
 770
 771 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 772 pub struct Src;
 773
 774 impl Step for Src {
 775     /// The output path of the src installer tarball
 776     type Output = GeneratedTarball;
 777     const DEFAULT: bool = true;
 778     const ONLY_HOSTS: bool = true;
 779
 780     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 781         run.alias("rust-src")
 782     }
 783
 784     fn make_run(run: RunConfig<'_>) {
 785         run.builder.ensure(Src);
 786     }
 787
 788     /// Creates the `rust-src` installer component
 789     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 790         let tarball = Tarball::new_targetless(builder, "rust-src");
 791
 792         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 793         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 794         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 795         // and fix them...
 796         //
 797         // NOTE: if you update the paths here, you also should update the "virtual" path
 798         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 799         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 800
 801         let src_files = ["Cargo.lock"];
 802         // This is the reduced set of paths which will become the rust-src component
 803         // (essentially libstd and all of its path dependencies).
 804         copy_src_dirs(
 805             builder,
 806             &builder.src,
 807             &["library", "src/llvm-project/libunwind"],
 808             &[
 809                 // not needed and contains symlinks which rustup currently
 810                 // chokes on when unpacking.
 811                 "library/backtrace/crates",
 812                 // these are 30MB combined and aren't necessary for building
 813                 // the standard library.
 814                 "library/stdarch/Cargo.toml",
 815                 "library/stdarch/crates/stdarch-verify",
 816                 "library/stdarch/crates/intrinsic-test",
 817             ],
 818             &dst_src,
 819         );
 820         for file in src_files.iter() {
 821             builder.copy(&builder.src.join(file), &dst_src.join(file));
 822         }
 823
 824         tarball.generate()
 825     }
 826 }
 827
 828 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 829 pub struct PlainSourceTarball;
 830
 831 impl Step for PlainSourceTarball {
 832     /// Produces the location of the tarball generated
 833     type Output = GeneratedTarball;
 834     const DEFAULT: bool = true;
 835     const ONLY_HOSTS: bool = true;
 836
 837     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 838         let builder = run.builder;
 839         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig<'_>) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 848         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 849         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 850         // It also contains symbolic links, unlike other any other dist tarball.
 851         // It's used for distros building rustc from source in a pre-vendored environment.
 852         let mut tarball = Tarball::new(builder, "rustc", "src");
 853         tarball.permit_symlinks(true);
 854         let plain_dst_src = tarball.image_dir();
 855
 856         // This is the set of root paths which will become part of the source package
 857         let src_files = [
 858             "COPYRIGHT",
 859             "LICENSE-APACHE",
 860             "LICENSE-MIT",
 861             "CONTRIBUTING.md",
 862             "README.md",
 863             "RELEASES.md",
 864             "configure",
 865             "x.py",
 866             "config.toml.example",
 867             "Cargo.toml",
 868             "Cargo.lock",
 869         ];
 870         let src_dirs = ["src", "compiler", "library"];
 871
 872         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 873
 874         // Copy the files normally
 875         for item in &src_files {
 876             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 877         }
 878
 879         // Create the version file
 880         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 881         if let Some(sha) = builder.rust_sha() {
 882             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 883         }
 884
 885         // If we're building from git sources, we need to vendor a complete distribution.
 886         if builder.rust_info.is_git() {
 887             // Ensure we have the submodules checked out.
 888             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 889
 890             // Vendor all Cargo dependencies
 891             let mut cmd = Command::new(&builder.initial_cargo);
 892             cmd.arg("vendor")
 893                 .arg("--sync")
 894                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 895                 .arg("--sync")
 896                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 897                 .arg("--sync")
 898                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 899                 .current_dir(&plain_dst_src);
 900
 901             let config = if !builder.config.dry_run {
 902                 t!(String::from_utf8(t!(cmd.output()).stdout))
 903             } else {
 904                 String::new()
 905             };
 906
 907             let cargo_config_dir = plain_dst_src.join(".cargo");
 908             builder.create_dir(&cargo_config_dir);
 909             builder.create(&cargo_config_dir.join("config.toml"), &config);
 910         }
 911
 912         tarball.bare()
 913     }
 914 }
 915
 916 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 917 pub struct Cargo {
 918     pub compiler: Compiler,
 919     pub target: TargetSelection,
 920 }
 921
 922 impl Step for Cargo {
 923     type Output = Option<GeneratedTarball>;
 924     const DEFAULT: bool = true;
 925     const ONLY_HOSTS: bool = true;
 926
 927     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 928         let default = should_build_extended_tool(&run.builder, "cargo");
 929         run.alias("cargo").default_condition(default)
 930     }
 931
 932     fn make_run(run: RunConfig<'_>) {
 933         run.builder.ensure(Cargo {
 934             compiler: run.builder.compiler_for(
 935                 run.builder.top_stage,
 936                 run.builder.config.build,
 937                 run.target,
 938             ),
 939             target: run.target,
 940         });
 941     }
 942
 943     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 944         let compiler = self.compiler;
 945         let target = self.target;
 946
 947         let cargo = builder.ensure(tool::Cargo { compiler, target });
 948         let src = builder.src.join("src/tools/cargo");
 949         let etc = src.join("src/etc");
 950
 951         // Prepare the image directory
 952         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 953         tarball.set_overlay(OverlayKind::Cargo);
 954
 955         tarball.add_file(&cargo, "bin", 0o755);
 956         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 957         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 958         tarball.add_dir(etc.join("man"), "share/man/man1");
 959         tarball.add_legal_and_readme_to("share/doc/cargo");
 960
 961         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 962             let dirent = dirent.expect("read dir entry");
 963             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 964                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 965             }
 966         }
 967
 968         Some(tarball.generate())
 969     }
 970 }
 971
 972 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 973 pub struct Rls {
 974     pub compiler: Compiler,
 975     pub target: TargetSelection,
 976 }
 977
 978 impl Step for Rls {
 979     type Output = Option<GeneratedTarball>;
 980     const ONLY_HOSTS: bool = true;
 981     const DEFAULT: bool = true;
 982
 983     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 984         let default = should_build_extended_tool(&run.builder, "rls");
 985         run.alias("rls").default_condition(default)
 986     }
 987
 988     fn make_run(run: RunConfig<'_>) {
 989         run.builder.ensure(Rls {
 990             compiler: run.builder.compiler_for(
 991                 run.builder.top_stage,
 992                 run.builder.config.build,
 993                 run.target,
 994             ),
 995             target: run.target,
 996         });
 997     }
 998
 999     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1000         let compiler = self.compiler;
1001         let target = self.target;
1002
1003         let rls = builder
1004             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1005             .or_else(|| {
1006                 missing_tool("RLS", builder.build.config.missing_tools);
1007                 None
1008             })?;
1009
1010         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1011         tarball.set_overlay(OverlayKind::RLS);
1012         tarball.is_preview(true);
1013         tarball.add_file(rls, "bin", 0o755);
1014         tarball.add_legal_and_readme_to("share/doc/rls");
1015         Some(tarball.generate())
1016     }
1017 }
1018
1019 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1020 pub struct RustAnalyzer {
1021     pub compiler: Compiler,
1022     pub target: TargetSelection,
1023 }
1024
1025 impl Step for RustAnalyzer {
1026     type Output = Option<GeneratedTarball>;
1027     const DEFAULT: bool = true;
1028     const ONLY_HOSTS: bool = true;
1029
1030     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1031         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1032         run.alias("rust-analyzer").default_condition(default)
1033     }
1034
1035     fn make_run(run: RunConfig<'_>) {
1036         run.builder.ensure(RustAnalyzer {
1037             compiler: run.builder.compiler_for(
1038                 run.builder.top_stage,
1039                 run.builder.config.build,
1040                 run.target,
1041             ),
1042             target: run.target,
1043         });
1044     }
1045
1046     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1047         // This prevents rust-analyzer from being built for "dist" or "install"
1048         // on the stable/beta channels. It is a nightly-only tool and should
1049         // not be included.
1050         if !builder.build.unstable_features() {
1051             return None;
1052         }
1053         let compiler = self.compiler;
1054         let target = self.target;
1055
1056         if target.contains("riscv64") {
1057             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1058             // to build. See #74813 for details.
1059             return None;
1060         }
1061
1062         let rust_analyzer = builder
1063             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1064             .expect("rust-analyzer always builds");
1065
1066         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1067         tarball.set_overlay(OverlayKind::RustAnalyzer);
1068         tarball.is_preview(true);
1069         tarball.add_file(rust_analyzer, "bin", 0o755);
1070         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1071         Some(tarball.generate())
1072     }
1073 }
1074
1075 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1076 pub struct Clippy {
1077     pub compiler: Compiler,
1078     pub target: TargetSelection,
1079 }
1080
1081 impl Step for Clippy {
1082     type Output = Option<GeneratedTarball>;
1083     const DEFAULT: bool = true;
1084     const ONLY_HOSTS: bool = true;
1085
1086     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1087         let default = should_build_extended_tool(&run.builder, "clippy");
1088         run.alias("clippy").default_condition(default)
1089     }
1090
1091     fn make_run(run: RunConfig<'_>) {
1092         run.builder.ensure(Clippy {
1093             compiler: run.builder.compiler_for(
1094                 run.builder.top_stage,
1095                 run.builder.config.build,
1096                 run.target,
1097             ),
1098             target: run.target,
1099         });
1100     }
1101
1102     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1103         let compiler = self.compiler;
1104         let target = self.target;
1105
1106         // Prepare the image directory
1107         // We expect clippy to build, because we've exited this step above if tool
1108         // state for clippy isn't testing.
1109         let clippy = builder
1110             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1111             .expect("clippy expected to build - essential tool");
1112         let cargoclippy = builder
1113             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1114             .expect("clippy expected to build - essential tool");
1115
1116         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1117         tarball.set_overlay(OverlayKind::Clippy);
1118         tarball.is_preview(true);
1119         tarball.add_file(clippy, "bin", 0o755);
1120         tarball.add_file(cargoclippy, "bin", 0o755);
1121         tarball.add_legal_and_readme_to("share/doc/clippy");
1122         Some(tarball.generate())
1123     }
1124 }
1125
1126 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1127 pub struct Miri {
1128     pub compiler: Compiler,
1129     pub target: TargetSelection,
1130 }
1131
1132 impl Step for Miri {
1133     type Output = Option<GeneratedTarball>;
1134     const DEFAULT: bool = true;
1135     const ONLY_HOSTS: bool = true;
1136
1137     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1138         let default = should_build_extended_tool(&run.builder, "miri");
1139         run.alias("miri").default_condition(default)
1140     }
1141
1142     fn make_run(run: RunConfig<'_>) {
1143         run.builder.ensure(Miri {
1144             compiler: run.builder.compiler_for(
1145                 run.builder.top_stage,
1146                 run.builder.config.build,
1147                 run.target,
1148             ),
1149             target: run.target,
1150         });
1151     }
1152
1153     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1154         // This prevents miri from being built for "dist" or "install"
1155         // on the stable/beta channels. It is a nightly-only tool and should
1156         // not be included.
1157         if !builder.build.unstable_features() {
1158             return None;
1159         }
1160         let compiler = self.compiler;
1161         let target = self.target;
1162
1163         let miri = builder
1164             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1165             .or_else(|| {
1166                 missing_tool("miri", builder.build.config.missing_tools);
1167                 None
1168             })?;
1169         let cargomiri = builder
1170             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1171             .or_else(|| {
1172                 missing_tool("cargo miri", builder.build.config.missing_tools);
1173                 None
1174             })?;
1175
1176         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1177         tarball.set_overlay(OverlayKind::Miri);
1178         tarball.is_preview(true);
1179         tarball.add_file(miri, "bin", 0o755);
1180         tarball.add_file(cargomiri, "bin", 0o755);
1181         tarball.add_legal_and_readme_to("share/doc/miri");
1182         Some(tarball.generate())
1183     }
1184 }
1185
1186 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1187 pub struct Rustfmt {
1188     pub compiler: Compiler,
1189     pub target: TargetSelection,
1190 }
1191
1192 impl Step for Rustfmt {
1193     type Output = Option<GeneratedTarball>;
1194     const DEFAULT: bool = true;
1195     const ONLY_HOSTS: bool = true;
1196
1197     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1198         let default = should_build_extended_tool(&run.builder, "rustfmt");
1199         run.alias("rustfmt").default_condition(default)
1200     }
1201
1202     fn make_run(run: RunConfig<'_>) {
1203         run.builder.ensure(Rustfmt {
1204             compiler: run.builder.compiler_for(
1205                 run.builder.top_stage,
1206                 run.builder.config.build,
1207                 run.target,
1208             ),
1209             target: run.target,
1210         });
1211     }
1212
1213     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1214         let compiler = self.compiler;
1215         let target = self.target;
1216
1217         let rustfmt = builder
1218             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1219             .or_else(|| {
1220                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1221                 None
1222             })?;
1223         let cargofmt = builder
1224             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1225             .or_else(|| {
1226                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1227                 None
1228             })?;
1229
1230         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1231         tarball.set_overlay(OverlayKind::Rustfmt);
1232         tarball.is_preview(true);
1233         tarball.add_file(rustfmt, "bin", 0o755);
1234         tarball.add_file(cargofmt, "bin", 0o755);
1235         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1236         Some(tarball.generate())
1237     }
1238 }
1239
1240 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1241 pub struct RustDemangler {
1242     pub compiler: Compiler,
1243     pub target: TargetSelection,
1244 }
1245
1246 impl Step for RustDemangler {
1247     type Output = Option<GeneratedTarball>;
1248     const DEFAULT: bool = true;
1249     const ONLY_HOSTS: bool = true;
1250
1251     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1252         // While other tools use `should_build_extended_tool` to decide whether to be run by
1253         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1254         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1255         // we run the step by default when only `extended = true`, and decide whether to actually
1256         // run it or not later.
1257         let default = run.builder.config.extended;
1258         run.alias("rust-demangler").default_condition(default)
1259     }
1260
1261     fn make_run(run: RunConfig<'_>) {
1262         run.builder.ensure(RustDemangler {
1263             compiler: run.builder.compiler_for(
1264                 run.builder.top_stage,
1265                 run.builder.config.build,
1266                 run.target,
1267             ),
1268             target: run.target,
1269         });
1270     }
1271
1272     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1273         let compiler = self.compiler;
1274         let target = self.target;
1275
1276         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1277         let condition = should_build_extended_tool(builder, "rust-demangler")
1278             || builder.config.profiler_enabled(target);
1279         if builder.config.extended && !condition {
1280             return None;
1281         }
1282
1283         let rust_demangler = builder
1284             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1285             .expect("rust-demangler expected to build - in-tree tool");
1286
1287         // Prepare the image directory
1288         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1289         tarball.set_overlay(OverlayKind::RustDemangler);
1290         tarball.is_preview(true);
1291         tarball.add_file(&rust_demangler, "bin", 0o755);
1292         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1293         Some(tarball.generate())
1294     }
1295 }
1296
1297 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1298 pub struct Extended {
1299     stage: u32,
1300     host: TargetSelection,
1301     target: TargetSelection,
1302 }
1303
1304 impl Step for Extended {
1305     type Output = ();
1306     const DEFAULT: bool = true;
1307     const ONLY_HOSTS: bool = true;
1308
1309     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1310         let builder = run.builder;
1311         run.alias("extended").default_condition(builder.config.extended)
1312     }
1313
1314     fn make_run(run: RunConfig<'_>) {
1315         run.builder.ensure(Extended {
1316             stage: run.builder.top_stage,
1317             host: run.builder.config.build,
1318             target: run.target,
1319         });
1320     }
1321
1322     /// Creates a combined installer for the specified target in the provided stage.
1323     fn run(self, builder: &Builder<'_>) {
1324         let target = self.target;
1325         let stage = self.stage;
1326         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1327
1328         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1329
1330         let mut tarballs = Vec::new();
1331         let mut built_tools = HashSet::new();
1332         macro_rules! add_component {
1333             ($name:expr => $step:expr) => {
1334                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1335                     tarballs.push(tarball);
1336                     built_tools.insert($name);
1337                 }
1338             };
1339         }
1340
1341         // When rust-std package split from rustc, we needed to ensure that during
1342         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1343         // the std files during uninstall. To do this ensure that rustc comes
1344         // before rust-std in the list below.
1345         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1346         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1347
1348         if target.ends_with("windows-gnu") {
1349             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1350         }
1351
1352         add_component!("rust-docs" => Docs { host: target });
1353         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1354         add_component!("cargo" => Cargo { compiler, target });
1355         add_component!("rustfmt" => Rustfmt { compiler, target });
1356         add_component!("rls" => Rls { compiler, target });
1357         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1358         add_component!("llvm-components" => LlvmTools { target });
1359         add_component!("clippy" => Clippy { compiler, target });
1360         add_component!("miri" => Miri { compiler, target });
1361         add_component!("analysis" => Analysis { compiler, target });
1362
1363         let etc = builder.src.join("src/etc/installer");
1364
1365         // Avoid producing tarballs during a dry run.
1366         if builder.config.dry_run {
1367             return;
1368         }
1369
1370         let tarball = Tarball::new(builder, "rust", &target.triple);
1371         let generated = tarball.combine(&tarballs);
1372
1373         let tmp = tmpdir(builder).join("combined-tarball");
1374         let work = generated.work_dir();
1375
1376         let mut license = String::new();
1377         license += &builder.read(&builder.src.join("COPYRIGHT"));
1378         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1379         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1380         license.push('\n');
1381         license.push('\n');
1382
1383         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1384         let mut rtf = rtf.to_string();
1385         rtf.push('\n');
1386         for line in license.lines() {
1387             rtf.push_str(line);
1388             rtf.push_str("\\line ");
1389         }
1390         rtf.push('}');
1391
1392         fn filter(contents: &str, marker: &str) -> String {
1393             let start = format!("tool-{}-start", marker);
1394             let end = format!("tool-{}-end", marker);
1395             let mut lines = Vec::new();
1396             let mut omitted = false;
1397             for line in contents.lines() {
1398                 if line.contains(&start) {
1399                     omitted = true;
1400                 } else if line.contains(&end) {
1401                     omitted = false;
1402                 } else if !omitted {
1403                     lines.push(line);
1404                 }
1405             }
1406
1407             lines.join("\n")
1408         }
1409
1410         let xform = |p: &Path| {
1411             let mut contents = t!(fs::read_to_string(p));
1412             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1413                 if !built_tools.contains(tool) {
1414                     contents = filter(&contents, tool);
1415                 }
1416             }
1417             let ret = tmp.join(p.file_name().unwrap());
1418             t!(fs::write(&ret, &contents));
1419             ret
1420         };
1421
1422         if target.contains("apple-darwin") {
1423             builder.info("building pkg installer");
1424             let pkg = tmp.join("pkg");
1425             let _ = fs::remove_dir_all(&pkg);
1426
1427             let pkgbuild = |component: &str| {
1428                 let mut cmd = Command::new("pkgbuild");
1429                 cmd.arg("--identifier")
1430                     .arg(format!("org.rust-lang.{}", component))
1431                     .arg("--scripts")
1432                     .arg(pkg.join(component))
1433                     .arg("--nopayload")
1434                     .arg(pkg.join(component).with_extension("pkg"));
1435                 builder.run(&mut cmd);
1436             };
1437
1438             let prepare = |name: &str| {
1439                 builder.create_dir(&pkg.join(name));
1440                 builder.cp_r(
1441                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1442                     &pkg.join(name),
1443                 );
1444                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1445                 pkgbuild(name);
1446             };
1447             prepare("rustc");
1448             prepare("cargo");
1449             prepare("rust-std");
1450             prepare("rust-analysis");
1451             prepare("clippy");
1452             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1453                 if built_tools.contains(tool) {
1454                     prepare(tool);
1455                 }
1456             }
1457             // create an 'uninstall' package
1458             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1459             pkgbuild("uninstall");
1460
1461             builder.create_dir(&pkg.join("res"));
1462             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1463             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1464             let mut cmd = Command::new("productbuild");
1465             cmd.arg("--distribution")
1466                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1467                 .arg("--resources")
1468                 .arg(pkg.join("res"))
1469                 .arg(distdir(builder).join(format!(
1470                     "{}-{}.pkg",
1471                     pkgname(builder, "rust"),
1472                     target.triple
1473                 )))
1474                 .arg("--package-path")
1475                 .arg(&pkg);
1476             let _time = timeit(builder);
1477             builder.run(&mut cmd);
1478         }
1479
1480         if target.contains("windows") {
1481             let exe = tmp.join("exe");
1482             let _ = fs::remove_dir_all(&exe);
1483
1484             let prepare = |name: &str| {
1485                 builder.create_dir(&exe.join(name));
1486                 let dir = if name == "rust-std" || name == "rust-analysis" {
1487                     format!("{}-{}", name, target.triple)
1488                 } else if name == "rls" {
1489                     "rls-preview".to_string()
1490                 } else if name == "rust-analyzer" {
1491                     "rust-analyzer-preview".to_string()
1492                 } else if name == "clippy" {
1493                     "clippy-preview".to_string()
1494                 } else if name == "rust-demangler" {
1495                     "rust-demangler-preview".to_string()
1496                 } else if name == "miri" {
1497                     "miri-preview".to_string()
1498                 } else {
1499                     name.to_string()
1500                 };
1501                 builder.cp_r(
1502                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1503                     &exe.join(name),
1504                 );
1505                 builder.remove(&exe.join(name).join("manifest.in"));
1506             };
1507             prepare("rustc");
1508             prepare("cargo");
1509             prepare("rust-analysis");
1510             prepare("rust-docs");
1511             prepare("rust-std");
1512             prepare("clippy");
1513             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1514                 if built_tools.contains(tool) {
1515                     prepare(tool);
1516                 }
1517             }
1518             if target.ends_with("windows-gnu") {
1519                 prepare("rust-mingw");
1520             }
1521
1522             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1523
1524             // Generate msi installer
1525             let wix_path = env::var_os("WIX")
1526                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1527             let wix = PathBuf::from(wix_path);
1528             let heat = wix.join("bin/heat.exe");
1529             let candle = wix.join("bin/candle.exe");
1530             let light = wix.join("bin/light.exe");
1531
1532             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1533             builder.run(
1534                 Command::new(&heat)
1535                     .current_dir(&exe)
1536                     .arg("dir")
1537                     .arg("rustc")
1538                     .args(&heat_flags)
1539                     .arg("-cg")
1540                     .arg("RustcGroup")
1541                     .arg("-dr")
1542                     .arg("Rustc")
1543                     .arg("-var")
1544                     .arg("var.RustcDir")
1545                     .arg("-out")
1546                     .arg(exe.join("RustcGroup.wxs")),
1547             );
1548             builder.run(
1549                 Command::new(&heat)
1550                     .current_dir(&exe)
1551                     .arg("dir")
1552                     .arg("rust-docs")
1553                     .args(&heat_flags)
1554                     .arg("-cg")
1555                     .arg("DocsGroup")
1556                     .arg("-dr")
1557                     .arg("Docs")
1558                     .arg("-var")
1559                     .arg("var.DocsDir")
1560                     .arg("-out")
1561                     .arg(exe.join("DocsGroup.wxs"))
1562                     .arg("-t")
1563                     .arg(etc.join("msi/squash-components.xsl")),
1564             );
1565             builder.run(
1566                 Command::new(&heat)
1567                     .current_dir(&exe)
1568                     .arg("dir")
1569                     .arg("cargo")
1570                     .args(&heat_flags)
1571                     .arg("-cg")
1572                     .arg("CargoGroup")
1573                     .arg("-dr")
1574                     .arg("Cargo")
1575                     .arg("-var")
1576                     .arg("var.CargoDir")
1577                     .arg("-out")
1578                     .arg(exe.join("CargoGroup.wxs"))
1579                     .arg("-t")
1580                     .arg(etc.join("msi/remove-duplicates.xsl")),
1581             );
1582             builder.run(
1583                 Command::new(&heat)
1584                     .current_dir(&exe)
1585                     .arg("dir")
1586                     .arg("rust-std")
1587                     .args(&heat_flags)
1588                     .arg("-cg")
1589                     .arg("StdGroup")
1590                     .arg("-dr")
1591                     .arg("Std")
1592                     .arg("-var")
1593                     .arg("var.StdDir")
1594                     .arg("-out")
1595                     .arg(exe.join("StdGroup.wxs")),
1596             );
1597             if built_tools.contains("rls") {
1598                 builder.run(
1599                     Command::new(&heat)
1600                         .current_dir(&exe)
1601                         .arg("dir")
1602                         .arg("rls")
1603                         .args(&heat_flags)
1604                         .arg("-cg")
1605                         .arg("RlsGroup")
1606                         .arg("-dr")
1607                         .arg("Rls")
1608                         .arg("-var")
1609                         .arg("var.RlsDir")
1610                         .arg("-out")
1611                         .arg(exe.join("RlsGroup.wxs"))
1612                         .arg("-t")
1613                         .arg(etc.join("msi/remove-duplicates.xsl")),
1614                 );
1615             }
1616             if built_tools.contains("rust-analyzer") {
1617                 builder.run(
1618                     Command::new(&heat)
1619                         .current_dir(&exe)
1620                         .arg("dir")
1621                         .arg("rust-analyzer")
1622                         .args(&heat_flags)
1623                         .arg("-cg")
1624                         .arg("RustAnalyzerGroup")
1625                         .arg("-dr")
1626                         .arg("RustAnalyzer")
1627                         .arg("-var")
1628                         .arg("var.RustAnalyzerDir")
1629                         .arg("-out")
1630                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1631                         .arg("-t")
1632                         .arg(etc.join("msi/remove-duplicates.xsl")),
1633                 );
1634             }
1635             builder.run(
1636                 Command::new(&heat)
1637                     .current_dir(&exe)
1638                     .arg("dir")
1639                     .arg("clippy")
1640                     .args(&heat_flags)
1641                     .arg("-cg")
1642                     .arg("ClippyGroup")
1643                     .arg("-dr")
1644                     .arg("Clippy")
1645                     .arg("-var")
1646                     .arg("var.ClippyDir")
1647                     .arg("-out")
1648                     .arg(exe.join("ClippyGroup.wxs"))
1649                     .arg("-t")
1650                     .arg(etc.join("msi/remove-duplicates.xsl")),
1651             );
1652             if built_tools.contains("rust-demangler") {
1653                 builder.run(
1654                     Command::new(&heat)
1655                         .current_dir(&exe)
1656                         .arg("dir")
1657                         .arg("rust-demangler")
1658                         .args(&heat_flags)
1659                         .arg("-cg")
1660                         .arg("RustDemanglerGroup")
1661                         .arg("-dr")
1662                         .arg("RustDemangler")
1663                         .arg("-var")
1664                         .arg("var.RustDemanglerDir")
1665                         .arg("-out")
1666                         .arg(exe.join("RustDemanglerGroup.wxs"))
1667                         .arg("-t")
1668                         .arg(etc.join("msi/remove-duplicates.xsl")),
1669                 );
1670             }
1671             if built_tools.contains("miri") {
1672                 builder.run(
1673                     Command::new(&heat)
1674                         .current_dir(&exe)
1675                         .arg("dir")
1676                         .arg("miri")
1677                         .args(&heat_flags)
1678                         .arg("-cg")
1679                         .arg("MiriGroup")
1680                         .arg("-dr")
1681                         .arg("Miri")
1682                         .arg("-var")
1683                         .arg("var.MiriDir")
1684                         .arg("-out")
1685                         .arg(exe.join("MiriGroup.wxs"))
1686                         .arg("-t")
1687                         .arg(etc.join("msi/remove-duplicates.xsl")),
1688                 );
1689             }
1690             builder.run(
1691                 Command::new(&heat)
1692                     .current_dir(&exe)
1693                     .arg("dir")
1694                     .arg("rust-analysis")
1695                     .args(&heat_flags)
1696                     .arg("-cg")
1697                     .arg("AnalysisGroup")
1698                     .arg("-dr")
1699                     .arg("Analysis")
1700                     .arg("-var")
1701                     .arg("var.AnalysisDir")
1702                     .arg("-out")
1703                     .arg(exe.join("AnalysisGroup.wxs"))
1704                     .arg("-t")
1705                     .arg(etc.join("msi/remove-duplicates.xsl")),
1706             );
1707             if target.ends_with("windows-gnu") {
1708                 builder.run(
1709                     Command::new(&heat)
1710                         .current_dir(&exe)
1711                         .arg("dir")
1712                         .arg("rust-mingw")
1713                         .args(&heat_flags)
1714                         .arg("-cg")
1715                         .arg("GccGroup")
1716                         .arg("-dr")
1717                         .arg("Gcc")
1718                         .arg("-var")
1719                         .arg("var.GccDir")
1720                         .arg("-out")
1721                         .arg(exe.join("GccGroup.wxs")),
1722                 );
1723             }
1724
1725             let candle = |input: &Path| {
1726                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1727                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1728                 let mut cmd = Command::new(&candle);
1729                 cmd.current_dir(&exe)
1730                     .arg("-nologo")
1731                     .arg("-dRustcDir=rustc")
1732                     .arg("-dDocsDir=rust-docs")
1733                     .arg("-dCargoDir=cargo")
1734                     .arg("-dStdDir=rust-std")
1735                     .arg("-dAnalysisDir=rust-analysis")
1736                     .arg("-dClippyDir=clippy")
1737                     .arg("-arch")
1738                     .arg(&arch)
1739                     .arg("-out")
1740                     .arg(&output)
1741                     .arg(&input);
1742                 add_env(builder, &mut cmd, target);
1743
1744                 if built_tools.contains("rust-demangler") {
1745                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1746                 }
1747                 if built_tools.contains("rls") {
1748                     cmd.arg("-dRlsDir=rls");
1749                 }
1750                 if built_tools.contains("rust-analyzer") {
1751                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1752                 }
1753                 if built_tools.contains("miri") {
1754                     cmd.arg("-dMiriDir=miri");
1755                 }
1756                 if target.ends_with("windows-gnu") {
1757                     cmd.arg("-dGccDir=rust-mingw");
1758                 }
1759                 builder.run(&mut cmd);
1760             };
1761             candle(&xform(&etc.join("msi/rust.wxs")));
1762             candle(&etc.join("msi/ui.wxs"));
1763             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1764             candle("RustcGroup.wxs".as_ref());
1765             candle("DocsGroup.wxs".as_ref());
1766             candle("CargoGroup.wxs".as_ref());
1767             candle("StdGroup.wxs".as_ref());
1768             candle("ClippyGroup.wxs".as_ref());
1769             if built_tools.contains("rust-demangler") {
1770                 candle("RustDemanglerGroup.wxs".as_ref());
1771             }
1772             if built_tools.contains("rls") {
1773                 candle("RlsGroup.wxs".as_ref());
1774             }
1775             if built_tools.contains("rust-analyzer") {
1776                 candle("RustAnalyzerGroup.wxs".as_ref());
1777             }
1778             if built_tools.contains("miri") {
1779                 candle("MiriGroup.wxs".as_ref());
1780             }
1781             candle("AnalysisGroup.wxs".as_ref());
1782
1783             if target.ends_with("windows-gnu") {
1784                 candle("GccGroup.wxs".as_ref());
1785             }
1786
1787             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1788             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1789             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1790
1791             builder.info(&format!("building `msi` installer with {:?}", light));
1792             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1793             let mut cmd = Command::new(&light);
1794             cmd.arg("-nologo")
1795                 .arg("-ext")
1796                 .arg("WixUIExtension")
1797                 .arg("-ext")
1798                 .arg("WixUtilExtension")
1799                 .arg("-out")
1800                 .arg(exe.join(&filename))
1801                 .arg("rust.wixobj")
1802                 .arg("ui.wixobj")
1803                 .arg("rustwelcomedlg.wixobj")
1804                 .arg("RustcGroup.wixobj")
1805                 .arg("DocsGroup.wixobj")
1806                 .arg("CargoGroup.wixobj")
1807                 .arg("StdGroup.wixobj")
1808                 .arg("AnalysisGroup.wixobj")
1809                 .arg("ClippyGroup.wixobj")
1810                 .current_dir(&exe);
1811
1812             if built_tools.contains("rls") {
1813                 cmd.arg("RlsGroup.wixobj");
1814             }
1815             if built_tools.contains("rust-analyzer") {
1816                 cmd.arg("RustAnalyzerGroup.wixobj");
1817             }
1818             if built_tools.contains("rust-demangler") {
1819                 cmd.arg("RustDemanglerGroup.wixobj");
1820             }
1821             if built_tools.contains("miri") {
1822                 cmd.arg("MiriGroup.wixobj");
1823             }
1824
1825             if target.ends_with("windows-gnu") {
1826                 cmd.arg("GccGroup.wixobj");
1827             }
1828             // ICE57 wrongly complains about the shortcuts
1829             cmd.arg("-sice:ICE57");
1830
1831             let _time = timeit(builder);
1832             builder.run(&mut cmd);
1833
1834             if !builder.config.dry_run {
1835                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1836             }
1837         }
1838     }
1839 }
1840
1841 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1842     let mut parts = builder.version.split('.');
1843     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1844         .env("CFG_RELEASE_NUM", &builder.version)
1845         .env("CFG_RELEASE", builder.rust_release())
1846         .env("CFG_VER_MAJOR", parts.next().unwrap())
1847         .env("CFG_VER_MINOR", parts.next().unwrap())
1848         .env("CFG_VER_PATCH", parts.next().unwrap())
1849         .env("CFG_VER_BUILD", "0") // just needed to build
1850         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1851         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1852         .env("CFG_BUILD", target.triple)
1853         .env("CFG_CHANNEL", &builder.config.channel);
1854
1855     if target.contains("windows-gnullvm") {
1856         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1857     } else if target.contains("windows-gnu") {
1858         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1859     } else {
1860         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1861     }
1862 }
1863
1864 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1865 ///
1866 /// Returns whether the files were actually copied.
1867 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1868     if let Some(config) = builder.config.target_config.get(&target) {
1869         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1870             // If the LLVM was externally provided, then we don't currently copy
1871             // artifacts into the sysroot. This is not necessarily the right
1872             // choice (in particular, it will require the LLVM dylib to be in
1873             // the linker's load path at runtime), but the common use case for
1874             // external LLVMs is distribution provided LLVMs, and in that case
1875             // they're usually in the standard search path (e.g., /usr/lib) and
1876             // copying them here is going to cause problems as we may end up
1877             // with the wrong files and isn't what distributions want.
1878             //
1879             // This behavior may be revisited in the future though.
1880             //
1881             // If the LLVM is coming from ourselves (just from CI) though, we
1882             // still want to install it, as it otherwise won't be available.
1883             return false;
1884         }
1885     }
1886
1887     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1888     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1889     // clear why this is the case, though. llvm-config will emit the versioned
1890     // paths and we don't want those in the sysroot (as we're expecting
1891     // unversioned paths).
1892     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1893         let src_libdir = builder.llvm_out(target).join("lib");
1894         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1895         if llvm_dylib_path.exists() {
1896             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1897         }
1898         !builder.config.dry_run
1899     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1900         let mut cmd = Command::new(llvm_config);
1901         cmd.arg("--libfiles");
1902         builder.verbose(&format!("running {:?}", cmd));
1903         let files = output(&mut cmd);
1904         let build_llvm_out = &builder.llvm_out(builder.config.build);
1905         let target_llvm_out = &builder.llvm_out(target);
1906         for file in files.trim_end().split(' ') {
1907             // If we're not using a custom LLVM, make sure we package for the target.
1908             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1909                 target_llvm_out.join(relative_path)
1910             } else {
1911                 PathBuf::from(file)
1912             };
1913             builder.install(&file, dst_libdir, 0o644);
1914         }
1915         !builder.config.dry_run
1916     } else {
1917         false
1918     }
1919 }
1920
1921 /// Maybe add libLLVM.so to the target lib-dir for linking.
1922 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1923     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1924     // We do not need to copy LLVM files into the sysroot if it is not
1925     // dynamically linked; it is already included into librustc_llvm
1926     // statically.
1927     if builder.llvm_link_shared() {
1928         maybe_install_llvm(builder, target, &dst_libdir);
1929     }
1930 }
1931
1932 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1933 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1934     let dst_libdir =
1935         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1936     // We do not need to copy LLVM files into the sysroot if it is not
1937     // dynamically linked; it is already included into librustc_llvm
1938     // statically.
1939     if builder.llvm_link_shared() {
1940         maybe_install_llvm(builder, target, &dst_libdir);
1941     }
1942 }
1943
1944 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1945 pub struct LlvmTools {
1946     pub target: TargetSelection,
1947 }
1948
1949 impl Step for LlvmTools {
1950     type Output = Option<GeneratedTarball>;
1951     const ONLY_HOSTS: bool = true;
1952     const DEFAULT: bool = true;
1953
1954     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1955         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1956         // FIXME: allow using the names of the tools themselves?
1957         run.alias("llvm-tools").default_condition(default)
1958     }
1959
1960     fn make_run(run: RunConfig<'_>) {
1961         run.builder.ensure(LlvmTools { target: run.target });
1962     }
1963
1964     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1965         let target = self.target;
1966
1967         /* run only if llvm-config isn't used */
1968         if let Some(config) = builder.config.target_config.get(&target) {
1969             if let Some(ref _s) = config.llvm_config {
1970                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1971                 return None;
1972             }
1973         }
1974
1975         builder.ensure(crate::native::Llvm { target });
1976
1977         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1978         tarball.set_overlay(OverlayKind::LLVM);
1979         tarball.is_preview(true);
1980
1981         // Prepare the image directory
1982         let src_bindir = builder.llvm_out(target).join("bin");
1983         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1984         for tool in LLVM_TOOLS {
1985             let exe = src_bindir.join(exe(tool, target));
1986             tarball.add_file(&exe, &dst_bindir, 0o755);
1987         }
1988
1989         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1990         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1991         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1992         // compiler libraries.
1993         maybe_install_llvm_target(builder, target, tarball.image_dir());
1994
1995         Some(tarball.generate())
1996     }
1997 }
1998
1999 // Tarball intended for internal consumption to ease rustc/std development.
2000 //
2001 // Should not be considered stable by end users.
2002 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2003 pub struct RustDev {
2004     pub target: TargetSelection,
2005 }
2006
2007 impl Step for RustDev {
2008     type Output = Option<GeneratedTarball>;
2009     const DEFAULT: bool = true;
2010     const ONLY_HOSTS: bool = true;
2011
2012     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2013         run.alias("rust-dev")
2014     }
2015
2016     fn make_run(run: RunConfig<'_>) {
2017         run.builder.ensure(RustDev { target: run.target });
2018     }
2019
2020     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2021         let target = self.target;
2022
2023         /* run only if llvm-config isn't used */
2024         if let Some(config) = builder.config.target_config.get(&target) {
2025             if let Some(ref _s) = config.llvm_config {
2026                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2027                 return None;
2028             }
2029         }
2030
2031         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2032         tarball.set_overlay(OverlayKind::LLVM);
2033
2034         let src_bindir = builder.llvm_out(target).join("bin");
2035         // If updating this list, you likely want to change
2036         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2037         // will not pick up the extra file until LLVM gets bumped.
2038         for bin in &[
2039             "llvm-config",
2040             "llvm-ar",
2041             "llvm-objdump",
2042             "llvm-profdata",
2043             "llvm-bcanalyzer",
2044             "llvm-cov",
2045             "llvm-dwp",
2046             "llvm-nm",
2047             "llvm-dwarfdump",
2048         ] {
2049             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2050         }
2051
2052         // We don't build LLD on some platforms, so only add it if it exists
2053         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2054         if lld_path.exists() {
2055             tarball.add_file(lld_path, "bin", 0o755);
2056         }
2057
2058         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2059
2060         // Copy the include directory as well; needed mostly to build
2061         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2062         // just broadly useful to be able to link against the bundled LLVM.
2063         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2064
2065         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2066         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2067         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2068         // compiler libraries.
2069         let dst_libdir = tarball.image_dir().join("lib");
2070         maybe_install_llvm(builder, target, &dst_libdir);
2071         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2072         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2073
2074         Some(tarball.generate())
2075     }
2076 }
2077
2078 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2079 /// release process to avoid cloning the monorepo and building stuff.
2080 ///
2081 /// Should not be considered stable by end users.
2082 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2083 pub struct BuildManifest {
2084     pub target: TargetSelection,
2085 }
2086
2087 impl Step for BuildManifest {
2088     type Output = GeneratedTarball;
2089     const DEFAULT: bool = false;
2090     const ONLY_HOSTS: bool = true;
2091
2092     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2093         run.alias("build-manifest")
2094     }
2095
2096     fn make_run(run: RunConfig<'_>) {
2097         run.builder.ensure(BuildManifest { target: run.target });
2098     }
2099
2100     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2101         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2102
2103         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2104         tarball.add_file(&build_manifest, "bin", 0o755);
2105         tarball.generate()
2106     }
2107 }
2108
2109 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2110 ///
2111 /// Currently this is the PGO profile data.
2112 ///
2113 /// Should not be considered stable by end users.
2114 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2115 pub struct ReproducibleArtifacts {
2116     pub target: TargetSelection,
2117 }
2118
2119 impl Step for ReproducibleArtifacts {
2120     type Output = Option<GeneratedTarball>;
2121     const DEFAULT: bool = true;
2122     const ONLY_HOSTS: bool = true;
2123
2124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2125         run.alias("reproducible-artifacts")
2126     }
2127
2128     fn make_run(run: RunConfig<'_>) {
2129         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2130     }
2131
2132     fn run(self, builder: &Builder<'_>) -> Self::Output {
2133         let mut added_anything = false;
2134         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2135         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2136             tarball.add_file(path, ".", 0o644);
2137             added_anything = true;
2138         }
2139         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2140             tarball.add_file(path, ".", 0o644);
2141             added_anything = true;
2142         }
2143         if added_anything { Some(tarball.generate()) } else { None }
2144     }
2145 }