src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.alias("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.alias("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.ends_with("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.alias("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.ends_with("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 let exe_name = exe("ld", compiler.host);
 411                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 412             }
 413
 414             // Man pages
 415             t!(fs::create_dir_all(image.join("share/man/man1")));
 416             let man_src = builder.src.join("src/doc/man");
 417             let man_dst = image.join("share/man/man1");
 418
 419             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 420             // to hardlink, and we don't want to edit the source templates
 421             for file_entry in builder.read_dir(&man_src) {
 422                 let page_src = file_entry.path();
 423                 let page_dst = man_dst.join(file_entry.file_name());
 424                 let src_text = t!(std::fs::read_to_string(&page_src));
 425                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 426                 t!(std::fs::write(&page_dst, &new_text));
 427                 t!(fs::copy(&page_src, &page_dst));
 428             }
 429
 430             // Debugger scripts
 431             builder
 432                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 433
 434             // Misc license info
 435             let cp = |file: &str| {
 436                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 437             };
 438             cp("COPYRIGHT");
 439             cp("LICENSE-APACHE");
 440             cp("LICENSE-MIT");
 441             cp("README.md");
 442         }
 443     }
 444 }
 445
 446 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 447 pub struct DebuggerScripts {
 448     pub sysroot: Interned<PathBuf>,
 449     pub host: TargetSelection,
 450 }
 451
 452 impl Step for DebuggerScripts {
 453     type Output = ();
 454
 455     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 456         run.never()
 457     }
 458
 459     /// Copies debugger scripts for `target` into the `sysroot` specified.
 460     fn run(self, builder: &Builder<'_>) {
 461         let host = self.host;
 462         let sysroot = self.sysroot;
 463         let dst = sysroot.join("lib/rustlib/etc");
 464         t!(fs::create_dir_all(&dst));
 465         let cp_debugger_script = |file: &str| {
 466             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 467         };
 468         if host.contains("windows-msvc") {
 469             // windbg debugger scripts
 470             builder.install(
 471                 &builder.src.join("src/etc/rust-windbg.cmd"),
 472                 &sysroot.join("bin"),
 473                 0o755,
 474             );
 475
 476             cp_debugger_script("natvis/intrinsic.natvis");
 477             cp_debugger_script("natvis/liballoc.natvis");
 478             cp_debugger_script("natvis/libcore.natvis");
 479             cp_debugger_script("natvis/libstd.natvis");
 480         } else {
 481             cp_debugger_script("rust_types.py");
 482
 483             // gdb debugger scripts
 484             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 485             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 486
 487             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 488             cp_debugger_script("gdb_lookup.py");
 489             cp_debugger_script("gdb_providers.py");
 490
 491             // lldb debugger scripts
 492             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 493
 494             cp_debugger_script("lldb_lookup.py");
 495             cp_debugger_script("lldb_providers.py");
 496             cp_debugger_script("lldb_commands")
 497         }
 498     }
 499 }
 500
 501 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 502     // The only true set of target libraries came from the build triple, so
 503     // let's reduce redundant work by only producing archives from that host.
 504     if compiler.host != builder.config.build {
 505         builder.info("\tskipping, not a build host");
 506         true
 507     } else {
 508         false
 509     }
 510 }
 511
 512 /// Copy stamped files into an image's `target/lib` directory.
 513 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 514     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 515     let self_contained_dst = dst.join("self-contained");
 516     t!(fs::create_dir_all(&dst));
 517     t!(fs::create_dir_all(&self_contained_dst));
 518     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 519         if dependency_type == DependencyType::TargetSelfContained {
 520             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 521         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 522             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 523         }
 524     }
 525 }
 526
 527 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 528 pub struct Std {
 529     pub compiler: Compiler,
 530     pub target: TargetSelection,
 531 }
 532
 533 impl Step for Std {
 534     type Output = Option<GeneratedTarball>;
 535     const DEFAULT: bool = true;
 536
 537     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 538         run.alias("rust-std")
 539     }
 540
 541     fn make_run(run: RunConfig<'_>) {
 542         run.builder.ensure(Std {
 543             compiler: run.builder.compiler_for(
 544                 run.builder.top_stage,
 545                 run.builder.config.build,
 546                 run.target,
 547             ),
 548             target: run.target,
 549         });
 550     }
 551
 552     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 553         let compiler = self.compiler;
 554         let target = self.target;
 555
 556         if skip_host_target_lib(builder, compiler) {
 557             return None;
 558         }
 559
 560         builder.ensure(compile::Std { compiler, target });
 561
 562         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 563         tarball.include_target_in_component_name(true);
 564
 565         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 566         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 567         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 568
 569         Some(tarball.generate())
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct RustcDev {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for RustcDev {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582     const ONLY_HOSTS: bool = true;
 583
 584     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 585         run.alias("rustc-dev")
 586     }
 587
 588     fn make_run(run: RunConfig<'_>) {
 589         run.builder.ensure(RustcDev {
 590             compiler: run.builder.compiler_for(
 591                 run.builder.top_stage,
 592                 run.builder.config.build,
 593                 run.target,
 594             ),
 595             target: run.target,
 596         });
 597     }
 598
 599     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 600         let compiler = self.compiler;
 601         let target = self.target;
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Rustc { compiler, target });
 607
 608         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 609
 610         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 611         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 612         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 613
 614         let src_files = &["Cargo.lock"];
 615         // This is the reduced set of paths which will become the rustc-dev component
 616         // (essentially the compiler crates and all of their path dependencies).
 617         copy_src_dirs(
 618             builder,
 619             &builder.src,
 620             &["compiler"],
 621             &[],
 622             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 623         );
 624         for file in src_files {
 625             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 626         }
 627
 628         Some(tarball.generate())
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: TargetSelection,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = Option<GeneratedTarball>;
 640     const DEFAULT: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         let default = should_build_extended_tool(&run.builder, "analysis");
 644         run.alias("rust-analysis").default_condition(default)
 645     }
 646
 647     fn make_run(run: RunConfig<'_>) {
 648         run.builder.ensure(Analysis {
 649             // Find the actual compiler (handling the full bootstrap option) which
 650             // produced the save-analysis data because that data isn't copied
 651             // through the sysroot uplifting.
 652             compiler: run.builder.compiler_for(
 653                 run.builder.top_stage,
 654                 run.builder.config.build,
 655                 run.target,
 656             ),
 657             target: run.target,
 658         });
 659     }
 660
 661     /// Creates a tarball of save-analysis metadata, if available.
 662     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 663         let compiler = self.compiler;
 664         let target = self.target;
 665         if compiler.host != builder.config.build {
 666             return None;
 667         }
 668
 669         builder.ensure(compile::Std { compiler, target });
 670         let src = builder
 671             .stage_out(compiler, Mode::Std)
 672             .join(target.triple)
 673             .join(builder.cargo_dir())
 674             .join("deps")
 675             .join("save-analysis");
 676
 677         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 678         tarball.include_target_in_component_name(true);
 679         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 680         Some(tarball.generate())
 681     }
 682 }
 683
 684 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 685 /// `dst_dir`.
 686 fn copy_src_dirs(
 687     builder: &Builder<'_>,
 688     base: &Path,
 689     src_dirs: &[&str],
 690     exclude_dirs: &[&str],
 691     dst_dir: &Path,
 692 ) {
 693     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 694         let spath = match path.to_str() {
 695             Some(path) => path,
 696             None => return false,
 697         };
 698         if spath.ends_with('~') || spath.ends_with(".pyc") {
 699             return false;
 700         }
 701
 702         const LLVM_PROJECTS: &[&str] = &[
 703             "llvm-project/clang",
 704             "llvm-project\\clang",
 705             "llvm-project/libunwind",
 706             "llvm-project\\libunwind",
 707             "llvm-project/lld",
 708             "llvm-project\\lld",
 709             "llvm-project/lldb",
 710             "llvm-project\\lldb",
 711             "llvm-project/llvm",
 712             "llvm-project\\llvm",
 713             "llvm-project/compiler-rt",
 714             "llvm-project\\compiler-rt",
 715             "llvm-project/cmake",
 716             "llvm-project\\cmake",
 717         ];
 718         if spath.contains("llvm-project")
 719             && !spath.ends_with("llvm-project")
 720             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 721         {
 722             return false;
 723         }
 724
 725         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 726         if LLVM_TEST.iter().any(|path| spath.contains(path))
 727             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 728         {
 729             return false;
 730         }
 731
 732         let full_path = Path::new(dir).join(path);
 733         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 734             return false;
 735         }
 736
 737         let excludes = [
 738             "CVS",
 739             "RCS",
 740             "SCCS",
 741             ".git",
 742             ".gitignore",
 743             ".gitmodules",
 744             ".gitattributes",
 745             ".cvsignore",
 746             ".svn",
 747             ".arch-ids",
 748             "{arch}",
 749             "=RELEASE-ID",
 750             "=meta-update",
 751             "=update",
 752             ".bzr",
 753             ".bzrignore",
 754             ".bzrtags",
 755             ".hg",
 756             ".hgignore",
 757             ".hgrags",
 758             "_darcs",
 759         ];
 760         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 761     }
 762
 763     // Copy the directories using our filter
 764     for item in src_dirs {
 765         let dst = &dst_dir.join(item);
 766         t!(fs::create_dir_all(dst));
 767         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 768     }
 769 }
 770
 771 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 772 pub struct Src;
 773
 774 impl Step for Src {
 775     /// The output path of the src installer tarball
 776     type Output = GeneratedTarball;
 777     const DEFAULT: bool = true;
 778     const ONLY_HOSTS: bool = true;
 779
 780     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 781         run.alias("rust-src")
 782     }
 783
 784     fn make_run(run: RunConfig<'_>) {
 785         run.builder.ensure(Src);
 786     }
 787
 788     /// Creates the `rust-src` installer component
 789     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 790         let tarball = Tarball::new_targetless(builder, "rust-src");
 791
 792         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 793         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 794         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 795         // and fix them...
 796         //
 797         // NOTE: if you update the paths here, you also should update the "virtual" path
 798         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 799         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 800
 801         let src_files = ["Cargo.lock"];
 802         // This is the reduced set of paths which will become the rust-src component
 803         // (essentially libstd and all of its path dependencies).
 804         copy_src_dirs(
 805             builder,
 806             &builder.src,
 807             &["library", "src/llvm-project/libunwind"],
 808             &[
 809                 // not needed and contains symlinks which rustup currently
 810                 // chokes on when unpacking.
 811                 "library/backtrace/crates",
 812                 // these are 30MB combined and aren't necessary for building
 813                 // the standard library.
 814                 "library/stdarch/Cargo.toml",
 815                 "library/stdarch/crates/stdarch-verify",
 816                 "library/stdarch/crates/intrinsic-test",
 817             ],
 818             &dst_src,
 819         );
 820         for file in src_files.iter() {
 821             builder.copy(&builder.src.join(file), &dst_src.join(file));
 822         }
 823
 824         tarball.generate()
 825     }
 826 }
 827
 828 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 829 pub struct PlainSourceTarball;
 830
 831 impl Step for PlainSourceTarball {
 832     /// Produces the location of the tarball generated
 833     type Output = GeneratedTarball;
 834     const DEFAULT: bool = true;
 835     const ONLY_HOSTS: bool = true;
 836
 837     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 838         let builder = run.builder;
 839         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig<'_>) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 848         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 849         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 850         // It also contains symbolic links, unlike other any other dist tarball.
 851         // It's used for distros building rustc from source in a pre-vendored environment.
 852         let mut tarball = Tarball::new(builder, "rustc", "src");
 853         tarball.permit_symlinks(true);
 854         let plain_dst_src = tarball.image_dir();
 855
 856         // This is the set of root paths which will become part of the source package
 857         let src_files = [
 858             "COPYRIGHT",
 859             "LICENSE-APACHE",
 860             "LICENSE-MIT",
 861             "CONTRIBUTING.md",
 862             "README.md",
 863             "RELEASES.md",
 864             "configure",
 865             "x.py",
 866             "config.toml.example",
 867             "Cargo.toml",
 868             "Cargo.lock",
 869         ];
 870         let src_dirs = ["src", "compiler", "library"];
 871
 872         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 873
 874         // Copy the files normally
 875         for item in &src_files {
 876             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 877         }
 878
 879         // Create the version file
 880         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 881         if let Some(sha) = builder.rust_sha() {
 882             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 883         }
 884
 885         // If we're building from git sources, we need to vendor a complete distribution.
 886         if builder.rust_info.is_git() {
 887             // Ensure we have the submodules checked out.
 888             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 889
 890             // Vendor all Cargo dependencies
 891             let mut cmd = Command::new(&builder.initial_cargo);
 892             cmd.arg("vendor")
 893                 .arg("--sync")
 894                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 895                 .arg("--sync")
 896                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 897                 .arg("--sync")
 898                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 899                 .current_dir(&plain_dst_src);
 900
 901             let config = if !builder.config.dry_run {
 902                 t!(String::from_utf8(t!(cmd.output()).stdout))
 903             } else {
 904                 String::new()
 905             };
 906
 907             let cargo_config_dir = plain_dst_src.join(".cargo");
 908             builder.create_dir(&cargo_config_dir);
 909             builder.create(&cargo_config_dir.join("config.toml"), &config);
 910         }
 911
 912         tarball.bare()
 913     }
 914 }
 915
 916 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 917 pub struct Cargo {
 918     pub compiler: Compiler,
 919     pub target: TargetSelection,
 920 }
 921
 922 impl Step for Cargo {
 923     type Output = Option<GeneratedTarball>;
 924     const DEFAULT: bool = true;
 925     const ONLY_HOSTS: bool = true;
 926
 927     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 928         let default = should_build_extended_tool(&run.builder, "cargo");
 929         run.alias("cargo").default_condition(default)
 930     }
 931
 932     fn make_run(run: RunConfig<'_>) {
 933         run.builder.ensure(Cargo {
 934             compiler: run.builder.compiler_for(
 935                 run.builder.top_stage,
 936                 run.builder.config.build,
 937                 run.target,
 938             ),
 939             target: run.target,
 940         });
 941     }
 942
 943     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 944         let compiler = self.compiler;
 945         let target = self.target;
 946
 947         let cargo = builder.ensure(tool::Cargo { compiler, target });
 948         let src = builder.src.join("src/tools/cargo");
 949         let etc = src.join("src/etc");
 950
 951         // Prepare the image directory
 952         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 953         tarball.set_overlay(OverlayKind::Cargo);
 954
 955         tarball.add_file(&cargo, "bin", 0o755);
 956         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 957         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 958         tarball.add_dir(etc.join("man"), "share/man/man1");
 959         tarball.add_legal_and_readme_to("share/doc/cargo");
 960
 961         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 962             let dirent = dirent.expect("read dir entry");
 963             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 964                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 965             }
 966         }
 967
 968         Some(tarball.generate())
 969     }
 970 }
 971
 972 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 973 pub struct Rls {
 974     pub compiler: Compiler,
 975     pub target: TargetSelection,
 976 }
 977
 978 impl Step for Rls {
 979     type Output = Option<GeneratedTarball>;
 980     const ONLY_HOSTS: bool = true;
 981     const DEFAULT: bool = true;
 982
 983     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 984         let default = should_build_extended_tool(&run.builder, "rls");
 985         run.alias("rls").default_condition(default)
 986     }
 987
 988     fn make_run(run: RunConfig<'_>) {
 989         run.builder.ensure(Rls {
 990             compiler: run.builder.compiler_for(
 991                 run.builder.top_stage,
 992                 run.builder.config.build,
 993                 run.target,
 994             ),
 995             target: run.target,
 996         });
 997     }
 998
 999     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1000         let compiler = self.compiler;
1001         let target = self.target;
1002
1003         let rls = builder
1004             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1005             .or_else(|| {
1006                 missing_tool("RLS", builder.build.config.missing_tools);
1007                 None
1008             })?;
1009
1010         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1011         tarball.set_overlay(OverlayKind::RLS);
1012         tarball.is_preview(true);
1013         tarball.add_file(rls, "bin", 0o755);
1014         tarball.add_legal_and_readme_to("share/doc/rls");
1015         Some(tarball.generate())
1016     }
1017 }
1018
1019 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1020 pub struct RustAnalyzer {
1021     pub compiler: Compiler,
1022     pub target: TargetSelection,
1023 }
1024
1025 impl Step for RustAnalyzer {
1026     type Output = Option<GeneratedTarball>;
1027     const DEFAULT: bool = true;
1028     const ONLY_HOSTS: bool = true;
1029
1030     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1031         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1032         run.alias("rust-analyzer").default_condition(default)
1033     }
1034
1035     fn make_run(run: RunConfig<'_>) {
1036         run.builder.ensure(RustAnalyzer {
1037             compiler: run.builder.compiler_for(
1038                 run.builder.top_stage,
1039                 run.builder.config.build,
1040                 run.target,
1041             ),
1042             target: run.target,
1043         });
1044     }
1045
1046     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1047         let compiler = self.compiler;
1048         let target = self.target;
1049
1050         if target.contains("riscv64") {
1051             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1052             // to build. See #74813 for details.
1053             return None;
1054         }
1055
1056         let rust_analyzer = builder
1057             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1058             .expect("rust-analyzer always builds");
1059
1060         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1061         tarball.set_overlay(OverlayKind::RustAnalyzer);
1062         tarball.is_preview(true);
1063         tarball.add_file(rust_analyzer, "bin", 0o755);
1064         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1065         Some(tarball.generate())
1066     }
1067 }
1068
1069 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1070 pub struct Clippy {
1071     pub compiler: Compiler,
1072     pub target: TargetSelection,
1073 }
1074
1075 impl Step for Clippy {
1076     type Output = Option<GeneratedTarball>;
1077     const DEFAULT: bool = true;
1078     const ONLY_HOSTS: bool = true;
1079
1080     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1081         let default = should_build_extended_tool(&run.builder, "clippy");
1082         run.alias("clippy").default_condition(default)
1083     }
1084
1085     fn make_run(run: RunConfig<'_>) {
1086         run.builder.ensure(Clippy {
1087             compiler: run.builder.compiler_for(
1088                 run.builder.top_stage,
1089                 run.builder.config.build,
1090                 run.target,
1091             ),
1092             target: run.target,
1093         });
1094     }
1095
1096     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1097         let compiler = self.compiler;
1098         let target = self.target;
1099
1100         // Prepare the image directory
1101         // We expect clippy to build, because we've exited this step above if tool
1102         // state for clippy isn't testing.
1103         let clippy = builder
1104             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1105             .expect("clippy expected to build - essential tool");
1106         let cargoclippy = builder
1107             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1108             .expect("clippy expected to build - essential tool");
1109
1110         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1111         tarball.set_overlay(OverlayKind::Clippy);
1112         tarball.is_preview(true);
1113         tarball.add_file(clippy, "bin", 0o755);
1114         tarball.add_file(cargoclippy, "bin", 0o755);
1115         tarball.add_legal_and_readme_to("share/doc/clippy");
1116         Some(tarball.generate())
1117     }
1118 }
1119
1120 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1121 pub struct Miri {
1122     pub compiler: Compiler,
1123     pub target: TargetSelection,
1124 }
1125
1126 impl Step for Miri {
1127     type Output = Option<GeneratedTarball>;
1128     const DEFAULT: bool = true;
1129     const ONLY_HOSTS: bool = true;
1130
1131     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1132         let default = should_build_extended_tool(&run.builder, "miri");
1133         run.alias("miri").default_condition(default)
1134     }
1135
1136     fn make_run(run: RunConfig<'_>) {
1137         run.builder.ensure(Miri {
1138             compiler: run.builder.compiler_for(
1139                 run.builder.top_stage,
1140                 run.builder.config.build,
1141                 run.target,
1142             ),
1143             target: run.target,
1144         });
1145     }
1146
1147     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1148         // This prevents miri from being built for "dist" or "install"
1149         // on the stable/beta channels. It is a nightly-only tool and should
1150         // not be included.
1151         if !builder.build.unstable_features() {
1152             return None;
1153         }
1154         let compiler = self.compiler;
1155         let target = self.target;
1156
1157         let miri = builder
1158             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1159             .or_else(|| {
1160                 missing_tool("miri", builder.build.config.missing_tools);
1161                 None
1162             })?;
1163         let cargomiri = builder
1164             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1165             .or_else(|| {
1166                 missing_tool("cargo miri", builder.build.config.missing_tools);
1167                 None
1168             })?;
1169
1170         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1171         tarball.set_overlay(OverlayKind::Miri);
1172         tarball.is_preview(true);
1173         tarball.add_file(miri, "bin", 0o755);
1174         tarball.add_file(cargomiri, "bin", 0o755);
1175         tarball.add_legal_and_readme_to("share/doc/miri");
1176         Some(tarball.generate())
1177     }
1178 }
1179
1180 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1181 pub struct Rustfmt {
1182     pub compiler: Compiler,
1183     pub target: TargetSelection,
1184 }
1185
1186 impl Step for Rustfmt {
1187     type Output = Option<GeneratedTarball>;
1188     const DEFAULT: bool = true;
1189     const ONLY_HOSTS: bool = true;
1190
1191     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1192         let default = should_build_extended_tool(&run.builder, "rustfmt");
1193         run.alias("rustfmt").default_condition(default)
1194     }
1195
1196     fn make_run(run: RunConfig<'_>) {
1197         run.builder.ensure(Rustfmt {
1198             compiler: run.builder.compiler_for(
1199                 run.builder.top_stage,
1200                 run.builder.config.build,
1201                 run.target,
1202             ),
1203             target: run.target,
1204         });
1205     }
1206
1207     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1208         let compiler = self.compiler;
1209         let target = self.target;
1210
1211         let rustfmt = builder
1212             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1213             .or_else(|| {
1214                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1215                 None
1216             })?;
1217         let cargofmt = builder
1218             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1219             .or_else(|| {
1220                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1221                 None
1222             })?;
1223
1224         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1225         tarball.set_overlay(OverlayKind::Rustfmt);
1226         tarball.is_preview(true);
1227         tarball.add_file(rustfmt, "bin", 0o755);
1228         tarball.add_file(cargofmt, "bin", 0o755);
1229         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1230         Some(tarball.generate())
1231     }
1232 }
1233
1234 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1235 pub struct RustDemangler {
1236     pub compiler: Compiler,
1237     pub target: TargetSelection,
1238 }
1239
1240 impl Step for RustDemangler {
1241     type Output = Option<GeneratedTarball>;
1242     const DEFAULT: bool = true;
1243     const ONLY_HOSTS: bool = true;
1244
1245     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1246         // While other tools use `should_build_extended_tool` to decide whether to be run by
1247         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1248         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1249         // we run the step by default when only `extended = true`, and decide whether to actually
1250         // run it or not later.
1251         let default = run.builder.config.extended;
1252         run.alias("rust-demangler").default_condition(default)
1253     }
1254
1255     fn make_run(run: RunConfig<'_>) {
1256         run.builder.ensure(RustDemangler {
1257             compiler: run.builder.compiler_for(
1258                 run.builder.top_stage,
1259                 run.builder.config.build,
1260                 run.target,
1261             ),
1262             target: run.target,
1263         });
1264     }
1265
1266     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1267         let compiler = self.compiler;
1268         let target = self.target;
1269
1270         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1271         let condition = should_build_extended_tool(builder, "rust-demangler")
1272             || builder.config.profiler_enabled(target);
1273         if builder.config.extended && !condition {
1274             return None;
1275         }
1276
1277         let rust_demangler = builder
1278             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1279             .expect("rust-demangler expected to build - in-tree tool");
1280
1281         // Prepare the image directory
1282         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1283         tarball.set_overlay(OverlayKind::RustDemangler);
1284         tarball.is_preview(true);
1285         tarball.add_file(&rust_demangler, "bin", 0o755);
1286         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1287         Some(tarball.generate())
1288     }
1289 }
1290
1291 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1292 pub struct Extended {
1293     stage: u32,
1294     host: TargetSelection,
1295     target: TargetSelection,
1296 }
1297
1298 impl Step for Extended {
1299     type Output = ();
1300     const DEFAULT: bool = true;
1301     const ONLY_HOSTS: bool = true;
1302
1303     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1304         let builder = run.builder;
1305         run.alias("extended").default_condition(builder.config.extended)
1306     }
1307
1308     fn make_run(run: RunConfig<'_>) {
1309         run.builder.ensure(Extended {
1310             stage: run.builder.top_stage,
1311             host: run.builder.config.build,
1312             target: run.target,
1313         });
1314     }
1315
1316     /// Creates a combined installer for the specified target in the provided stage.
1317     fn run(self, builder: &Builder<'_>) {
1318         let target = self.target;
1319         let stage = self.stage;
1320         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1321
1322         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1323
1324         let mut tarballs = Vec::new();
1325         let mut built_tools = HashSet::new();
1326         macro_rules! add_component {
1327             ($name:expr => $step:expr) => {
1328                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1329                     tarballs.push(tarball);
1330                     built_tools.insert($name);
1331                 }
1332             };
1333         }
1334
1335         // When rust-std package split from rustc, we needed to ensure that during
1336         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1337         // the std files during uninstall. To do this ensure that rustc comes
1338         // before rust-std in the list below.
1339         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1340         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1341
1342         if target.ends_with("windows-gnu") {
1343             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1344         }
1345
1346         add_component!("rust-docs" => Docs { host: target });
1347         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1348         add_component!("cargo" => Cargo { compiler, target });
1349         add_component!("rustfmt" => Rustfmt { compiler, target });
1350         add_component!("rls" => Rls { compiler, target });
1351         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1352         add_component!("llvm-components" => LlvmTools { target });
1353         add_component!("clippy" => Clippy { compiler, target });
1354         add_component!("miri" => Miri { compiler, target });
1355         add_component!("analysis" => Analysis { compiler, target });
1356
1357         let etc = builder.src.join("src/etc/installer");
1358
1359         // Avoid producing tarballs during a dry run.
1360         if builder.config.dry_run {
1361             return;
1362         }
1363
1364         let tarball = Tarball::new(builder, "rust", &target.triple);
1365         let generated = tarball.combine(&tarballs);
1366
1367         let tmp = tmpdir(builder).join("combined-tarball");
1368         let work = generated.work_dir();
1369
1370         let mut license = String::new();
1371         license += &builder.read(&builder.src.join("COPYRIGHT"));
1372         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1373         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1374         license.push('\n');
1375         license.push('\n');
1376
1377         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1378         let mut rtf = rtf.to_string();
1379         rtf.push('\n');
1380         for line in license.lines() {
1381             rtf.push_str(line);
1382             rtf.push_str("\\line ");
1383         }
1384         rtf.push('}');
1385
1386         fn filter(contents: &str, marker: &str) -> String {
1387             let start = format!("tool-{}-start", marker);
1388             let end = format!("tool-{}-end", marker);
1389             let mut lines = Vec::new();
1390             let mut omitted = false;
1391             for line in contents.lines() {
1392                 if line.contains(&start) {
1393                     omitted = true;
1394                 } else if line.contains(&end) {
1395                     omitted = false;
1396                 } else if !omitted {
1397                     lines.push(line);
1398                 }
1399             }
1400
1401             lines.join("\n")
1402         }
1403
1404         let xform = |p: &Path| {
1405             let mut contents = t!(fs::read_to_string(p));
1406             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1407                 if !built_tools.contains(tool) {
1408                     contents = filter(&contents, tool);
1409                 }
1410             }
1411             let ret = tmp.join(p.file_name().unwrap());
1412             t!(fs::write(&ret, &contents));
1413             ret
1414         };
1415
1416         if target.contains("apple-darwin") {
1417             builder.info("building pkg installer");
1418             let pkg = tmp.join("pkg");
1419             let _ = fs::remove_dir_all(&pkg);
1420
1421             let pkgbuild = |component: &str| {
1422                 let mut cmd = Command::new("pkgbuild");
1423                 cmd.arg("--identifier")
1424                     .arg(format!("org.rust-lang.{}", component))
1425                     .arg("--scripts")
1426                     .arg(pkg.join(component))
1427                     .arg("--nopayload")
1428                     .arg(pkg.join(component).with_extension("pkg"));
1429                 builder.run(&mut cmd);
1430             };
1431
1432             let prepare = |name: &str| {
1433                 builder.create_dir(&pkg.join(name));
1434                 builder.cp_r(
1435                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1436                     &pkg.join(name),
1437                 );
1438                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1439                 pkgbuild(name);
1440             };
1441             prepare("rustc");
1442             prepare("cargo");
1443             prepare("rust-std");
1444             prepare("rust-analysis");
1445             prepare("clippy");
1446             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1447                 if built_tools.contains(tool) {
1448                     prepare(tool);
1449                 }
1450             }
1451             // create an 'uninstall' package
1452             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1453             pkgbuild("uninstall");
1454
1455             builder.create_dir(&pkg.join("res"));
1456             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1457             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1458             let mut cmd = Command::new("productbuild");
1459             cmd.arg("--distribution")
1460                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1461                 .arg("--resources")
1462                 .arg(pkg.join("res"))
1463                 .arg(distdir(builder).join(format!(
1464                     "{}-{}.pkg",
1465                     pkgname(builder, "rust"),
1466                     target.triple
1467                 )))
1468                 .arg("--package-path")
1469                 .arg(&pkg);
1470             let _time = timeit(builder);
1471             builder.run(&mut cmd);
1472         }
1473
1474         if target.contains("windows") {
1475             let exe = tmp.join("exe");
1476             let _ = fs::remove_dir_all(&exe);
1477
1478             let prepare = |name: &str| {
1479                 builder.create_dir(&exe.join(name));
1480                 let dir = if name == "rust-std" || name == "rust-analysis" {
1481                     format!("{}-{}", name, target.triple)
1482                 } else if name == "rls" {
1483                     "rls-preview".to_string()
1484                 } else if name == "rust-analyzer" {
1485                     "rust-analyzer-preview".to_string()
1486                 } else if name == "clippy" {
1487                     "clippy-preview".to_string()
1488                 } else if name == "rust-demangler" {
1489                     "rust-demangler-preview".to_string()
1490                 } else if name == "miri" {
1491                     "miri-preview".to_string()
1492                 } else {
1493                     name.to_string()
1494                 };
1495                 builder.cp_r(
1496                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1497                     &exe.join(name),
1498                 );
1499                 builder.remove(&exe.join(name).join("manifest.in"));
1500             };
1501             prepare("rustc");
1502             prepare("cargo");
1503             prepare("rust-analysis");
1504             prepare("rust-docs");
1505             prepare("rust-std");
1506             prepare("clippy");
1507             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1508                 if built_tools.contains(tool) {
1509                     prepare(tool);
1510                 }
1511             }
1512             if target.ends_with("windows-gnu") {
1513                 prepare("rust-mingw");
1514             }
1515
1516             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1517
1518             // Generate msi installer
1519             let wix_path = env::var_os("WIX")
1520                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1521             let wix = PathBuf::from(wix_path);
1522             let heat = wix.join("bin/heat.exe");
1523             let candle = wix.join("bin/candle.exe");
1524             let light = wix.join("bin/light.exe");
1525
1526             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1527             builder.run(
1528                 Command::new(&heat)
1529                     .current_dir(&exe)
1530                     .arg("dir")
1531                     .arg("rustc")
1532                     .args(&heat_flags)
1533                     .arg("-cg")
1534                     .arg("RustcGroup")
1535                     .arg("-dr")
1536                     .arg("Rustc")
1537                     .arg("-var")
1538                     .arg("var.RustcDir")
1539                     .arg("-out")
1540                     .arg(exe.join("RustcGroup.wxs")),
1541             );
1542             builder.run(
1543                 Command::new(&heat)
1544                     .current_dir(&exe)
1545                     .arg("dir")
1546                     .arg("rust-docs")
1547                     .args(&heat_flags)
1548                     .arg("-cg")
1549                     .arg("DocsGroup")
1550                     .arg("-dr")
1551                     .arg("Docs")
1552                     .arg("-var")
1553                     .arg("var.DocsDir")
1554                     .arg("-out")
1555                     .arg(exe.join("DocsGroup.wxs"))
1556                     .arg("-t")
1557                     .arg(etc.join("msi/squash-components.xsl")),
1558             );
1559             builder.run(
1560                 Command::new(&heat)
1561                     .current_dir(&exe)
1562                     .arg("dir")
1563                     .arg("cargo")
1564                     .args(&heat_flags)
1565                     .arg("-cg")
1566                     .arg("CargoGroup")
1567                     .arg("-dr")
1568                     .arg("Cargo")
1569                     .arg("-var")
1570                     .arg("var.CargoDir")
1571                     .arg("-out")
1572                     .arg(exe.join("CargoGroup.wxs"))
1573                     .arg("-t")
1574                     .arg(etc.join("msi/remove-duplicates.xsl")),
1575             );
1576             builder.run(
1577                 Command::new(&heat)
1578                     .current_dir(&exe)
1579                     .arg("dir")
1580                     .arg("rust-std")
1581                     .args(&heat_flags)
1582                     .arg("-cg")
1583                     .arg("StdGroup")
1584                     .arg("-dr")
1585                     .arg("Std")
1586                     .arg("-var")
1587                     .arg("var.StdDir")
1588                     .arg("-out")
1589                     .arg(exe.join("StdGroup.wxs")),
1590             );
1591             if built_tools.contains("rls") {
1592                 builder.run(
1593                     Command::new(&heat)
1594                         .current_dir(&exe)
1595                         .arg("dir")
1596                         .arg("rls")
1597                         .args(&heat_flags)
1598                         .arg("-cg")
1599                         .arg("RlsGroup")
1600                         .arg("-dr")
1601                         .arg("Rls")
1602                         .arg("-var")
1603                         .arg("var.RlsDir")
1604                         .arg("-out")
1605                         .arg(exe.join("RlsGroup.wxs"))
1606                         .arg("-t")
1607                         .arg(etc.join("msi/remove-duplicates.xsl")),
1608                 );
1609             }
1610             if built_tools.contains("rust-analyzer") {
1611                 builder.run(
1612                     Command::new(&heat)
1613                         .current_dir(&exe)
1614                         .arg("dir")
1615                         .arg("rust-analyzer")
1616                         .args(&heat_flags)
1617                         .arg("-cg")
1618                         .arg("RustAnalyzerGroup")
1619                         .arg("-dr")
1620                         .arg("RustAnalyzer")
1621                         .arg("-var")
1622                         .arg("var.RustAnalyzerDir")
1623                         .arg("-out")
1624                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1625                         .arg("-t")
1626                         .arg(etc.join("msi/remove-duplicates.xsl")),
1627                 );
1628             }
1629             builder.run(
1630                 Command::new(&heat)
1631                     .current_dir(&exe)
1632                     .arg("dir")
1633                     .arg("clippy")
1634                     .args(&heat_flags)
1635                     .arg("-cg")
1636                     .arg("ClippyGroup")
1637                     .arg("-dr")
1638                     .arg("Clippy")
1639                     .arg("-var")
1640                     .arg("var.ClippyDir")
1641                     .arg("-out")
1642                     .arg(exe.join("ClippyGroup.wxs"))
1643                     .arg("-t")
1644                     .arg(etc.join("msi/remove-duplicates.xsl")),
1645             );
1646             if built_tools.contains("rust-demangler") {
1647                 builder.run(
1648                     Command::new(&heat)
1649                         .current_dir(&exe)
1650                         .arg("dir")
1651                         .arg("rust-demangler")
1652                         .args(&heat_flags)
1653                         .arg("-cg")
1654                         .arg("RustDemanglerGroup")
1655                         .arg("-dr")
1656                         .arg("RustDemangler")
1657                         .arg("-var")
1658                         .arg("var.RustDemanglerDir")
1659                         .arg("-out")
1660                         .arg(exe.join("RustDemanglerGroup.wxs"))
1661                         .arg("-t")
1662                         .arg(etc.join("msi/remove-duplicates.xsl")),
1663                 );
1664             }
1665             if built_tools.contains("miri") {
1666                 builder.run(
1667                     Command::new(&heat)
1668                         .current_dir(&exe)
1669                         .arg("dir")
1670                         .arg("miri")
1671                         .args(&heat_flags)
1672                         .arg("-cg")
1673                         .arg("MiriGroup")
1674                         .arg("-dr")
1675                         .arg("Miri")
1676                         .arg("-var")
1677                         .arg("var.MiriDir")
1678                         .arg("-out")
1679                         .arg(exe.join("MiriGroup.wxs"))
1680                         .arg("-t")
1681                         .arg(etc.join("msi/remove-duplicates.xsl")),
1682                 );
1683             }
1684             builder.run(
1685                 Command::new(&heat)
1686                     .current_dir(&exe)
1687                     .arg("dir")
1688                     .arg("rust-analysis")
1689                     .args(&heat_flags)
1690                     .arg("-cg")
1691                     .arg("AnalysisGroup")
1692                     .arg("-dr")
1693                     .arg("Analysis")
1694                     .arg("-var")
1695                     .arg("var.AnalysisDir")
1696                     .arg("-out")
1697                     .arg(exe.join("AnalysisGroup.wxs"))
1698                     .arg("-t")
1699                     .arg(etc.join("msi/remove-duplicates.xsl")),
1700             );
1701             if target.ends_with("windows-gnu") {
1702                 builder.run(
1703                     Command::new(&heat)
1704                         .current_dir(&exe)
1705                         .arg("dir")
1706                         .arg("rust-mingw")
1707                         .args(&heat_flags)
1708                         .arg("-cg")
1709                         .arg("GccGroup")
1710                         .arg("-dr")
1711                         .arg("Gcc")
1712                         .arg("-var")
1713                         .arg("var.GccDir")
1714                         .arg("-out")
1715                         .arg(exe.join("GccGroup.wxs")),
1716                 );
1717             }
1718
1719             let candle = |input: &Path| {
1720                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1721                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1722                 let mut cmd = Command::new(&candle);
1723                 cmd.current_dir(&exe)
1724                     .arg("-nologo")
1725                     .arg("-dRustcDir=rustc")
1726                     .arg("-dDocsDir=rust-docs")
1727                     .arg("-dCargoDir=cargo")
1728                     .arg("-dStdDir=rust-std")
1729                     .arg("-dAnalysisDir=rust-analysis")
1730                     .arg("-dClippyDir=clippy")
1731                     .arg("-arch")
1732                     .arg(&arch)
1733                     .arg("-out")
1734                     .arg(&output)
1735                     .arg(&input);
1736                 add_env(builder, &mut cmd, target);
1737
1738                 if built_tools.contains("rust-demangler") {
1739                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1740                 }
1741                 if built_tools.contains("rls") {
1742                     cmd.arg("-dRlsDir=rls");
1743                 }
1744                 if built_tools.contains("rust-analyzer") {
1745                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1746                 }
1747                 if built_tools.contains("miri") {
1748                     cmd.arg("-dMiriDir=miri");
1749                 }
1750                 if target.ends_with("windows-gnu") {
1751                     cmd.arg("-dGccDir=rust-mingw");
1752                 }
1753                 builder.run(&mut cmd);
1754             };
1755             candle(&xform(&etc.join("msi/rust.wxs")));
1756             candle(&etc.join("msi/ui.wxs"));
1757             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1758             candle("RustcGroup.wxs".as_ref());
1759             candle("DocsGroup.wxs".as_ref());
1760             candle("CargoGroup.wxs".as_ref());
1761             candle("StdGroup.wxs".as_ref());
1762             candle("ClippyGroup.wxs".as_ref());
1763             if built_tools.contains("rust-demangler") {
1764                 candle("RustDemanglerGroup.wxs".as_ref());
1765             }
1766             if built_tools.contains("rls") {
1767                 candle("RlsGroup.wxs".as_ref());
1768             }
1769             if built_tools.contains("rust-analyzer") {
1770                 candle("RustAnalyzerGroup.wxs".as_ref());
1771             }
1772             if built_tools.contains("miri") {
1773                 candle("MiriGroup.wxs".as_ref());
1774             }
1775             candle("AnalysisGroup.wxs".as_ref());
1776
1777             if target.ends_with("windows-gnu") {
1778                 candle("GccGroup.wxs".as_ref());
1779             }
1780
1781             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1782             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1783             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1784
1785             builder.info(&format!("building `msi` installer with {:?}", light));
1786             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1787             let mut cmd = Command::new(&light);
1788             cmd.arg("-nologo")
1789                 .arg("-ext")
1790                 .arg("WixUIExtension")
1791                 .arg("-ext")
1792                 .arg("WixUtilExtension")
1793                 .arg("-out")
1794                 .arg(exe.join(&filename))
1795                 .arg("rust.wixobj")
1796                 .arg("ui.wixobj")
1797                 .arg("rustwelcomedlg.wixobj")
1798                 .arg("RustcGroup.wixobj")
1799                 .arg("DocsGroup.wixobj")
1800                 .arg("CargoGroup.wixobj")
1801                 .arg("StdGroup.wixobj")
1802                 .arg("AnalysisGroup.wixobj")
1803                 .arg("ClippyGroup.wixobj")
1804                 .current_dir(&exe);
1805
1806             if built_tools.contains("rls") {
1807                 cmd.arg("RlsGroup.wixobj");
1808             }
1809             if built_tools.contains("rust-analyzer") {
1810                 cmd.arg("RustAnalyzerGroup.wixobj");
1811             }
1812             if built_tools.contains("rust-demangler") {
1813                 cmd.arg("RustDemanglerGroup.wixobj");
1814             }
1815             if built_tools.contains("miri") {
1816                 cmd.arg("MiriGroup.wixobj");
1817             }
1818
1819             if target.ends_with("windows-gnu") {
1820                 cmd.arg("GccGroup.wixobj");
1821             }
1822             // ICE57 wrongly complains about the shortcuts
1823             cmd.arg("-sice:ICE57");
1824
1825             let _time = timeit(builder);
1826             builder.run(&mut cmd);
1827
1828             if !builder.config.dry_run {
1829                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1830             }
1831         }
1832     }
1833 }
1834
1835 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1836     let mut parts = builder.version.split('.');
1837     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1838         .env("CFG_RELEASE_NUM", &builder.version)
1839         .env("CFG_RELEASE", builder.rust_release())
1840         .env("CFG_VER_MAJOR", parts.next().unwrap())
1841         .env("CFG_VER_MINOR", parts.next().unwrap())
1842         .env("CFG_VER_PATCH", parts.next().unwrap())
1843         .env("CFG_VER_BUILD", "0") // just needed to build
1844         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1845         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1846         .env("CFG_BUILD", target.triple)
1847         .env("CFG_CHANNEL", &builder.config.channel);
1848
1849     if target.contains("windows-gnullvm") {
1850         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1851     } else if target.contains("windows-gnu") {
1852         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1853     } else {
1854         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1855     }
1856 }
1857
1858 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1859 ///
1860 /// Returns whether the files were actually copied.
1861 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1862     if let Some(config) = builder.config.target_config.get(&target) {
1863         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1864             // If the LLVM was externally provided, then we don't currently copy
1865             // artifacts into the sysroot. This is not necessarily the right
1866             // choice (in particular, it will require the LLVM dylib to be in
1867             // the linker's load path at runtime), but the common use case for
1868             // external LLVMs is distribution provided LLVMs, and in that case
1869             // they're usually in the standard search path (e.g., /usr/lib) and
1870             // copying them here is going to cause problems as we may end up
1871             // with the wrong files and isn't what distributions want.
1872             //
1873             // This behavior may be revisited in the future though.
1874             //
1875             // If the LLVM is coming from ourselves (just from CI) though, we
1876             // still want to install it, as it otherwise won't be available.
1877             return false;
1878         }
1879     }
1880
1881     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1882     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1883     // clear why this is the case, though. llvm-config will emit the versioned
1884     // paths and we don't want those in the sysroot (as we're expecting
1885     // unversioned paths).
1886     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1887         let src_libdir = builder.llvm_out(target).join("lib");
1888         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1889         if llvm_dylib_path.exists() {
1890             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1891         }
1892         !builder.config.dry_run
1893     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1894         let mut cmd = Command::new(llvm_config);
1895         cmd.arg("--libfiles");
1896         builder.verbose(&format!("running {:?}", cmd));
1897         let files = output(&mut cmd);
1898         let build_llvm_out = &builder.llvm_out(builder.config.build);
1899         let target_llvm_out = &builder.llvm_out(target);
1900         for file in files.trim_end().split(' ') {
1901             // If we're not using a custom LLVM, make sure we package for the target.
1902             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1903                 target_llvm_out.join(relative_path)
1904             } else {
1905                 PathBuf::from(file)
1906             };
1907             builder.install(&file, dst_libdir, 0o644);
1908         }
1909         !builder.config.dry_run
1910     } else {
1911         false
1912     }
1913 }
1914
1915 /// Maybe add libLLVM.so to the target lib-dir for linking.
1916 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1917     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1918     // We do not need to copy LLVM files into the sysroot if it is not
1919     // dynamically linked; it is already included into librustc_llvm
1920     // statically.
1921     if builder.llvm_link_shared() {
1922         maybe_install_llvm(builder, target, &dst_libdir);
1923     }
1924 }
1925
1926 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1927 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1928     let dst_libdir =
1929         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1930     // We do not need to copy LLVM files into the sysroot if it is not
1931     // dynamically linked; it is already included into librustc_llvm
1932     // statically.
1933     if builder.llvm_link_shared() {
1934         maybe_install_llvm(builder, target, &dst_libdir);
1935     }
1936 }
1937
1938 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1939 pub struct LlvmTools {
1940     pub target: TargetSelection,
1941 }
1942
1943 impl Step for LlvmTools {
1944     type Output = Option<GeneratedTarball>;
1945     const ONLY_HOSTS: bool = true;
1946     const DEFAULT: bool = true;
1947
1948     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1949         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1950         // FIXME: allow using the names of the tools themselves?
1951         run.alias("llvm-tools").default_condition(default)
1952     }
1953
1954     fn make_run(run: RunConfig<'_>) {
1955         run.builder.ensure(LlvmTools { target: run.target });
1956     }
1957
1958     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1959         let target = self.target;
1960
1961         /* run only if llvm-config isn't used */
1962         if let Some(config) = builder.config.target_config.get(&target) {
1963             if let Some(ref _s) = config.llvm_config {
1964                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1965                 return None;
1966             }
1967         }
1968
1969         builder.ensure(crate::native::Llvm { target });
1970
1971         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1972         tarball.set_overlay(OverlayKind::LLVM);
1973         tarball.is_preview(true);
1974
1975         // Prepare the image directory
1976         let src_bindir = builder.llvm_out(target).join("bin");
1977         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1978         for tool in LLVM_TOOLS {
1979             let exe = src_bindir.join(exe(tool, target));
1980             tarball.add_file(&exe, &dst_bindir, 0o755);
1981         }
1982
1983         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1984         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1985         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1986         // compiler libraries.
1987         maybe_install_llvm_target(builder, target, tarball.image_dir());
1988
1989         Some(tarball.generate())
1990     }
1991 }
1992
1993 // Tarball intended for internal consumption to ease rustc/std development.
1994 //
1995 // Should not be considered stable by end users.
1996 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1997 pub struct RustDev {
1998     pub target: TargetSelection,
1999 }
2000
2001 impl Step for RustDev {
2002     type Output = Option<GeneratedTarball>;
2003     const DEFAULT: bool = true;
2004     const ONLY_HOSTS: bool = true;
2005
2006     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2007         run.alias("rust-dev")
2008     }
2009
2010     fn make_run(run: RunConfig<'_>) {
2011         run.builder.ensure(RustDev { target: run.target });
2012     }
2013
2014     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2015         let target = self.target;
2016
2017         /* run only if llvm-config isn't used */
2018         if let Some(config) = builder.config.target_config.get(&target) {
2019             if let Some(ref _s) = config.llvm_config {
2020                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2021                 return None;
2022             }
2023         }
2024
2025         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2026         tarball.set_overlay(OverlayKind::LLVM);
2027
2028         builder.ensure(crate::native::Llvm { target });
2029
2030         let src_bindir = builder.llvm_out(target).join("bin");
2031         // If updating this list, you likely want to change
2032         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2033         // will not pick up the extra file until LLVM gets bumped.
2034         for bin in &[
2035             "llvm-config",
2036             "llvm-ar",
2037             "llvm-objdump",
2038             "llvm-profdata",
2039             "llvm-bcanalyzer",
2040             "llvm-cov",
2041             "llvm-dwp",
2042             "llvm-nm",
2043             "llvm-dwarfdump",
2044         ] {
2045             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2046         }
2047
2048         // We don't build LLD on some platforms, so only add it if it exists
2049         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2050         if lld_path.exists() {
2051             tarball.add_file(lld_path, "bin", 0o755);
2052         }
2053
2054         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2055
2056         // Copy the include directory as well; needed mostly to build
2057         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2058         // just broadly useful to be able to link against the bundled LLVM.
2059         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2060
2061         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2062         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2063         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2064         // compiler libraries.
2065         let dst_libdir = tarball.image_dir().join("lib");
2066         maybe_install_llvm(builder, target, &dst_libdir);
2067         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2068         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2069
2070         Some(tarball.generate())
2071     }
2072 }
2073
2074 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2075 /// release process to avoid cloning the monorepo and building stuff.
2076 ///
2077 /// Should not be considered stable by end users.
2078 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2079 pub struct BuildManifest {
2080     pub target: TargetSelection,
2081 }
2082
2083 impl Step for BuildManifest {
2084     type Output = GeneratedTarball;
2085     const DEFAULT: bool = false;
2086     const ONLY_HOSTS: bool = true;
2087
2088     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2089         run.alias("build-manifest")
2090     }
2091
2092     fn make_run(run: RunConfig<'_>) {
2093         run.builder.ensure(BuildManifest { target: run.target });
2094     }
2095
2096     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2097         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2098
2099         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2100         tarball.add_file(&build_manifest, "bin", 0o755);
2101         tarball.generate()
2102     }
2103 }
2104
2105 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2106 ///
2107 /// Currently this is the PGO profile data.
2108 ///
2109 /// Should not be considered stable by end users.
2110 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2111 pub struct ReproducibleArtifacts {
2112     pub target: TargetSelection,
2113 }
2114
2115 impl Step for ReproducibleArtifacts {
2116     type Output = Option<GeneratedTarball>;
2117     const DEFAULT: bool = true;
2118     const ONLY_HOSTS: bool = true;
2119
2120     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2121         run.alias("reproducible-artifacts")
2122     }
2123
2124     fn make_run(run: RunConfig<'_>) {
2125         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2126     }
2127
2128     fn run(self, builder: &Builder<'_>) -> Self::Output {
2129         let mut added_anything = false;
2130         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2131         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2132             tarball.add_file(path, ".", 0o644);
2133             added_anything = true;
2134         }
2135         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2136             tarball.add_file(path, ".", 0o644);
2137             added_anything = true;
2138         }
2139         if added_anything { Some(tarball.generate()) } else { None }
2140     }
2141 }