src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.alias("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.alias("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.ends_with("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.alias("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.ends_with("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 let exe_name = exe("ld", compiler.host);
 411                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 412             }
 413
 414             // Man pages
 415             t!(fs::create_dir_all(image.join("share/man/man1")));
 416             let man_src = builder.src.join("src/doc/man");
 417             let man_dst = image.join("share/man/man1");
 418
 419             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 420             // to hardlink, and we don't want to edit the source templates
 421             for file_entry in builder.read_dir(&man_src) {
 422                 let page_src = file_entry.path();
 423                 let page_dst = man_dst.join(file_entry.file_name());
 424                 let src_text = t!(std::fs::read_to_string(&page_src));
 425                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 426                 t!(std::fs::write(&page_dst, &new_text));
 427                 t!(fs::copy(&page_src, &page_dst));
 428             }
 429
 430             // Debugger scripts
 431             builder
 432                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 433
 434             // Misc license info
 435             let cp = |file: &str| {
 436                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 437             };
 438             cp("COPYRIGHT");
 439             cp("LICENSE-APACHE");
 440             cp("LICENSE-MIT");
 441             cp("README.md");
 442         }
 443     }
 444 }
 445
 446 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 447 pub struct DebuggerScripts {
 448     pub sysroot: Interned<PathBuf>,
 449     pub host: TargetSelection,
 450 }
 451
 452 impl Step for DebuggerScripts {
 453     type Output = ();
 454
 455     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 456         run.never()
 457     }
 458
 459     /// Copies debugger scripts for `target` into the `sysroot` specified.
 460     fn run(self, builder: &Builder<'_>) {
 461         let host = self.host;
 462         let sysroot = self.sysroot;
 463         let dst = sysroot.join("lib/rustlib/etc");
 464         t!(fs::create_dir_all(&dst));
 465         let cp_debugger_script = |file: &str| {
 466             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 467         };
 468         if host.contains("windows-msvc") {
 469             // windbg debugger scripts
 470             builder.install(
 471                 &builder.src.join("src/etc/rust-windbg.cmd"),
 472                 &sysroot.join("bin"),
 473                 0o755,
 474             );
 475
 476             cp_debugger_script("natvis/intrinsic.natvis");
 477             cp_debugger_script("natvis/liballoc.natvis");
 478             cp_debugger_script("natvis/libcore.natvis");
 479             cp_debugger_script("natvis/libstd.natvis");
 480         } else {
 481             cp_debugger_script("rust_types.py");
 482
 483             // gdb debugger scripts
 484             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 485             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 486
 487             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 488             cp_debugger_script("gdb_lookup.py");
 489             cp_debugger_script("gdb_providers.py");
 490
 491             // lldb debugger scripts
 492             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 493
 494             cp_debugger_script("lldb_lookup.py");
 495             cp_debugger_script("lldb_providers.py");
 496             cp_debugger_script("lldb_commands")
 497         }
 498     }
 499 }
 500
 501 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 502     // The only true set of target libraries came from the build triple, so
 503     // let's reduce redundant work by only producing archives from that host.
 504     if compiler.host != builder.config.build {
 505         builder.info("\tskipping, not a build host");
 506         true
 507     } else {
 508         false
 509     }
 510 }
 511
 512 /// Copy stamped files into an image's `target/lib` directory.
 513 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 514     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 515     let self_contained_dst = dst.join("self-contained");
 516     t!(fs::create_dir_all(&dst));
 517     t!(fs::create_dir_all(&self_contained_dst));
 518     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 519         if dependency_type == DependencyType::TargetSelfContained {
 520             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 521         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 522             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 523         }
 524     }
 525 }
 526
 527 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 528 pub struct Std {
 529     pub compiler: Compiler,
 530     pub target: TargetSelection,
 531 }
 532
 533 impl Step for Std {
 534     type Output = Option<GeneratedTarball>;
 535     const DEFAULT: bool = true;
 536
 537     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 538         run.alias("rust-std")
 539     }
 540
 541     fn make_run(run: RunConfig<'_>) {
 542         run.builder.ensure(Std {
 543             compiler: run.builder.compiler_for(
 544                 run.builder.top_stage,
 545                 run.builder.config.build,
 546                 run.target,
 547             ),
 548             target: run.target,
 549         });
 550     }
 551
 552     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 553         let compiler = self.compiler;
 554         let target = self.target;
 555
 556         if skip_host_target_lib(builder, compiler) {
 557             return None;
 558         }
 559
 560         builder.ensure(compile::Std { compiler, target });
 561
 562         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 563         tarball.include_target_in_component_name(true);
 564
 565         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 566         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 567         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 568
 569         Some(tarball.generate())
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct RustcDev {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for RustcDev {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582     const ONLY_HOSTS: bool = true;
 583
 584     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 585         run.alias("rustc-dev")
 586     }
 587
 588     fn make_run(run: RunConfig<'_>) {
 589         run.builder.ensure(RustcDev {
 590             compiler: run.builder.compiler_for(
 591                 run.builder.top_stage,
 592                 run.builder.config.build,
 593                 run.target,
 594             ),
 595             target: run.target,
 596         });
 597     }
 598
 599     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 600         let compiler = self.compiler;
 601         let target = self.target;
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Rustc { compiler, target });
 607
 608         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 609
 610         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 611         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 612         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 613
 614         let src_files = &["Cargo.lock"];
 615         // This is the reduced set of paths which will become the rustc-dev component
 616         // (essentially the compiler crates and all of their path dependencies).
 617         copy_src_dirs(
 618             builder,
 619             &builder.src,
 620             &["compiler"],
 621             &[],
 622             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 623         );
 624         for file in src_files {
 625             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 626         }
 627
 628         Some(tarball.generate())
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: TargetSelection,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = Option<GeneratedTarball>;
 640     const DEFAULT: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         let default = should_build_extended_tool(&run.builder, "analysis");
 644         run.alias("rust-analysis").default_condition(default)
 645     }
 646
 647     fn make_run(run: RunConfig<'_>) {
 648         run.builder.ensure(Analysis {
 649             // Find the actual compiler (handling the full bootstrap option) which
 650             // produced the save-analysis data because that data isn't copied
 651             // through the sysroot uplifting.
 652             compiler: run.builder.compiler_for(
 653                 run.builder.top_stage,
 654                 run.builder.config.build,
 655                 run.target,
 656             ),
 657             target: run.target,
 658         });
 659     }
 660
 661     /// Creates a tarball of save-analysis metadata, if available.
 662     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 663         let compiler = self.compiler;
 664         let target = self.target;
 665         if compiler.host != builder.config.build {
 666             return None;
 667         }
 668
 669         builder.ensure(compile::Std { compiler, target });
 670         let src = builder
 671             .stage_out(compiler, Mode::Std)
 672             .join(target.triple)
 673             .join(builder.cargo_dir())
 674             .join("deps")
 675             .join("save-analysis");
 676
 677         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 678         tarball.include_target_in_component_name(true);
 679         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 680         Some(tarball.generate())
 681     }
 682 }
 683
 684 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 685 /// `dst_dir`.
 686 fn copy_src_dirs(
 687     builder: &Builder<'_>,
 688     base: &Path,
 689     src_dirs: &[&str],
 690     exclude_dirs: &[&str],
 691     dst_dir: &Path,
 692 ) {
 693     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 694         let spath = match path.to_str() {
 695             Some(path) => path,
 696             None => return false,
 697         };
 698         if spath.ends_with('~') || spath.ends_with(".pyc") {
 699             return false;
 700         }
 701
 702         const LLVM_PROJECTS: &[&str] = &[
 703             "llvm-project/clang",
 704             "llvm-project\\clang",
 705             "llvm-project/libunwind",
 706             "llvm-project\\libunwind",
 707             "llvm-project/lld",
 708             "llvm-project\\lld",
 709             "llvm-project/lldb",
 710             "llvm-project\\lldb",
 711             "llvm-project/llvm",
 712             "llvm-project\\llvm",
 713             "llvm-project/compiler-rt",
 714             "llvm-project\\compiler-rt",
 715             "llvm-project/cmake",
 716             "llvm-project\\cmake",
 717         ];
 718         if spath.contains("llvm-project")
 719             && !spath.ends_with("llvm-project")
 720             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 721         {
 722             return false;
 723         }
 724
 725         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 726         if LLVM_TEST.iter().any(|path| spath.contains(path))
 727             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 728         {
 729             return false;
 730         }
 731
 732         let full_path = Path::new(dir).join(path);
 733         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 734             return false;
 735         }
 736
 737         let excludes = [
 738             "CVS",
 739             "RCS",
 740             "SCCS",
 741             ".git",
 742             ".gitignore",
 743             ".gitmodules",
 744             ".gitattributes",
 745             ".cvsignore",
 746             ".svn",
 747             ".arch-ids",
 748             "{arch}",
 749             "=RELEASE-ID",
 750             "=meta-update",
 751             "=update",
 752             ".bzr",
 753             ".bzrignore",
 754             ".bzrtags",
 755             ".hg",
 756             ".hgignore",
 757             ".hgrags",
 758             "_darcs",
 759         ];
 760         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 761     }
 762
 763     // Copy the directories using our filter
 764     for item in src_dirs {
 765         let dst = &dst_dir.join(item);
 766         t!(fs::create_dir_all(dst));
 767         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 768     }
 769 }
 770
 771 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 772 pub struct Src;
 773
 774 impl Step for Src {
 775     /// The output path of the src installer tarball
 776     type Output = GeneratedTarball;
 777     const DEFAULT: bool = true;
 778     const ONLY_HOSTS: bool = true;
 779
 780     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 781         run.alias("rust-src")
 782     }
 783
 784     fn make_run(run: RunConfig<'_>) {
 785         run.builder.ensure(Src);
 786     }
 787
 788     /// Creates the `rust-src` installer component
 789     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 790         let tarball = Tarball::new_targetless(builder, "rust-src");
 791
 792         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 793         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 794         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 795         // and fix them...
 796         //
 797         // NOTE: if you update the paths here, you also should update the "virtual" path
 798         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 799         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 800
 801         let src_files = ["Cargo.lock"];
 802         // This is the reduced set of paths which will become the rust-src component
 803         // (essentially libstd and all of its path dependencies).
 804         copy_src_dirs(
 805             builder,
 806             &builder.src,
 807             &["library", "src/llvm-project/libunwind"],
 808             &[
 809                 // not needed and contains symlinks which rustup currently
 810                 // chokes on when unpacking.
 811                 "library/backtrace/crates",
 812                 // these are 30MB combined and aren't necessary for building
 813                 // the standard library.
 814                 "library/stdarch/Cargo.toml",
 815                 "library/stdarch/crates/stdarch-verify",
 816                 "library/stdarch/crates/intrinsic-test",
 817             ],
 818             &dst_src,
 819         );
 820         for file in src_files.iter() {
 821             builder.copy(&builder.src.join(file), &dst_src.join(file));
 822         }
 823
 824         tarball.generate()
 825     }
 826 }
 827
 828 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 829 pub struct PlainSourceTarball;
 830
 831 impl Step for PlainSourceTarball {
 832     /// Produces the location of the tarball generated
 833     type Output = GeneratedTarball;
 834     const DEFAULT: bool = true;
 835     const ONLY_HOSTS: bool = true;
 836
 837     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 838         let builder = run.builder;
 839         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig<'_>) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 848         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 849         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 850         // It also contains symbolic links, unlike other any other dist tarball.
 851         // It's used for distros building rustc from source in a pre-vendored environment.
 852         let mut tarball = Tarball::new(builder, "rustc", "src");
 853         tarball.permit_symlinks(true);
 854         let plain_dst_src = tarball.image_dir();
 855
 856         // This is the set of root paths which will become part of the source package
 857         let src_files = [
 858             "COPYRIGHT",
 859             "LICENSE-APACHE",
 860             "LICENSE-MIT",
 861             "CONTRIBUTING.md",
 862             "README.md",
 863             "RELEASES.md",
 864             "configure",
 865             "x.py",
 866             "config.toml.example",
 867             "Cargo.toml",
 868             "Cargo.lock",
 869         ];
 870         let src_dirs = ["src", "compiler", "library"];
 871
 872         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 873
 874         // Copy the files normally
 875         for item in &src_files {
 876             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 877         }
 878
 879         // Create the version file
 880         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 881         if let Some(sha) = builder.rust_sha() {
 882             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 883         }
 884
 885         // If we're building from git sources, we need to vendor a complete distribution.
 886         if builder.rust_info.is_git() {
 887             // Ensure we have the submodules checked out.
 888             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 889
 890             // Vendor all Cargo dependencies
 891             let mut cmd = Command::new(&builder.initial_cargo);
 892             cmd.arg("vendor")
 893                 .arg("--sync")
 894                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 895                 .arg("--sync")
 896                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 897                 .current_dir(&plain_dst_src);
 898             builder.run(&mut cmd);
 899         }
 900
 901         tarball.bare()
 902     }
 903 }
 904
 905 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 906 pub struct Cargo {
 907     pub compiler: Compiler,
 908     pub target: TargetSelection,
 909 }
 910
 911 impl Step for Cargo {
 912     type Output = Option<GeneratedTarball>;
 913     const DEFAULT: bool = true;
 914     const ONLY_HOSTS: bool = true;
 915
 916     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 917         let default = should_build_extended_tool(&run.builder, "cargo");
 918         run.alias("cargo").default_condition(default)
 919     }
 920
 921     fn make_run(run: RunConfig<'_>) {
 922         run.builder.ensure(Cargo {
 923             compiler: run.builder.compiler_for(
 924                 run.builder.top_stage,
 925                 run.builder.config.build,
 926                 run.target,
 927             ),
 928             target: run.target,
 929         });
 930     }
 931
 932     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 933         let compiler = self.compiler;
 934         let target = self.target;
 935
 936         let cargo = builder.ensure(tool::Cargo { compiler, target });
 937         let src = builder.src.join("src/tools/cargo");
 938         let etc = src.join("src/etc");
 939
 940         // Prepare the image directory
 941         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 942         tarball.set_overlay(OverlayKind::Cargo);
 943
 944         tarball.add_file(&cargo, "bin", 0o755);
 945         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 946         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 947         tarball.add_dir(etc.join("man"), "share/man/man1");
 948         tarball.add_legal_and_readme_to("share/doc/cargo");
 949
 950         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 951             let dirent = dirent.expect("read dir entry");
 952             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 953                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 954             }
 955         }
 956
 957         Some(tarball.generate())
 958     }
 959 }
 960
 961 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 962 pub struct Rls {
 963     pub compiler: Compiler,
 964     pub target: TargetSelection,
 965 }
 966
 967 impl Step for Rls {
 968     type Output = Option<GeneratedTarball>;
 969     const ONLY_HOSTS: bool = true;
 970     const DEFAULT: bool = true;
 971
 972     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 973         let default = should_build_extended_tool(&run.builder, "rls");
 974         run.alias("rls").default_condition(default)
 975     }
 976
 977     fn make_run(run: RunConfig<'_>) {
 978         run.builder.ensure(Rls {
 979             compiler: run.builder.compiler_for(
 980                 run.builder.top_stage,
 981                 run.builder.config.build,
 982                 run.target,
 983             ),
 984             target: run.target,
 985         });
 986     }
 987
 988     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 989         let compiler = self.compiler;
 990         let target = self.target;
 991
 992         let rls = builder
 993             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
 994             .or_else(|| {
 995                 missing_tool("RLS", builder.build.config.missing_tools);
 996                 None
 997             })?;
 998
 999         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1000         tarball.set_overlay(OverlayKind::RLS);
1001         tarball.is_preview(true);
1002         tarball.add_file(rls, "bin", 0o755);
1003         tarball.add_legal_and_readme_to("share/doc/rls");
1004         Some(tarball.generate())
1005     }
1006 }
1007
1008 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1009 pub struct RustAnalyzer {
1010     pub compiler: Compiler,
1011     pub target: TargetSelection,
1012 }
1013
1014 impl Step for RustAnalyzer {
1015     type Output = Option<GeneratedTarball>;
1016     const DEFAULT: bool = true;
1017     const ONLY_HOSTS: bool = true;
1018
1019     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1020         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1021         run.alias("rust-analyzer").default_condition(default)
1022     }
1023
1024     fn make_run(run: RunConfig<'_>) {
1025         run.builder.ensure(RustAnalyzer {
1026             compiler: run.builder.compiler_for(
1027                 run.builder.top_stage,
1028                 run.builder.config.build,
1029                 run.target,
1030             ),
1031             target: run.target,
1032         });
1033     }
1034
1035     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1036         // This prevents rust-analyzer from being built for "dist" or "install"
1037         // on the stable/beta channels. It is a nightly-only tool and should
1038         // not be included.
1039         if !builder.build.unstable_features() {
1040             return None;
1041         }
1042         let compiler = self.compiler;
1043         let target = self.target;
1044
1045         if target.contains("riscv64") {
1046             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1047             // to build. See #74813 for details.
1048             return None;
1049         }
1050
1051         let rust_analyzer = builder
1052             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1053             .expect("rust-analyzer always builds");
1054
1055         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1056         tarball.set_overlay(OverlayKind::RustAnalyzer);
1057         tarball.is_preview(true);
1058         tarball.add_file(rust_analyzer, "bin", 0o755);
1059         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1060         Some(tarball.generate())
1061     }
1062 }
1063
1064 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1065 pub struct Clippy {
1066     pub compiler: Compiler,
1067     pub target: TargetSelection,
1068 }
1069
1070 impl Step for Clippy {
1071     type Output = Option<GeneratedTarball>;
1072     const DEFAULT: bool = true;
1073     const ONLY_HOSTS: bool = true;
1074
1075     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1076         let default = should_build_extended_tool(&run.builder, "clippy");
1077         run.alias("clippy").default_condition(default)
1078     }
1079
1080     fn make_run(run: RunConfig<'_>) {
1081         run.builder.ensure(Clippy {
1082             compiler: run.builder.compiler_for(
1083                 run.builder.top_stage,
1084                 run.builder.config.build,
1085                 run.target,
1086             ),
1087             target: run.target,
1088         });
1089     }
1090
1091     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1092         let compiler = self.compiler;
1093         let target = self.target;
1094
1095         // Prepare the image directory
1096         // We expect clippy to build, because we've exited this step above if tool
1097         // state for clippy isn't testing.
1098         let clippy = builder
1099             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1100             .expect("clippy expected to build - essential tool");
1101         let cargoclippy = builder
1102             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1103             .expect("clippy expected to build - essential tool");
1104
1105         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1106         tarball.set_overlay(OverlayKind::Clippy);
1107         tarball.is_preview(true);
1108         tarball.add_file(clippy, "bin", 0o755);
1109         tarball.add_file(cargoclippy, "bin", 0o755);
1110         tarball.add_legal_and_readme_to("share/doc/clippy");
1111         Some(tarball.generate())
1112     }
1113 }
1114
1115 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1116 pub struct Miri {
1117     pub compiler: Compiler,
1118     pub target: TargetSelection,
1119 }
1120
1121 impl Step for Miri {
1122     type Output = Option<GeneratedTarball>;
1123     const DEFAULT: bool = true;
1124     const ONLY_HOSTS: bool = true;
1125
1126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1127         let default = should_build_extended_tool(&run.builder, "miri");
1128         run.alias("miri").default_condition(default)
1129     }
1130
1131     fn make_run(run: RunConfig<'_>) {
1132         run.builder.ensure(Miri {
1133             compiler: run.builder.compiler_for(
1134                 run.builder.top_stage,
1135                 run.builder.config.build,
1136                 run.target,
1137             ),
1138             target: run.target,
1139         });
1140     }
1141
1142     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1143         // This prevents miri from being built for "dist" or "install"
1144         // on the stable/beta channels. It is a nightly-only tool and should
1145         // not be included.
1146         if !builder.build.unstable_features() {
1147             return None;
1148         }
1149         let compiler = self.compiler;
1150         let target = self.target;
1151
1152         let miri = builder
1153             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1154             .or_else(|| {
1155                 missing_tool("miri", builder.build.config.missing_tools);
1156                 None
1157             })?;
1158         let cargomiri = builder
1159             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1160             .or_else(|| {
1161                 missing_tool("cargo miri", builder.build.config.missing_tools);
1162                 None
1163             })?;
1164
1165         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1166         tarball.set_overlay(OverlayKind::Miri);
1167         tarball.is_preview(true);
1168         tarball.add_file(miri, "bin", 0o755);
1169         tarball.add_file(cargomiri, "bin", 0o755);
1170         tarball.add_legal_and_readme_to("share/doc/miri");
1171         Some(tarball.generate())
1172     }
1173 }
1174
1175 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1176 pub struct Rustfmt {
1177     pub compiler: Compiler,
1178     pub target: TargetSelection,
1179 }
1180
1181 impl Step for Rustfmt {
1182     type Output = Option<GeneratedTarball>;
1183     const DEFAULT: bool = true;
1184     const ONLY_HOSTS: bool = true;
1185
1186     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1187         let default = should_build_extended_tool(&run.builder, "rustfmt");
1188         run.alias("rustfmt").default_condition(default)
1189     }
1190
1191     fn make_run(run: RunConfig<'_>) {
1192         run.builder.ensure(Rustfmt {
1193             compiler: run.builder.compiler_for(
1194                 run.builder.top_stage,
1195                 run.builder.config.build,
1196                 run.target,
1197             ),
1198             target: run.target,
1199         });
1200     }
1201
1202     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1203         let compiler = self.compiler;
1204         let target = self.target;
1205
1206         let rustfmt = builder
1207             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1208             .or_else(|| {
1209                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1210                 None
1211             })?;
1212         let cargofmt = builder
1213             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1214             .or_else(|| {
1215                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1216                 None
1217             })?;
1218
1219         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1220         tarball.set_overlay(OverlayKind::Rustfmt);
1221         tarball.is_preview(true);
1222         tarball.add_file(rustfmt, "bin", 0o755);
1223         tarball.add_file(cargofmt, "bin", 0o755);
1224         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1225         Some(tarball.generate())
1226     }
1227 }
1228
1229 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1230 pub struct RustDemangler {
1231     pub compiler: Compiler,
1232     pub target: TargetSelection,
1233 }
1234
1235 impl Step for RustDemangler {
1236     type Output = Option<GeneratedTarball>;
1237     const DEFAULT: bool = true;
1238     const ONLY_HOSTS: bool = true;
1239
1240     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1241         // While other tools use `should_build_extended_tool` to decide whether to be run by
1242         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1243         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1244         // we run the step by default when only `extended = true`, and decide whether to actually
1245         // run it or not later.
1246         let default = run.builder.config.extended;
1247         run.alias("rust-demangler").default_condition(default)
1248     }
1249
1250     fn make_run(run: RunConfig<'_>) {
1251         run.builder.ensure(RustDemangler {
1252             compiler: run.builder.compiler_for(
1253                 run.builder.top_stage,
1254                 run.builder.config.build,
1255                 run.target,
1256             ),
1257             target: run.target,
1258         });
1259     }
1260
1261     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1262         let compiler = self.compiler;
1263         let target = self.target;
1264
1265         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1266         let condition = should_build_extended_tool(builder, "rust-demangler")
1267             || builder.config.profiler_enabled(target);
1268         if builder.config.extended && !condition {
1269             return None;
1270         }
1271
1272         let rust_demangler = builder
1273             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1274             .expect("rust-demangler expected to build - in-tree tool");
1275
1276         // Prepare the image directory
1277         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1278         tarball.set_overlay(OverlayKind::RustDemangler);
1279         tarball.is_preview(true);
1280         tarball.add_file(&rust_demangler, "bin", 0o755);
1281         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1282         Some(tarball.generate())
1283     }
1284 }
1285
1286 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1287 pub struct Extended {
1288     stage: u32,
1289     host: TargetSelection,
1290     target: TargetSelection,
1291 }
1292
1293 impl Step for Extended {
1294     type Output = ();
1295     const DEFAULT: bool = true;
1296     const ONLY_HOSTS: bool = true;
1297
1298     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1299         let builder = run.builder;
1300         run.alias("extended").default_condition(builder.config.extended)
1301     }
1302
1303     fn make_run(run: RunConfig<'_>) {
1304         run.builder.ensure(Extended {
1305             stage: run.builder.top_stage,
1306             host: run.builder.config.build,
1307             target: run.target,
1308         });
1309     }
1310
1311     /// Creates a combined installer for the specified target in the provided stage.
1312     fn run(self, builder: &Builder<'_>) {
1313         let target = self.target;
1314         let stage = self.stage;
1315         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1316
1317         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1318
1319         let mut tarballs = Vec::new();
1320         let mut built_tools = HashSet::new();
1321         macro_rules! add_component {
1322             ($name:expr => $step:expr) => {
1323                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1324                     tarballs.push(tarball);
1325                     built_tools.insert($name);
1326                 }
1327             };
1328         }
1329
1330         // When rust-std package split from rustc, we needed to ensure that during
1331         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1332         // the std files during uninstall. To do this ensure that rustc comes
1333         // before rust-std in the list below.
1334         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1335         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1336
1337         if target.ends_with("windows-gnu") {
1338             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1339         }
1340
1341         add_component!("rust-docs" => Docs { host: target });
1342         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1343         add_component!("cargo" => Cargo { compiler, target });
1344         add_component!("rustfmt" => Rustfmt { compiler, target });
1345         add_component!("rls" => Rls { compiler, target });
1346         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1347         add_component!("llvm-components" => LlvmTools { target });
1348         add_component!("clippy" => Clippy { compiler, target });
1349         add_component!("miri" => Miri { compiler, target });
1350         add_component!("analysis" => Analysis { compiler, target });
1351
1352         let etc = builder.src.join("src/etc/installer");
1353
1354         // Avoid producing tarballs during a dry run.
1355         if builder.config.dry_run {
1356             return;
1357         }
1358
1359         let tarball = Tarball::new(builder, "rust", &target.triple);
1360         let generated = tarball.combine(&tarballs);
1361
1362         let tmp = tmpdir(builder).join("combined-tarball");
1363         let work = generated.work_dir();
1364
1365         let mut license = String::new();
1366         license += &builder.read(&builder.src.join("COPYRIGHT"));
1367         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1368         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1369         license.push('\n');
1370         license.push('\n');
1371
1372         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1373         let mut rtf = rtf.to_string();
1374         rtf.push('\n');
1375         for line in license.lines() {
1376             rtf.push_str(line);
1377             rtf.push_str("\\line ");
1378         }
1379         rtf.push('}');
1380
1381         fn filter(contents: &str, marker: &str) -> String {
1382             let start = format!("tool-{}-start", marker);
1383             let end = format!("tool-{}-end", marker);
1384             let mut lines = Vec::new();
1385             let mut omitted = false;
1386             for line in contents.lines() {
1387                 if line.contains(&start) {
1388                     omitted = true;
1389                 } else if line.contains(&end) {
1390                     omitted = false;
1391                 } else if !omitted {
1392                     lines.push(line);
1393                 }
1394             }
1395
1396             lines.join("\n")
1397         }
1398
1399         let xform = |p: &Path| {
1400             let mut contents = t!(fs::read_to_string(p));
1401             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1402                 if !built_tools.contains(tool) {
1403                     contents = filter(&contents, tool);
1404                 }
1405             }
1406             let ret = tmp.join(p.file_name().unwrap());
1407             t!(fs::write(&ret, &contents));
1408             ret
1409         };
1410
1411         if target.contains("apple-darwin") {
1412             builder.info("building pkg installer");
1413             let pkg = tmp.join("pkg");
1414             let _ = fs::remove_dir_all(&pkg);
1415
1416             let pkgbuild = |component: &str| {
1417                 let mut cmd = Command::new("pkgbuild");
1418                 cmd.arg("--identifier")
1419                     .arg(format!("org.rust-lang.{}", component))
1420                     .arg("--scripts")
1421                     .arg(pkg.join(component))
1422                     .arg("--nopayload")
1423                     .arg(pkg.join(component).with_extension("pkg"));
1424                 builder.run(&mut cmd);
1425             };
1426
1427             let prepare = |name: &str| {
1428                 builder.create_dir(&pkg.join(name));
1429                 builder.cp_r(
1430                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1431                     &pkg.join(name),
1432                 );
1433                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1434                 pkgbuild(name);
1435             };
1436             prepare("rustc");
1437             prepare("cargo");
1438             prepare("rust-std");
1439             prepare("rust-analysis");
1440             prepare("clippy");
1441             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1442                 if built_tools.contains(tool) {
1443                     prepare(tool);
1444                 }
1445             }
1446             // create an 'uninstall' package
1447             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1448             pkgbuild("uninstall");
1449
1450             builder.create_dir(&pkg.join("res"));
1451             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1452             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1453             let mut cmd = Command::new("productbuild");
1454             cmd.arg("--distribution")
1455                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1456                 .arg("--resources")
1457                 .arg(pkg.join("res"))
1458                 .arg(distdir(builder).join(format!(
1459                     "{}-{}.pkg",
1460                     pkgname(builder, "rust"),
1461                     target.triple
1462                 )))
1463                 .arg("--package-path")
1464                 .arg(&pkg);
1465             let _time = timeit(builder);
1466             builder.run(&mut cmd);
1467         }
1468
1469         if target.contains("windows") {
1470             let exe = tmp.join("exe");
1471             let _ = fs::remove_dir_all(&exe);
1472
1473             let prepare = |name: &str| {
1474                 builder.create_dir(&exe.join(name));
1475                 let dir = if name == "rust-std" || name == "rust-analysis" {
1476                     format!("{}-{}", name, target.triple)
1477                 } else if name == "rls" {
1478                     "rls-preview".to_string()
1479                 } else if name == "rust-analyzer" {
1480                     "rust-analyzer-preview".to_string()
1481                 } else if name == "clippy" {
1482                     "clippy-preview".to_string()
1483                 } else if name == "rust-demangler" {
1484                     "rust-demangler-preview".to_string()
1485                 } else if name == "miri" {
1486                     "miri-preview".to_string()
1487                 } else {
1488                     name.to_string()
1489                 };
1490                 builder.cp_r(
1491                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1492                     &exe.join(name),
1493                 );
1494                 builder.remove(&exe.join(name).join("manifest.in"));
1495             };
1496             prepare("rustc");
1497             prepare("cargo");
1498             prepare("rust-analysis");
1499             prepare("rust-docs");
1500             prepare("rust-std");
1501             prepare("clippy");
1502             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1503                 if built_tools.contains(tool) {
1504                     prepare(tool);
1505                 }
1506             }
1507             if target.ends_with("windows-gnu") {
1508                 prepare("rust-mingw");
1509             }
1510
1511             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1512
1513             // Generate msi installer
1514             let wix_path = env::var_os("WIX")
1515                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1516             let wix = PathBuf::from(wix_path);
1517             let heat = wix.join("bin/heat.exe");
1518             let candle = wix.join("bin/candle.exe");
1519             let light = wix.join("bin/light.exe");
1520
1521             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1522             builder.run(
1523                 Command::new(&heat)
1524                     .current_dir(&exe)
1525                     .arg("dir")
1526                     .arg("rustc")
1527                     .args(&heat_flags)
1528                     .arg("-cg")
1529                     .arg("RustcGroup")
1530                     .arg("-dr")
1531                     .arg("Rustc")
1532                     .arg("-var")
1533                     .arg("var.RustcDir")
1534                     .arg("-out")
1535                     .arg(exe.join("RustcGroup.wxs")),
1536             );
1537             builder.run(
1538                 Command::new(&heat)
1539                     .current_dir(&exe)
1540                     .arg("dir")
1541                     .arg("rust-docs")
1542                     .args(&heat_flags)
1543                     .arg("-cg")
1544                     .arg("DocsGroup")
1545                     .arg("-dr")
1546                     .arg("Docs")
1547                     .arg("-var")
1548                     .arg("var.DocsDir")
1549                     .arg("-out")
1550                     .arg(exe.join("DocsGroup.wxs"))
1551                     .arg("-t")
1552                     .arg(etc.join("msi/squash-components.xsl")),
1553             );
1554             builder.run(
1555                 Command::new(&heat)
1556                     .current_dir(&exe)
1557                     .arg("dir")
1558                     .arg("cargo")
1559                     .args(&heat_flags)
1560                     .arg("-cg")
1561                     .arg("CargoGroup")
1562                     .arg("-dr")
1563                     .arg("Cargo")
1564                     .arg("-var")
1565                     .arg("var.CargoDir")
1566                     .arg("-out")
1567                     .arg(exe.join("CargoGroup.wxs"))
1568                     .arg("-t")
1569                     .arg(etc.join("msi/remove-duplicates.xsl")),
1570             );
1571             builder.run(
1572                 Command::new(&heat)
1573                     .current_dir(&exe)
1574                     .arg("dir")
1575                     .arg("rust-std")
1576                     .args(&heat_flags)
1577                     .arg("-cg")
1578                     .arg("StdGroup")
1579                     .arg("-dr")
1580                     .arg("Std")
1581                     .arg("-var")
1582                     .arg("var.StdDir")
1583                     .arg("-out")
1584                     .arg(exe.join("StdGroup.wxs")),
1585             );
1586             if built_tools.contains("rls") {
1587                 builder.run(
1588                     Command::new(&heat)
1589                         .current_dir(&exe)
1590                         .arg("dir")
1591                         .arg("rls")
1592                         .args(&heat_flags)
1593                         .arg("-cg")
1594                         .arg("RlsGroup")
1595                         .arg("-dr")
1596                         .arg("Rls")
1597                         .arg("-var")
1598                         .arg("var.RlsDir")
1599                         .arg("-out")
1600                         .arg(exe.join("RlsGroup.wxs"))
1601                         .arg("-t")
1602                         .arg(etc.join("msi/remove-duplicates.xsl")),
1603                 );
1604             }
1605             if built_tools.contains("rust-analyzer") {
1606                 builder.run(
1607                     Command::new(&heat)
1608                         .current_dir(&exe)
1609                         .arg("dir")
1610                         .arg("rust-analyzer")
1611                         .args(&heat_flags)
1612                         .arg("-cg")
1613                         .arg("RustAnalyzerGroup")
1614                         .arg("-dr")
1615                         .arg("RustAnalyzer")
1616                         .arg("-var")
1617                         .arg("var.RustAnalyzerDir")
1618                         .arg("-out")
1619                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1620                         .arg("-t")
1621                         .arg(etc.join("msi/remove-duplicates.xsl")),
1622                 );
1623             }
1624             builder.run(
1625                 Command::new(&heat)
1626                     .current_dir(&exe)
1627                     .arg("dir")
1628                     .arg("clippy")
1629                     .args(&heat_flags)
1630                     .arg("-cg")
1631                     .arg("ClippyGroup")
1632                     .arg("-dr")
1633                     .arg("Clippy")
1634                     .arg("-var")
1635                     .arg("var.ClippyDir")
1636                     .arg("-out")
1637                     .arg(exe.join("ClippyGroup.wxs"))
1638                     .arg("-t")
1639                     .arg(etc.join("msi/remove-duplicates.xsl")),
1640             );
1641             if built_tools.contains("rust-demangler") {
1642                 builder.run(
1643                     Command::new(&heat)
1644                         .current_dir(&exe)
1645                         .arg("dir")
1646                         .arg("rust-demangler")
1647                         .args(&heat_flags)
1648                         .arg("-cg")
1649                         .arg("RustDemanglerGroup")
1650                         .arg("-dr")
1651                         .arg("RustDemangler")
1652                         .arg("-var")
1653                         .arg("var.RustDemanglerDir")
1654                         .arg("-out")
1655                         .arg(exe.join("RustDemanglerGroup.wxs"))
1656                         .arg("-t")
1657                         .arg(etc.join("msi/remove-duplicates.xsl")),
1658                 );
1659             }
1660             if built_tools.contains("miri") {
1661                 builder.run(
1662                     Command::new(&heat)
1663                         .current_dir(&exe)
1664                         .arg("dir")
1665                         .arg("miri")
1666                         .args(&heat_flags)
1667                         .arg("-cg")
1668                         .arg("MiriGroup")
1669                         .arg("-dr")
1670                         .arg("Miri")
1671                         .arg("-var")
1672                         .arg("var.MiriDir")
1673                         .arg("-out")
1674                         .arg(exe.join("MiriGroup.wxs"))
1675                         .arg("-t")
1676                         .arg(etc.join("msi/remove-duplicates.xsl")),
1677                 );
1678             }
1679             builder.run(
1680                 Command::new(&heat)
1681                     .current_dir(&exe)
1682                     .arg("dir")
1683                     .arg("rust-analysis")
1684                     .args(&heat_flags)
1685                     .arg("-cg")
1686                     .arg("AnalysisGroup")
1687                     .arg("-dr")
1688                     .arg("Analysis")
1689                     .arg("-var")
1690                     .arg("var.AnalysisDir")
1691                     .arg("-out")
1692                     .arg(exe.join("AnalysisGroup.wxs"))
1693                     .arg("-t")
1694                     .arg(etc.join("msi/remove-duplicates.xsl")),
1695             );
1696             if target.ends_with("windows-gnu") {
1697                 builder.run(
1698                     Command::new(&heat)
1699                         .current_dir(&exe)
1700                         .arg("dir")
1701                         .arg("rust-mingw")
1702                         .args(&heat_flags)
1703                         .arg("-cg")
1704                         .arg("GccGroup")
1705                         .arg("-dr")
1706                         .arg("Gcc")
1707                         .arg("-var")
1708                         .arg("var.GccDir")
1709                         .arg("-out")
1710                         .arg(exe.join("GccGroup.wxs")),
1711                 );
1712             }
1713
1714             let candle = |input: &Path| {
1715                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1716                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1717                 let mut cmd = Command::new(&candle);
1718                 cmd.current_dir(&exe)
1719                     .arg("-nologo")
1720                     .arg("-dRustcDir=rustc")
1721                     .arg("-dDocsDir=rust-docs")
1722                     .arg("-dCargoDir=cargo")
1723                     .arg("-dStdDir=rust-std")
1724                     .arg("-dAnalysisDir=rust-analysis")
1725                     .arg("-dClippyDir=clippy")
1726                     .arg("-arch")
1727                     .arg(&arch)
1728                     .arg("-out")
1729                     .arg(&output)
1730                     .arg(&input);
1731                 add_env(builder, &mut cmd, target);
1732
1733                 if built_tools.contains("rust-demangler") {
1734                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1735                 }
1736                 if built_tools.contains("rls") {
1737                     cmd.arg("-dRlsDir=rls");
1738                 }
1739                 if built_tools.contains("rust-analyzer") {
1740                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1741                 }
1742                 if built_tools.contains("miri") {
1743                     cmd.arg("-dMiriDir=miri");
1744                 }
1745                 if target.ends_with("windows-gnu") {
1746                     cmd.arg("-dGccDir=rust-mingw");
1747                 }
1748                 builder.run(&mut cmd);
1749             };
1750             candle(&xform(&etc.join("msi/rust.wxs")));
1751             candle(&etc.join("msi/ui.wxs"));
1752             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1753             candle("RustcGroup.wxs".as_ref());
1754             candle("DocsGroup.wxs".as_ref());
1755             candle("CargoGroup.wxs".as_ref());
1756             candle("StdGroup.wxs".as_ref());
1757             candle("ClippyGroup.wxs".as_ref());
1758             if built_tools.contains("rust-demangler") {
1759                 candle("RustDemanglerGroup.wxs".as_ref());
1760             }
1761             if built_tools.contains("rls") {
1762                 candle("RlsGroup.wxs".as_ref());
1763             }
1764             if built_tools.contains("rust-analyzer") {
1765                 candle("RustAnalyzerGroup.wxs".as_ref());
1766             }
1767             if built_tools.contains("miri") {
1768                 candle("MiriGroup.wxs".as_ref());
1769             }
1770             candle("AnalysisGroup.wxs".as_ref());
1771
1772             if target.ends_with("windows-gnu") {
1773                 candle("GccGroup.wxs".as_ref());
1774             }
1775
1776             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1777             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1778             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1779
1780             builder.info(&format!("building `msi` installer with {:?}", light));
1781             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1782             let mut cmd = Command::new(&light);
1783             cmd.arg("-nologo")
1784                 .arg("-ext")
1785                 .arg("WixUIExtension")
1786                 .arg("-ext")
1787                 .arg("WixUtilExtension")
1788                 .arg("-out")
1789                 .arg(exe.join(&filename))
1790                 .arg("rust.wixobj")
1791                 .arg("ui.wixobj")
1792                 .arg("rustwelcomedlg.wixobj")
1793                 .arg("RustcGroup.wixobj")
1794                 .arg("DocsGroup.wixobj")
1795                 .arg("CargoGroup.wixobj")
1796                 .arg("StdGroup.wixobj")
1797                 .arg("AnalysisGroup.wixobj")
1798                 .arg("ClippyGroup.wixobj")
1799                 .current_dir(&exe);
1800
1801             if built_tools.contains("rls") {
1802                 cmd.arg("RlsGroup.wixobj");
1803             }
1804             if built_tools.contains("rust-analyzer") {
1805                 cmd.arg("RustAnalyzerGroup.wixobj");
1806             }
1807             if built_tools.contains("rust-demangler") {
1808                 cmd.arg("RustDemanglerGroup.wixobj");
1809             }
1810             if built_tools.contains("miri") {
1811                 cmd.arg("MiriGroup.wixobj");
1812             }
1813
1814             if target.ends_with("windows-gnu") {
1815                 cmd.arg("GccGroup.wixobj");
1816             }
1817             // ICE57 wrongly complains about the shortcuts
1818             cmd.arg("-sice:ICE57");
1819
1820             let _time = timeit(builder);
1821             builder.run(&mut cmd);
1822
1823             if !builder.config.dry_run {
1824                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1825             }
1826         }
1827     }
1828 }
1829
1830 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1831     let mut parts = builder.version.split('.');
1832     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1833         .env("CFG_RELEASE_NUM", &builder.version)
1834         .env("CFG_RELEASE", builder.rust_release())
1835         .env("CFG_VER_MAJOR", parts.next().unwrap())
1836         .env("CFG_VER_MINOR", parts.next().unwrap())
1837         .env("CFG_VER_PATCH", parts.next().unwrap())
1838         .env("CFG_VER_BUILD", "0") // just needed to build
1839         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1840         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1841         .env("CFG_BUILD", target.triple)
1842         .env("CFG_CHANNEL", &builder.config.channel);
1843
1844     if target.contains("windows-gnullvm") {
1845         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1846     } else if target.contains("windows-gnu") {
1847         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1848     } else {
1849         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1850     }
1851 }
1852
1853 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1854 ///
1855 /// Returns whether the files were actually copied.
1856 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1857     if let Some(config) = builder.config.target_config.get(&target) {
1858         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1859             // If the LLVM was externally provided, then we don't currently copy
1860             // artifacts into the sysroot. This is not necessarily the right
1861             // choice (in particular, it will require the LLVM dylib to be in
1862             // the linker's load path at runtime), but the common use case for
1863             // external LLVMs is distribution provided LLVMs, and in that case
1864             // they're usually in the standard search path (e.g., /usr/lib) and
1865             // copying them here is going to cause problems as we may end up
1866             // with the wrong files and isn't what distributions want.
1867             //
1868             // This behavior may be revisited in the future though.
1869             //
1870             // If the LLVM is coming from ourselves (just from CI) though, we
1871             // still want to install it, as it otherwise won't be available.
1872             return false;
1873         }
1874     }
1875
1876     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1877     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1878     // clear why this is the case, though. llvm-config will emit the versioned
1879     // paths and we don't want those in the sysroot (as we're expecting
1880     // unversioned paths).
1881     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1882         let src_libdir = builder.llvm_out(target).join("lib");
1883         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1884         if llvm_dylib_path.exists() {
1885             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1886         }
1887         !builder.config.dry_run
1888     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1889         let mut cmd = Command::new(llvm_config);
1890         cmd.arg("--libfiles");
1891         builder.verbose(&format!("running {:?}", cmd));
1892         let files = output(&mut cmd);
1893         let build_llvm_out = &builder.llvm_out(builder.config.build);
1894         let target_llvm_out = &builder.llvm_out(target);
1895         for file in files.trim_end().split(' ') {
1896             // If we're not using a custom LLVM, make sure we package for the target.
1897             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1898                 target_llvm_out.join(relative_path)
1899             } else {
1900                 PathBuf::from(file)
1901             };
1902             builder.install(&file, dst_libdir, 0o644);
1903         }
1904         !builder.config.dry_run
1905     } else {
1906         false
1907     }
1908 }
1909
1910 /// Maybe add libLLVM.so to the target lib-dir for linking.
1911 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1912     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1913     // We do not need to copy LLVM files into the sysroot if it is not
1914     // dynamically linked; it is already included into librustc_llvm
1915     // statically.
1916     if builder.llvm_link_shared() {
1917         maybe_install_llvm(builder, target, &dst_libdir);
1918     }
1919 }
1920
1921 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1922 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1923     let dst_libdir =
1924         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1925     // We do not need to copy LLVM files into the sysroot if it is not
1926     // dynamically linked; it is already included into librustc_llvm
1927     // statically.
1928     if builder.llvm_link_shared() {
1929         maybe_install_llvm(builder, target, &dst_libdir);
1930     }
1931 }
1932
1933 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1934 pub struct LlvmTools {
1935     pub target: TargetSelection,
1936 }
1937
1938 impl Step for LlvmTools {
1939     type Output = Option<GeneratedTarball>;
1940     const ONLY_HOSTS: bool = true;
1941     const DEFAULT: bool = true;
1942
1943     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1944         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1945         // FIXME: allow using the names of the tools themselves?
1946         run.alias("llvm-tools").default_condition(default)
1947     }
1948
1949     fn make_run(run: RunConfig<'_>) {
1950         run.builder.ensure(LlvmTools { target: run.target });
1951     }
1952
1953     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1954         let target = self.target;
1955
1956         /* run only if llvm-config isn't used */
1957         if let Some(config) = builder.config.target_config.get(&target) {
1958             if let Some(ref _s) = config.llvm_config {
1959                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1960                 return None;
1961             }
1962         }
1963
1964         builder.ensure(crate::native::Llvm { target });
1965
1966         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1967         tarball.set_overlay(OverlayKind::LLVM);
1968         tarball.is_preview(true);
1969
1970         // Prepare the image directory
1971         let src_bindir = builder.llvm_out(target).join("bin");
1972         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1973         for tool in LLVM_TOOLS {
1974             let exe = src_bindir.join(exe(tool, target));
1975             tarball.add_file(&exe, &dst_bindir, 0o755);
1976         }
1977
1978         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1979         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1980         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1981         // compiler libraries.
1982         maybe_install_llvm_target(builder, target, tarball.image_dir());
1983
1984         Some(tarball.generate())
1985     }
1986 }
1987
1988 // Tarball intended for internal consumption to ease rustc/std development.
1989 //
1990 // Should not be considered stable by end users.
1991 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1992 pub struct RustDev {
1993     pub target: TargetSelection,
1994 }
1995
1996 impl Step for RustDev {
1997     type Output = Option<GeneratedTarball>;
1998     const DEFAULT: bool = true;
1999     const ONLY_HOSTS: bool = true;
2000
2001     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2002         run.alias("rust-dev")
2003     }
2004
2005     fn make_run(run: RunConfig<'_>) {
2006         run.builder.ensure(RustDev { target: run.target });
2007     }
2008
2009     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2010         let target = self.target;
2011
2012         /* run only if llvm-config isn't used */
2013         if let Some(config) = builder.config.target_config.get(&target) {
2014             if let Some(ref _s) = config.llvm_config {
2015                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2016                 return None;
2017             }
2018         }
2019
2020         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2021         tarball.set_overlay(OverlayKind::LLVM);
2022
2023         let src_bindir = builder.llvm_out(target).join("bin");
2024         // If updating this list, you likely want to change
2025         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2026         // will not pick up the extra file until LLVM gets bumped.
2027         for bin in &[
2028             "llvm-config",
2029             "llvm-ar",
2030             "llvm-objdump",
2031             "llvm-profdata",
2032             "llvm-bcanalyzer",
2033             "llvm-cov",
2034             "llvm-dwp",
2035             "llvm-nm",
2036             "llvm-dwarfdump",
2037         ] {
2038             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2039         }
2040
2041         // We don't build LLD on some platforms, so only add it if it exists
2042         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2043         if lld_path.exists() {
2044             tarball.add_file(lld_path, "bin", 0o755);
2045         }
2046
2047         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2048
2049         // Copy the include directory as well; needed mostly to build
2050         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2051         // just broadly useful to be able to link against the bundled LLVM.
2052         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2053
2054         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2055         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2056         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2057         // compiler libraries.
2058         let dst_libdir = tarball.image_dir().join("lib");
2059         maybe_install_llvm(builder, target, &dst_libdir);
2060         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2061         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2062
2063         Some(tarball.generate())
2064     }
2065 }
2066
2067 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2068 /// release process to avoid cloning the monorepo and building stuff.
2069 ///
2070 /// Should not be considered stable by end users.
2071 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2072 pub struct BuildManifest {
2073     pub target: TargetSelection,
2074 }
2075
2076 impl Step for BuildManifest {
2077     type Output = GeneratedTarball;
2078     const DEFAULT: bool = false;
2079     const ONLY_HOSTS: bool = true;
2080
2081     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2082         run.alias("build-manifest")
2083     }
2084
2085     fn make_run(run: RunConfig<'_>) {
2086         run.builder.ensure(BuildManifest { target: run.target });
2087     }
2088
2089     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2090         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2091
2092         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2093         tarball.add_file(&build_manifest, "bin", 0o755);
2094         tarball.generate()
2095     }
2096 }
2097
2098 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2099 ///
2100 /// Currently this is the PGO profile data.
2101 ///
2102 /// Should not be considered stable by end users.
2103 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2104 pub struct ReproducibleArtifacts {
2105     pub target: TargetSelection,
2106 }
2107
2108 impl Step for ReproducibleArtifacts {
2109     type Output = Option<GeneratedTarball>;
2110     const DEFAULT: bool = true;
2111     const ONLY_HOSTS: bool = true;
2112
2113     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2114         run.alias("reproducible-artifacts")
2115     }
2116
2117     fn make_run(run: RunConfig<'_>) {
2118         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2119     }
2120
2121     fn run(self, builder: &Builder<'_>) -> Self::Output {
2122         let mut added_anything = false;
2123         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2124         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2125             tarball.add_file(path, ".", 0o644);
2126             added_anything = true;
2127         }
2128         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2129             tarball.add_file(path, ".", 0o644);
2130             added_anything = true;
2131         }
2132         if added_anything { Some(tarball.generate()) } else { None }
2133     }
2134 }