src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.alias("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.alias("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.ends_with("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.alias("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.ends_with("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 let exe_name = exe("ld", compiler.host);
 411                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 412             }
 413
 414             // Man pages
 415             t!(fs::create_dir_all(image.join("share/man/man1")));
 416             let man_src = builder.src.join("src/doc/man");
 417             let man_dst = image.join("share/man/man1");
 418
 419             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 420             // to hardlink, and we don't want to edit the source templates
 421             for file_entry in builder.read_dir(&man_src) {
 422                 let page_src = file_entry.path();
 423                 let page_dst = man_dst.join(file_entry.file_name());
 424                 let src_text = t!(std::fs::read_to_string(&page_src));
 425                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 426                 t!(std::fs::write(&page_dst, &new_text));
 427                 t!(fs::copy(&page_src, &page_dst));
 428             }
 429
 430             // Debugger scripts
 431             builder
 432                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 433
 434             // Misc license info
 435             let cp = |file: &str| {
 436                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 437             };
 438             cp("COPYRIGHT");
 439             cp("LICENSE-APACHE");
 440             cp("LICENSE-MIT");
 441             cp("README.md");
 442         }
 443     }
 444 }
 445
 446 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 447 pub struct DebuggerScripts {
 448     pub sysroot: Interned<PathBuf>,
 449     pub host: TargetSelection,
 450 }
 451
 452 impl Step for DebuggerScripts {
 453     type Output = ();
 454
 455     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 456         run.never()
 457     }
 458
 459     /// Copies debugger scripts for `target` into the `sysroot` specified.
 460     fn run(self, builder: &Builder<'_>) {
 461         let host = self.host;
 462         let sysroot = self.sysroot;
 463         let dst = sysroot.join("lib/rustlib/etc");
 464         t!(fs::create_dir_all(&dst));
 465         let cp_debugger_script = |file: &str| {
 466             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 467         };
 468         if host.contains("windows-msvc") {
 469             // windbg debugger scripts
 470             builder.install(
 471                 &builder.src.join("src/etc/rust-windbg.cmd"),
 472                 &sysroot.join("bin"),
 473                 0o755,
 474             );
 475
 476             cp_debugger_script("natvis/intrinsic.natvis");
 477             cp_debugger_script("natvis/liballoc.natvis");
 478             cp_debugger_script("natvis/libcore.natvis");
 479             cp_debugger_script("natvis/libstd.natvis");
 480         } else {
 481             cp_debugger_script("rust_types.py");
 482
 483             // gdb debugger scripts
 484             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 485             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 486
 487             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 488             cp_debugger_script("gdb_lookup.py");
 489             cp_debugger_script("gdb_providers.py");
 490
 491             // lldb debugger scripts
 492             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 493
 494             cp_debugger_script("lldb_lookup.py");
 495             cp_debugger_script("lldb_providers.py");
 496             cp_debugger_script("lldb_commands")
 497         }
 498     }
 499 }
 500
 501 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 502     // The only true set of target libraries came from the build triple, so
 503     // let's reduce redundant work by only producing archives from that host.
 504     if compiler.host != builder.config.build {
 505         builder.info("\tskipping, not a build host");
 506         true
 507     } else {
 508         false
 509     }
 510 }
 511
 512 /// Copy stamped files into an image's `target/lib` directory.
 513 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 514     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 515     let self_contained_dst = dst.join("self-contained");
 516     t!(fs::create_dir_all(&dst));
 517     t!(fs::create_dir_all(&self_contained_dst));
 518     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 519         if dependency_type == DependencyType::TargetSelfContained {
 520             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 521         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 522             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 523         }
 524     }
 525 }
 526
 527 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 528 pub struct Std {
 529     pub compiler: Compiler,
 530     pub target: TargetSelection,
 531 }
 532
 533 impl Step for Std {
 534     type Output = Option<GeneratedTarball>;
 535     const DEFAULT: bool = true;
 536
 537     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 538         run.alias("rust-std")
 539     }
 540
 541     fn make_run(run: RunConfig<'_>) {
 542         run.builder.ensure(Std {
 543             compiler: run.builder.compiler_for(
 544                 run.builder.top_stage,
 545                 run.builder.config.build,
 546                 run.target,
 547             ),
 548             target: run.target,
 549         });
 550     }
 551
 552     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 553         let compiler = self.compiler;
 554         let target = self.target;
 555
 556         if skip_host_target_lib(builder, compiler) {
 557             return None;
 558         }
 559
 560         builder.ensure(compile::Std { compiler, target });
 561
 562         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 563         tarball.include_target_in_component_name(true);
 564
 565         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 566         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 567         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 568
 569         Some(tarball.generate())
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct RustcDev {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for RustcDev {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582     const ONLY_HOSTS: bool = true;
 583
 584     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 585         run.alias("rustc-dev")
 586     }
 587
 588     fn make_run(run: RunConfig<'_>) {
 589         run.builder.ensure(RustcDev {
 590             compiler: run.builder.compiler_for(
 591                 run.builder.top_stage,
 592                 run.builder.config.build,
 593                 run.target,
 594             ),
 595             target: run.target,
 596         });
 597     }
 598
 599     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 600         let compiler = self.compiler;
 601         let target = self.target;
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Rustc { compiler, target });
 607
 608         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 609
 610         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 611         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 612         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 613
 614         let src_files = &["Cargo.lock"];
 615         // This is the reduced set of paths which will become the rustc-dev component
 616         // (essentially the compiler crates and all of their path dependencies).
 617         copy_src_dirs(
 618             builder,
 619             &builder.src,
 620             &["compiler"],
 621             &[],
 622             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 623         );
 624         for file in src_files {
 625             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 626         }
 627
 628         Some(tarball.generate())
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: TargetSelection,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = Option<GeneratedTarball>;
 640     const DEFAULT: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         let default = should_build_extended_tool(&run.builder, "analysis");
 644         run.alias("rust-analysis").default_condition(default)
 645     }
 646
 647     fn make_run(run: RunConfig<'_>) {
 648         run.builder.ensure(Analysis {
 649             // Find the actual compiler (handling the full bootstrap option) which
 650             // produced the save-analysis data because that data isn't copied
 651             // through the sysroot uplifting.
 652             compiler: run.builder.compiler_for(
 653                 run.builder.top_stage,
 654                 run.builder.config.build,
 655                 run.target,
 656             ),
 657             target: run.target,
 658         });
 659     }
 660
 661     /// Creates a tarball of save-analysis metadata, if available.
 662     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 663         let compiler = self.compiler;
 664         let target = self.target;
 665         if compiler.host != builder.config.build {
 666             return None;
 667         }
 668
 669         builder.ensure(compile::Std { compiler, target });
 670         let src = builder
 671             .stage_out(compiler, Mode::Std)
 672             .join(target.triple)
 673             .join(builder.cargo_dir())
 674             .join("deps")
 675             .join("save-analysis");
 676
 677         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 678         tarball.include_target_in_component_name(true);
 679         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 680         Some(tarball.generate())
 681     }
 682 }
 683
 684 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 685 /// `dst_dir`.
 686 fn copy_src_dirs(
 687     builder: &Builder<'_>,
 688     base: &Path,
 689     src_dirs: &[&str],
 690     exclude_dirs: &[&str],
 691     dst_dir: &Path,
 692 ) {
 693     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 694         let spath = match path.to_str() {
 695             Some(path) => path,
 696             None => return false,
 697         };
 698         if spath.ends_with('~') || spath.ends_with(".pyc") {
 699             return false;
 700         }
 701
 702         const LLVM_PROJECTS: &[&str] = &[
 703             "llvm-project/clang",
 704             "llvm-project\\clang",
 705             "llvm-project/libunwind",
 706             "llvm-project\\libunwind",
 707             "llvm-project/lld",
 708             "llvm-project\\lld",
 709             "llvm-project/lldb",
 710             "llvm-project\\lldb",
 711             "llvm-project/llvm",
 712             "llvm-project\\llvm",
 713             "llvm-project/compiler-rt",
 714             "llvm-project\\compiler-rt",
 715             "llvm-project/cmake",
 716             "llvm-project\\cmake",
 717         ];
 718         if spath.contains("llvm-project")
 719             && !spath.ends_with("llvm-project")
 720             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 721         {
 722             return false;
 723         }
 724
 725         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 726         if LLVM_TEST.iter().any(|path| spath.contains(path))
 727             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 728         {
 729             return false;
 730         }
 731
 732         let full_path = Path::new(dir).join(path);
 733         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 734             return false;
 735         }
 736
 737         let excludes = [
 738             "CVS",
 739             "RCS",
 740             "SCCS",
 741             ".git",
 742             ".gitignore",
 743             ".gitmodules",
 744             ".gitattributes",
 745             ".cvsignore",
 746             ".svn",
 747             ".arch-ids",
 748             "{arch}",
 749             "=RELEASE-ID",
 750             "=meta-update",
 751             "=update",
 752             ".bzr",
 753             ".bzrignore",
 754             ".bzrtags",
 755             ".hg",
 756             ".hgignore",
 757             ".hgrags",
 758             "_darcs",
 759         ];
 760         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 761     }
 762
 763     // Copy the directories using our filter
 764     for item in src_dirs {
 765         let dst = &dst_dir.join(item);
 766         t!(fs::create_dir_all(dst));
 767         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 768     }
 769 }
 770
 771 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 772 pub struct Src;
 773
 774 impl Step for Src {
 775     /// The output path of the src installer tarball
 776     type Output = GeneratedTarball;
 777     const DEFAULT: bool = true;
 778     const ONLY_HOSTS: bool = true;
 779
 780     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 781         run.alias("rust-src")
 782     }
 783
 784     fn make_run(run: RunConfig<'_>) {
 785         run.builder.ensure(Src);
 786     }
 787
 788     /// Creates the `rust-src` installer component
 789     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 790         let tarball = Tarball::new_targetless(builder, "rust-src");
 791
 792         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 793         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 794         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 795         // and fix them...
 796         //
 797         // NOTE: if you update the paths here, you also should update the "virtual" path
 798         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 799         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 800
 801         let src_files = ["Cargo.lock"];
 802         // This is the reduced set of paths which will become the rust-src component
 803         // (essentially libstd and all of its path dependencies).
 804         copy_src_dirs(
 805             builder,
 806             &builder.src,
 807             &["library", "src/llvm-project/libunwind"],
 808             &[
 809                 // not needed and contains symlinks which rustup currently
 810                 // chokes on when unpacking.
 811                 "library/backtrace/crates",
 812                 // these are 30MB combined and aren't necessary for building
 813                 // the standard library.
 814                 "library/stdarch/Cargo.toml",
 815                 "library/stdarch/crates/stdarch-verify",
 816                 "library/stdarch/crates/intrinsic-test",
 817             ],
 818             &dst_src,
 819         );
 820         for file in src_files.iter() {
 821             builder.copy(&builder.src.join(file), &dst_src.join(file));
 822         }
 823
 824         tarball.generate()
 825     }
 826 }
 827
 828 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 829 pub struct PlainSourceTarball;
 830
 831 impl Step for PlainSourceTarball {
 832     /// Produces the location of the tarball generated
 833     type Output = GeneratedTarball;
 834     const DEFAULT: bool = true;
 835     const ONLY_HOSTS: bool = true;
 836
 837     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 838         let builder = run.builder;
 839         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig<'_>) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 848         let tarball = Tarball::new(builder, "rustc", "src");
 849         let plain_dst_src = tarball.image_dir();
 850
 851         // This is the set of root paths which will become part of the source package
 852         let src_files = [
 853             "COPYRIGHT",
 854             "LICENSE-APACHE",
 855             "LICENSE-MIT",
 856             "CONTRIBUTING.md",
 857             "README.md",
 858             "RELEASES.md",
 859             "configure",
 860             "x.py",
 861             "config.toml.example",
 862             "Cargo.toml",
 863             "Cargo.lock",
 864         ];
 865         let src_dirs = ["src", "compiler", "library"];
 866
 867         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 868
 869         // Copy the files normally
 870         for item in &src_files {
 871             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 872         }
 873
 874         // Create the version file
 875         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 876         if let Some(sha) = builder.rust_sha() {
 877             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 878         }
 879
 880         // If we're building from git sources, we need to vendor a complete distribution.
 881         if builder.rust_info.is_git() {
 882             // Vendor all Cargo dependencies
 883             let mut cmd = Command::new(&builder.initial_cargo);
 884             cmd.arg("vendor")
 885                 .arg("--sync")
 886                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 887                 .arg("--sync")
 888                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 889                 .current_dir(&plain_dst_src);
 890             builder.run(&mut cmd);
 891         }
 892
 893         tarball.bare()
 894     }
 895 }
 896
 897 // We have to run a few shell scripts, which choke quite a bit on both `\`
 898 // characters and on `C:\` paths, so normalize both of them away.
 899 pub fn sanitize_sh(path: &Path) -> String {
 900     let path = path.to_str().unwrap().replace("\\", "/");
 901     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 902
 903     fn unc_to_lfs(s: &str) -> &str {
 904         s.strip_prefix("//?/").unwrap_or(s)
 905     }
 906
 907     fn change_drive(s: &str) -> Option<String> {
 908         let mut ch = s.chars();
 909         let drive = ch.next().unwrap_or('C');
 910         if ch.next() != Some(':') {
 911             return None;
 912         }
 913         if ch.next() != Some('/') {
 914             return None;
 915         }
 916         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 917     }
 918 }
 919
 920 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 921 pub struct Cargo {
 922     pub compiler: Compiler,
 923     pub target: TargetSelection,
 924 }
 925
 926 impl Step for Cargo {
 927     type Output = Option<GeneratedTarball>;
 928     const DEFAULT: bool = true;
 929     const ONLY_HOSTS: bool = true;
 930
 931     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 932         let default = should_build_extended_tool(&run.builder, "cargo");
 933         run.alias("cargo").default_condition(default)
 934     }
 935
 936     fn make_run(run: RunConfig<'_>) {
 937         run.builder.ensure(Cargo {
 938             compiler: run.builder.compiler_for(
 939                 run.builder.top_stage,
 940                 run.builder.config.build,
 941                 run.target,
 942             ),
 943             target: run.target,
 944         });
 945     }
 946
 947     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 948         let compiler = self.compiler;
 949         let target = self.target;
 950
 951         let cargo = builder.ensure(tool::Cargo { compiler, target });
 952         let src = builder.src.join("src/tools/cargo");
 953         let etc = src.join("src/etc");
 954
 955         // Prepare the image directory
 956         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 957         tarball.set_overlay(OverlayKind::Cargo);
 958
 959         tarball.add_file(&cargo, "bin", 0o755);
 960         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 961         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 962         tarball.add_dir(etc.join("man"), "share/man/man1");
 963         tarball.add_legal_and_readme_to("share/doc/cargo");
 964
 965         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 966             let dirent = dirent.expect("read dir entry");
 967             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 968                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 969             }
 970         }
 971
 972         Some(tarball.generate())
 973     }
 974 }
 975
 976 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 977 pub struct Rls {
 978     pub compiler: Compiler,
 979     pub target: TargetSelection,
 980 }
 981
 982 impl Step for Rls {
 983     type Output = Option<GeneratedTarball>;
 984     const ONLY_HOSTS: bool = true;
 985     const DEFAULT: bool = true;
 986
 987     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 988         let default = should_build_extended_tool(&run.builder, "rls");
 989         run.alias("rls").default_condition(default)
 990     }
 991
 992     fn make_run(run: RunConfig<'_>) {
 993         run.builder.ensure(Rls {
 994             compiler: run.builder.compiler_for(
 995                 run.builder.top_stage,
 996                 run.builder.config.build,
 997                 run.target,
 998             ),
 999             target: run.target,
1000         });
1001     }
1002
1003     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1004         let compiler = self.compiler;
1005         let target = self.target;
1006
1007         let rls = builder
1008             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1009             .or_else(|| {
1010                 missing_tool("RLS", builder.build.config.missing_tools);
1011                 None
1012             })?;
1013
1014         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1015         tarball.set_overlay(OverlayKind::RLS);
1016         tarball.is_preview(true);
1017         tarball.add_file(rls, "bin", 0o755);
1018         tarball.add_legal_and_readme_to("share/doc/rls");
1019         Some(tarball.generate())
1020     }
1021 }
1022
1023 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1024 pub struct RustAnalyzer {
1025     pub compiler: Compiler,
1026     pub target: TargetSelection,
1027 }
1028
1029 impl Step for RustAnalyzer {
1030     type Output = Option<GeneratedTarball>;
1031     const DEFAULT: bool = true;
1032     const ONLY_HOSTS: bool = true;
1033
1034     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1035         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1036         run.alias("rust-analyzer").default_condition(default)
1037     }
1038
1039     fn make_run(run: RunConfig<'_>) {
1040         run.builder.ensure(RustAnalyzer {
1041             compiler: run.builder.compiler_for(
1042                 run.builder.top_stage,
1043                 run.builder.config.build,
1044                 run.target,
1045             ),
1046             target: run.target,
1047         });
1048     }
1049
1050     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1051         // This prevents rust-analyzer from being built for "dist" or "install"
1052         // on the stable/beta channels. It is a nightly-only tool and should
1053         // not be included.
1054         if !builder.build.unstable_features() {
1055             return None;
1056         }
1057         let compiler = self.compiler;
1058         let target = self.target;
1059
1060         if target.contains("riscv64") {
1061             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1062             // to build. See #74813 for details.
1063             return None;
1064         }
1065
1066         let rust_analyzer = builder
1067             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1068             .expect("rust-analyzer always builds");
1069
1070         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1071         tarball.set_overlay(OverlayKind::RustAnalyzer);
1072         tarball.is_preview(true);
1073         tarball.add_file(rust_analyzer, "bin", 0o755);
1074         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1075         Some(tarball.generate())
1076     }
1077 }
1078
1079 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1080 pub struct Clippy {
1081     pub compiler: Compiler,
1082     pub target: TargetSelection,
1083 }
1084
1085 impl Step for Clippy {
1086     type Output = Option<GeneratedTarball>;
1087     const DEFAULT: bool = true;
1088     const ONLY_HOSTS: bool = true;
1089
1090     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1091         let default = should_build_extended_tool(&run.builder, "clippy");
1092         run.alias("clippy").default_condition(default)
1093     }
1094
1095     fn make_run(run: RunConfig<'_>) {
1096         run.builder.ensure(Clippy {
1097             compiler: run.builder.compiler_for(
1098                 run.builder.top_stage,
1099                 run.builder.config.build,
1100                 run.target,
1101             ),
1102             target: run.target,
1103         });
1104     }
1105
1106     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1107         let compiler = self.compiler;
1108         let target = self.target;
1109
1110         // Prepare the image directory
1111         // We expect clippy to build, because we've exited this step above if tool
1112         // state for clippy isn't testing.
1113         let clippy = builder
1114             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1115             .expect("clippy expected to build - essential tool");
1116         let cargoclippy = builder
1117             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1118             .expect("clippy expected to build - essential tool");
1119
1120         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1121         tarball.set_overlay(OverlayKind::Clippy);
1122         tarball.is_preview(true);
1123         tarball.add_file(clippy, "bin", 0o755);
1124         tarball.add_file(cargoclippy, "bin", 0o755);
1125         tarball.add_legal_and_readme_to("share/doc/clippy");
1126         Some(tarball.generate())
1127     }
1128 }
1129
1130 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1131 pub struct Miri {
1132     pub compiler: Compiler,
1133     pub target: TargetSelection,
1134 }
1135
1136 impl Step for Miri {
1137     type Output = Option<GeneratedTarball>;
1138     const DEFAULT: bool = true;
1139     const ONLY_HOSTS: bool = true;
1140
1141     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1142         let default = should_build_extended_tool(&run.builder, "miri");
1143         run.alias("miri").default_condition(default)
1144     }
1145
1146     fn make_run(run: RunConfig<'_>) {
1147         run.builder.ensure(Miri {
1148             compiler: run.builder.compiler_for(
1149                 run.builder.top_stage,
1150                 run.builder.config.build,
1151                 run.target,
1152             ),
1153             target: run.target,
1154         });
1155     }
1156
1157     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1158         // This prevents miri from being built for "dist" or "install"
1159         // on the stable/beta channels. It is a nightly-only tool and should
1160         // not be included.
1161         if !builder.build.unstable_features() {
1162             return None;
1163         }
1164         let compiler = self.compiler;
1165         let target = self.target;
1166
1167         let miri = builder
1168             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1169             .or_else(|| {
1170                 missing_tool("miri", builder.build.config.missing_tools);
1171                 None
1172             })?;
1173         let cargomiri = builder
1174             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1175             .or_else(|| {
1176                 missing_tool("cargo miri", builder.build.config.missing_tools);
1177                 None
1178             })?;
1179
1180         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1181         tarball.set_overlay(OverlayKind::Miri);
1182         tarball.is_preview(true);
1183         tarball.add_file(miri, "bin", 0o755);
1184         tarball.add_file(cargomiri, "bin", 0o755);
1185         tarball.add_legal_and_readme_to("share/doc/miri");
1186         Some(tarball.generate())
1187     }
1188 }
1189
1190 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1191 pub struct Rustfmt {
1192     pub compiler: Compiler,
1193     pub target: TargetSelection,
1194 }
1195
1196 impl Step for Rustfmt {
1197     type Output = Option<GeneratedTarball>;
1198     const DEFAULT: bool = true;
1199     const ONLY_HOSTS: bool = true;
1200
1201     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1202         let default = should_build_extended_tool(&run.builder, "rustfmt");
1203         run.alias("rustfmt").default_condition(default)
1204     }
1205
1206     fn make_run(run: RunConfig<'_>) {
1207         run.builder.ensure(Rustfmt {
1208             compiler: run.builder.compiler_for(
1209                 run.builder.top_stage,
1210                 run.builder.config.build,
1211                 run.target,
1212             ),
1213             target: run.target,
1214         });
1215     }
1216
1217     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1218         let compiler = self.compiler;
1219         let target = self.target;
1220
1221         let rustfmt = builder
1222             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1223             .or_else(|| {
1224                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1225                 None
1226             })?;
1227         let cargofmt = builder
1228             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1229             .or_else(|| {
1230                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1231                 None
1232             })?;
1233
1234         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1235         tarball.set_overlay(OverlayKind::Rustfmt);
1236         tarball.is_preview(true);
1237         tarball.add_file(rustfmt, "bin", 0o755);
1238         tarball.add_file(cargofmt, "bin", 0o755);
1239         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1240         Some(tarball.generate())
1241     }
1242 }
1243
1244 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1245 pub struct RustDemangler {
1246     pub compiler: Compiler,
1247     pub target: TargetSelection,
1248 }
1249
1250 impl Step for RustDemangler {
1251     type Output = Option<GeneratedTarball>;
1252     const DEFAULT: bool = true;
1253     const ONLY_HOSTS: bool = true;
1254
1255     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1256         // While other tools use `should_build_extended_tool` to decide whether to be run by
1257         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1258         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1259         // we run the step by default when only `extended = true`, and decide whether to actually
1260         // run it or not later.
1261         let default = run.builder.config.extended;
1262         run.alias("rust-demangler").default_condition(default)
1263     }
1264
1265     fn make_run(run: RunConfig<'_>) {
1266         run.builder.ensure(RustDemangler {
1267             compiler: run.builder.compiler_for(
1268                 run.builder.top_stage,
1269                 run.builder.config.build,
1270                 run.target,
1271             ),
1272             target: run.target,
1273         });
1274     }
1275
1276     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1277         let compiler = self.compiler;
1278         let target = self.target;
1279
1280         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1281         let condition = should_build_extended_tool(builder, "rust-demangler")
1282             || builder.config.profiler_enabled(target);
1283         if builder.config.extended && !condition {
1284             return None;
1285         }
1286
1287         let rust_demangler = builder
1288             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1289             .expect("rust-demangler expected to build - in-tree tool");
1290
1291         // Prepare the image directory
1292         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1293         tarball.set_overlay(OverlayKind::RustDemangler);
1294         tarball.is_preview(true);
1295         tarball.add_file(&rust_demangler, "bin", 0o755);
1296         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1297         Some(tarball.generate())
1298     }
1299 }
1300
1301 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1302 pub struct Extended {
1303     stage: u32,
1304     host: TargetSelection,
1305     target: TargetSelection,
1306 }
1307
1308 impl Step for Extended {
1309     type Output = ();
1310     const DEFAULT: bool = true;
1311     const ONLY_HOSTS: bool = true;
1312
1313     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1314         let builder = run.builder;
1315         run.alias("extended").default_condition(builder.config.extended)
1316     }
1317
1318     fn make_run(run: RunConfig<'_>) {
1319         run.builder.ensure(Extended {
1320             stage: run.builder.top_stage,
1321             host: run.builder.config.build,
1322             target: run.target,
1323         });
1324     }
1325
1326     /// Creates a combined installer for the specified target in the provided stage.
1327     fn run(self, builder: &Builder<'_>) {
1328         let target = self.target;
1329         let stage = self.stage;
1330         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1331
1332         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1333
1334         let mut tarballs = Vec::new();
1335         let mut built_tools = HashSet::new();
1336         macro_rules! add_component {
1337             ($name:expr => $step:expr) => {
1338                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1339                     tarballs.push(tarball);
1340                     built_tools.insert($name);
1341                 }
1342             };
1343         }
1344
1345         // When rust-std package split from rustc, we needed to ensure that during
1346         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1347         // the std files during uninstall. To do this ensure that rustc comes
1348         // before rust-std in the list below.
1349         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1350         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1351
1352         if target.ends_with("windows-gnu") {
1353             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1354         }
1355
1356         add_component!("rust-docs" => Docs { host: target });
1357         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1358         add_component!("cargo" => Cargo { compiler, target });
1359         add_component!("rustfmt" => Rustfmt { compiler, target });
1360         add_component!("rls" => Rls { compiler, target });
1361         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1362         add_component!("llvm-components" => LlvmTools { target });
1363         add_component!("clippy" => Clippy { compiler, target });
1364         add_component!("miri" => Miri { compiler, target });
1365         add_component!("analysis" => Analysis { compiler, target });
1366
1367         let etc = builder.src.join("src/etc/installer");
1368
1369         // Avoid producing tarballs during a dry run.
1370         if builder.config.dry_run {
1371             return;
1372         }
1373
1374         let tarball = Tarball::new(builder, "rust", &target.triple);
1375         let generated = tarball.combine(&tarballs);
1376
1377         let tmp = tmpdir(builder).join("combined-tarball");
1378         let work = generated.work_dir();
1379
1380         let mut license = String::new();
1381         license += &builder.read(&builder.src.join("COPYRIGHT"));
1382         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1383         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1384         license.push('\n');
1385         license.push('\n');
1386
1387         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1388         let mut rtf = rtf.to_string();
1389         rtf.push('\n');
1390         for line in license.lines() {
1391             rtf.push_str(line);
1392             rtf.push_str("\\line ");
1393         }
1394         rtf.push('}');
1395
1396         fn filter(contents: &str, marker: &str) -> String {
1397             let start = format!("tool-{}-start", marker);
1398             let end = format!("tool-{}-end", marker);
1399             let mut lines = Vec::new();
1400             let mut omitted = false;
1401             for line in contents.lines() {
1402                 if line.contains(&start) {
1403                     omitted = true;
1404                 } else if line.contains(&end) {
1405                     omitted = false;
1406                 } else if !omitted {
1407                     lines.push(line);
1408                 }
1409             }
1410
1411             lines.join("\n")
1412         }
1413
1414         let xform = |p: &Path| {
1415             let mut contents = t!(fs::read_to_string(p));
1416             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1417                 if !built_tools.contains(tool) {
1418                     contents = filter(&contents, tool);
1419                 }
1420             }
1421             let ret = tmp.join(p.file_name().unwrap());
1422             t!(fs::write(&ret, &contents));
1423             ret
1424         };
1425
1426         if target.contains("apple-darwin") {
1427             builder.info("building pkg installer");
1428             let pkg = tmp.join("pkg");
1429             let _ = fs::remove_dir_all(&pkg);
1430
1431             let pkgbuild = |component: &str| {
1432                 let mut cmd = Command::new("pkgbuild");
1433                 cmd.arg("--identifier")
1434                     .arg(format!("org.rust-lang.{}", component))
1435                     .arg("--scripts")
1436                     .arg(pkg.join(component))
1437                     .arg("--nopayload")
1438                     .arg(pkg.join(component).with_extension("pkg"));
1439                 builder.run(&mut cmd);
1440             };
1441
1442             let prepare = |name: &str| {
1443                 builder.create_dir(&pkg.join(name));
1444                 builder.cp_r(
1445                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1446                     &pkg.join(name),
1447                 );
1448                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1449                 pkgbuild(name);
1450             };
1451             prepare("rustc");
1452             prepare("cargo");
1453             prepare("rust-std");
1454             prepare("rust-analysis");
1455             prepare("clippy");
1456             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1457                 if built_tools.contains(tool) {
1458                     prepare(tool);
1459                 }
1460             }
1461             // create an 'uninstall' package
1462             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1463             pkgbuild("uninstall");
1464
1465             builder.create_dir(&pkg.join("res"));
1466             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1467             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1468             let mut cmd = Command::new("productbuild");
1469             cmd.arg("--distribution")
1470                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1471                 .arg("--resources")
1472                 .arg(pkg.join("res"))
1473                 .arg(distdir(builder).join(format!(
1474                     "{}-{}.pkg",
1475                     pkgname(builder, "rust"),
1476                     target.triple
1477                 )))
1478                 .arg("--package-path")
1479                 .arg(&pkg);
1480             let _time = timeit(builder);
1481             builder.run(&mut cmd);
1482         }
1483
1484         if target.contains("windows") {
1485             let exe = tmp.join("exe");
1486             let _ = fs::remove_dir_all(&exe);
1487
1488             let prepare = |name: &str| {
1489                 builder.create_dir(&exe.join(name));
1490                 let dir = if name == "rust-std" || name == "rust-analysis" {
1491                     format!("{}-{}", name, target.triple)
1492                 } else if name == "rls" {
1493                     "rls-preview".to_string()
1494                 } else if name == "rust-analyzer" {
1495                     "rust-analyzer-preview".to_string()
1496                 } else if name == "clippy" {
1497                     "clippy-preview".to_string()
1498                 } else if name == "rust-demangler" {
1499                     "rust-demangler-preview".to_string()
1500                 } else if name == "miri" {
1501                     "miri-preview".to_string()
1502                 } else {
1503                     name.to_string()
1504                 };
1505                 builder.cp_r(
1506                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1507                     &exe.join(name),
1508                 );
1509                 builder.remove(&exe.join(name).join("manifest.in"));
1510             };
1511             prepare("rustc");
1512             prepare("cargo");
1513             prepare("rust-analysis");
1514             prepare("rust-docs");
1515             prepare("rust-std");
1516             prepare("clippy");
1517             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1518                 if built_tools.contains(tool) {
1519                     prepare(tool);
1520                 }
1521             }
1522             if target.ends_with("windows-gnu") {
1523                 prepare("rust-mingw");
1524             }
1525
1526             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1527
1528             // Generate msi installer
1529             let wix_path = env::var_os("WIX")
1530                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1531             let wix = PathBuf::from(wix_path);
1532             let heat = wix.join("bin/heat.exe");
1533             let candle = wix.join("bin/candle.exe");
1534             let light = wix.join("bin/light.exe");
1535
1536             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1537             builder.run(
1538                 Command::new(&heat)
1539                     .current_dir(&exe)
1540                     .arg("dir")
1541                     .arg("rustc")
1542                     .args(&heat_flags)
1543                     .arg("-cg")
1544                     .arg("RustcGroup")
1545                     .arg("-dr")
1546                     .arg("Rustc")
1547                     .arg("-var")
1548                     .arg("var.RustcDir")
1549                     .arg("-out")
1550                     .arg(exe.join("RustcGroup.wxs")),
1551             );
1552             builder.run(
1553                 Command::new(&heat)
1554                     .current_dir(&exe)
1555                     .arg("dir")
1556                     .arg("rust-docs")
1557                     .args(&heat_flags)
1558                     .arg("-cg")
1559                     .arg("DocsGroup")
1560                     .arg("-dr")
1561                     .arg("Docs")
1562                     .arg("-var")
1563                     .arg("var.DocsDir")
1564                     .arg("-out")
1565                     .arg(exe.join("DocsGroup.wxs"))
1566                     .arg("-t")
1567                     .arg(etc.join("msi/squash-components.xsl")),
1568             );
1569             builder.run(
1570                 Command::new(&heat)
1571                     .current_dir(&exe)
1572                     .arg("dir")
1573                     .arg("cargo")
1574                     .args(&heat_flags)
1575                     .arg("-cg")
1576                     .arg("CargoGroup")
1577                     .arg("-dr")
1578                     .arg("Cargo")
1579                     .arg("-var")
1580                     .arg("var.CargoDir")
1581                     .arg("-out")
1582                     .arg(exe.join("CargoGroup.wxs"))
1583                     .arg("-t")
1584                     .arg(etc.join("msi/remove-duplicates.xsl")),
1585             );
1586             builder.run(
1587                 Command::new(&heat)
1588                     .current_dir(&exe)
1589                     .arg("dir")
1590                     .arg("rust-std")
1591                     .args(&heat_flags)
1592                     .arg("-cg")
1593                     .arg("StdGroup")
1594                     .arg("-dr")
1595                     .arg("Std")
1596                     .arg("-var")
1597                     .arg("var.StdDir")
1598                     .arg("-out")
1599                     .arg(exe.join("StdGroup.wxs")),
1600             );
1601             if built_tools.contains("rls") {
1602                 builder.run(
1603                     Command::new(&heat)
1604                         .current_dir(&exe)
1605                         .arg("dir")
1606                         .arg("rls")
1607                         .args(&heat_flags)
1608                         .arg("-cg")
1609                         .arg("RlsGroup")
1610                         .arg("-dr")
1611                         .arg("Rls")
1612                         .arg("-var")
1613                         .arg("var.RlsDir")
1614                         .arg("-out")
1615                         .arg(exe.join("RlsGroup.wxs"))
1616                         .arg("-t")
1617                         .arg(etc.join("msi/remove-duplicates.xsl")),
1618                 );
1619             }
1620             if built_tools.contains("rust-analyzer") {
1621                 builder.run(
1622                     Command::new(&heat)
1623                         .current_dir(&exe)
1624                         .arg("dir")
1625                         .arg("rust-analyzer")
1626                         .args(&heat_flags)
1627                         .arg("-cg")
1628                         .arg("RustAnalyzerGroup")
1629                         .arg("-dr")
1630                         .arg("RustAnalyzer")
1631                         .arg("-var")
1632                         .arg("var.RustAnalyzerDir")
1633                         .arg("-out")
1634                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1635                         .arg("-t")
1636                         .arg(etc.join("msi/remove-duplicates.xsl")),
1637                 );
1638             }
1639             builder.run(
1640                 Command::new(&heat)
1641                     .current_dir(&exe)
1642                     .arg("dir")
1643                     .arg("clippy")
1644                     .args(&heat_flags)
1645                     .arg("-cg")
1646                     .arg("ClippyGroup")
1647                     .arg("-dr")
1648                     .arg("Clippy")
1649                     .arg("-var")
1650                     .arg("var.ClippyDir")
1651                     .arg("-out")
1652                     .arg(exe.join("ClippyGroup.wxs"))
1653                     .arg("-t")
1654                     .arg(etc.join("msi/remove-duplicates.xsl")),
1655             );
1656             if built_tools.contains("rust-demangler") {
1657                 builder.run(
1658                     Command::new(&heat)
1659                         .current_dir(&exe)
1660                         .arg("dir")
1661                         .arg("rust-demangler")
1662                         .args(&heat_flags)
1663                         .arg("-cg")
1664                         .arg("RustDemanglerGroup")
1665                         .arg("-dr")
1666                         .arg("RustDemangler")
1667                         .arg("-var")
1668                         .arg("var.RustDemanglerDir")
1669                         .arg("-out")
1670                         .arg(exe.join("RustDemanglerGroup.wxs"))
1671                         .arg("-t")
1672                         .arg(etc.join("msi/remove-duplicates.xsl")),
1673                 );
1674             }
1675             if built_tools.contains("miri") {
1676                 builder.run(
1677                     Command::new(&heat)
1678                         .current_dir(&exe)
1679                         .arg("dir")
1680                         .arg("miri")
1681                         .args(&heat_flags)
1682                         .arg("-cg")
1683                         .arg("MiriGroup")
1684                         .arg("-dr")
1685                         .arg("Miri")
1686                         .arg("-var")
1687                         .arg("var.MiriDir")
1688                         .arg("-out")
1689                         .arg(exe.join("MiriGroup.wxs"))
1690                         .arg("-t")
1691                         .arg(etc.join("msi/remove-duplicates.xsl")),
1692                 );
1693             }
1694             builder.run(
1695                 Command::new(&heat)
1696                     .current_dir(&exe)
1697                     .arg("dir")
1698                     .arg("rust-analysis")
1699                     .args(&heat_flags)
1700                     .arg("-cg")
1701                     .arg("AnalysisGroup")
1702                     .arg("-dr")
1703                     .arg("Analysis")
1704                     .arg("-var")
1705                     .arg("var.AnalysisDir")
1706                     .arg("-out")
1707                     .arg(exe.join("AnalysisGroup.wxs"))
1708                     .arg("-t")
1709                     .arg(etc.join("msi/remove-duplicates.xsl")),
1710             );
1711             if target.ends_with("windows-gnu") {
1712                 builder.run(
1713                     Command::new(&heat)
1714                         .current_dir(&exe)
1715                         .arg("dir")
1716                         .arg("rust-mingw")
1717                         .args(&heat_flags)
1718                         .arg("-cg")
1719                         .arg("GccGroup")
1720                         .arg("-dr")
1721                         .arg("Gcc")
1722                         .arg("-var")
1723                         .arg("var.GccDir")
1724                         .arg("-out")
1725                         .arg(exe.join("GccGroup.wxs")),
1726                 );
1727             }
1728
1729             let candle = |input: &Path| {
1730                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1731                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1732                 let mut cmd = Command::new(&candle);
1733                 cmd.current_dir(&exe)
1734                     .arg("-nologo")
1735                     .arg("-dRustcDir=rustc")
1736                     .arg("-dDocsDir=rust-docs")
1737                     .arg("-dCargoDir=cargo")
1738                     .arg("-dStdDir=rust-std")
1739                     .arg("-dAnalysisDir=rust-analysis")
1740                     .arg("-dClippyDir=clippy")
1741                     .arg("-arch")
1742                     .arg(&arch)
1743                     .arg("-out")
1744                     .arg(&output)
1745                     .arg(&input);
1746                 add_env(builder, &mut cmd, target);
1747
1748                 if built_tools.contains("rust-demangler") {
1749                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1750                 }
1751                 if built_tools.contains("rls") {
1752                     cmd.arg("-dRlsDir=rls");
1753                 }
1754                 if built_tools.contains("rust-analyzer") {
1755                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1756                 }
1757                 if built_tools.contains("miri") {
1758                     cmd.arg("-dMiriDir=miri");
1759                 }
1760                 if target.ends_with("windows-gnu") {
1761                     cmd.arg("-dGccDir=rust-mingw");
1762                 }
1763                 builder.run(&mut cmd);
1764             };
1765             candle(&xform(&etc.join("msi/rust.wxs")));
1766             candle(&etc.join("msi/ui.wxs"));
1767             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1768             candle("RustcGroup.wxs".as_ref());
1769             candle("DocsGroup.wxs".as_ref());
1770             candle("CargoGroup.wxs".as_ref());
1771             candle("StdGroup.wxs".as_ref());
1772             candle("ClippyGroup.wxs".as_ref());
1773             if built_tools.contains("rust-demangler") {
1774                 candle("RustDemanglerGroup.wxs".as_ref());
1775             }
1776             if built_tools.contains("rls") {
1777                 candle("RlsGroup.wxs".as_ref());
1778             }
1779             if built_tools.contains("rust-analyzer") {
1780                 candle("RustAnalyzerGroup.wxs".as_ref());
1781             }
1782             if built_tools.contains("miri") {
1783                 candle("MiriGroup.wxs".as_ref());
1784             }
1785             candle("AnalysisGroup.wxs".as_ref());
1786
1787             if target.ends_with("windows-gnu") {
1788                 candle("GccGroup.wxs".as_ref());
1789             }
1790
1791             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1792             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1793             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1794
1795             builder.info(&format!("building `msi` installer with {:?}", light));
1796             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1797             let mut cmd = Command::new(&light);
1798             cmd.arg("-nologo")
1799                 .arg("-ext")
1800                 .arg("WixUIExtension")
1801                 .arg("-ext")
1802                 .arg("WixUtilExtension")
1803                 .arg("-out")
1804                 .arg(exe.join(&filename))
1805                 .arg("rust.wixobj")
1806                 .arg("ui.wixobj")
1807                 .arg("rustwelcomedlg.wixobj")
1808                 .arg("RustcGroup.wixobj")
1809                 .arg("DocsGroup.wixobj")
1810                 .arg("CargoGroup.wixobj")
1811                 .arg("StdGroup.wixobj")
1812                 .arg("AnalysisGroup.wixobj")
1813                 .arg("ClippyGroup.wixobj")
1814                 .current_dir(&exe);
1815
1816             if built_tools.contains("rls") {
1817                 cmd.arg("RlsGroup.wixobj");
1818             }
1819             if built_tools.contains("rust-analyzer") {
1820                 cmd.arg("RustAnalyzerGroup.wixobj");
1821             }
1822             if built_tools.contains("rust-demangler") {
1823                 cmd.arg("RustDemanglerGroup.wixobj");
1824             }
1825             if built_tools.contains("miri") {
1826                 cmd.arg("MiriGroup.wixobj");
1827             }
1828
1829             if target.ends_with("windows-gnu") {
1830                 cmd.arg("GccGroup.wixobj");
1831             }
1832             // ICE57 wrongly complains about the shortcuts
1833             cmd.arg("-sice:ICE57");
1834
1835             let _time = timeit(builder);
1836             builder.run(&mut cmd);
1837
1838             if !builder.config.dry_run {
1839                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1840             }
1841         }
1842     }
1843 }
1844
1845 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1846     let mut parts = builder.version.split('.');
1847     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1848         .env("CFG_RELEASE_NUM", &builder.version)
1849         .env("CFG_RELEASE", builder.rust_release())
1850         .env("CFG_VER_MAJOR", parts.next().unwrap())
1851         .env("CFG_VER_MINOR", parts.next().unwrap())
1852         .env("CFG_VER_PATCH", parts.next().unwrap())
1853         .env("CFG_VER_BUILD", "0") // just needed to build
1854         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1855         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1856         .env("CFG_BUILD", target.triple)
1857         .env("CFG_CHANNEL", &builder.config.channel);
1858
1859     if target.contains("windows-gnullvm") {
1860         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1861     } else if target.contains("windows-gnu") {
1862         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1863     } else {
1864         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1865     }
1866 }
1867
1868 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1869 ///
1870
1871 /// Returns whether the files were actually copied.
1872 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1873     if let Some(config) = builder.config.target_config.get(&target) {
1874         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1875             // If the LLVM was externally provided, then we don't currently copy
1876             // artifacts into the sysroot. This is not necessarily the right
1877             // choice (in particular, it will require the LLVM dylib to be in
1878             // the linker's load path at runtime), but the common use case for
1879             // external LLVMs is distribution provided LLVMs, and in that case
1880             // they're usually in the standard search path (e.g., /usr/lib) and
1881             // copying them here is going to cause problems as we may end up
1882             // with the wrong files and isn't what distributions want.
1883             //
1884             // This behavior may be revisited in the future though.
1885             //
1886             // If the LLVM is coming from ourselves (just from CI) though, we
1887             // still want to install it, as it otherwise won't be available.
1888             return false;
1889         }
1890     }
1891
1892     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1893     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1894     // clear why this is the case, though. llvm-config will emit the versioned
1895     // paths and we don't want those in the sysroot (as we're expecting
1896     // unversioned paths).
1897     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1898         let src_libdir = builder.llvm_out(target).join("lib");
1899         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1900         if llvm_dylib_path.exists() {
1901             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1902         }
1903         !builder.config.dry_run
1904     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1905         let mut cmd = Command::new(llvm_config);
1906         cmd.arg("--libfiles");
1907         builder.verbose(&format!("running {:?}", cmd));
1908         let files = output(&mut cmd);
1909         let build_llvm_out = &builder.llvm_out(builder.config.build);
1910         let target_llvm_out = &builder.llvm_out(target);
1911         for file in files.trim_end().split(' ') {
1912             // If we're not using a custom LLVM, make sure we package for the target.
1913             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1914                 target_llvm_out.join(relative_path)
1915             } else {
1916                 PathBuf::from(file)
1917             };
1918             builder.install(&file, dst_libdir, 0o644);
1919         }
1920         !builder.config.dry_run
1921     } else {
1922         false
1923     }
1924 }
1925
1926 /// Maybe add libLLVM.so to the target lib-dir for linking.
1927 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1928     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1929     // We do not need to copy LLVM files into the sysroot if it is not
1930     // dynamically linked; it is already included into librustc_llvm
1931     // statically.
1932     if builder.llvm_link_shared() {
1933         maybe_install_llvm(builder, target, &dst_libdir);
1934     }
1935 }
1936
1937 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1938 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1939     let dst_libdir =
1940         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1941     // We do not need to copy LLVM files into the sysroot if it is not
1942     // dynamically linked; it is already included into librustc_llvm
1943     // statically.
1944     if builder.llvm_link_shared() {
1945         maybe_install_llvm(builder, target, &dst_libdir);
1946     }
1947 }
1948
1949 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1950 pub struct LlvmTools {
1951     pub target: TargetSelection,
1952 }
1953
1954 impl Step for LlvmTools {
1955     type Output = Option<GeneratedTarball>;
1956     const ONLY_HOSTS: bool = true;
1957     const DEFAULT: bool = true;
1958
1959     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1960         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1961         // FIXME: allow using the names of the tools themselves?
1962         run.alias("llvm-tools").default_condition(default)
1963     }
1964
1965     fn make_run(run: RunConfig<'_>) {
1966         run.builder.ensure(LlvmTools { target: run.target });
1967     }
1968
1969     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1970         let target = self.target;
1971
1972         /* run only if llvm-config isn't used */
1973         if let Some(config) = builder.config.target_config.get(&target) {
1974             if let Some(ref _s) = config.llvm_config {
1975                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1976                 return None;
1977             }
1978         }
1979
1980         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1981         tarball.set_overlay(OverlayKind::LLVM);
1982         tarball.is_preview(true);
1983
1984         // Prepare the image directory
1985         let src_bindir = builder.llvm_out(target).join("bin");
1986         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1987         for tool in LLVM_TOOLS {
1988             let exe = src_bindir.join(exe(tool, target));
1989             tarball.add_file(&exe, &dst_bindir, 0o755);
1990         }
1991
1992         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1993         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1994         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1995         // compiler libraries.
1996         maybe_install_llvm_target(builder, target, tarball.image_dir());
1997
1998         Some(tarball.generate())
1999     }
2000 }
2001
2002 // Tarball intended for internal consumption to ease rustc/std development.
2003 //
2004 // Should not be considered stable by end users.
2005 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2006 pub struct RustDev {
2007     pub target: TargetSelection,
2008 }
2009
2010 impl Step for RustDev {
2011     type Output = Option<GeneratedTarball>;
2012     const DEFAULT: bool = true;
2013     const ONLY_HOSTS: bool = true;
2014
2015     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2016         run.alias("rust-dev")
2017     }
2018
2019     fn make_run(run: RunConfig<'_>) {
2020         run.builder.ensure(RustDev { target: run.target });
2021     }
2022
2023     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2024         let target = self.target;
2025
2026         /* run only if llvm-config isn't used */
2027         if let Some(config) = builder.config.target_config.get(&target) {
2028             if let Some(ref _s) = config.llvm_config {
2029                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2030                 return None;
2031             }
2032         }
2033
2034         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2035         tarball.set_overlay(OverlayKind::LLVM);
2036
2037         let src_bindir = builder.llvm_out(target).join("bin");
2038         // If updating this list, you likely want to change
2039         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2040         // will not pick up the extra file until LLVM gets bumped.
2041         for bin in &[
2042             "llvm-config",
2043             "llvm-ar",
2044             "llvm-objdump",
2045             "llvm-profdata",
2046             "llvm-bcanalyzer",
2047             "llvm-cov",
2048             "llvm-dwp",
2049             "llvm-nm",
2050             "llvm-dwarfdump",
2051         ] {
2052             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2053         }
2054
2055         // We don't build LLD on some platforms, so only add it if it exists
2056         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2057         if lld_path.exists() {
2058             tarball.add_file(lld_path, "bin", 0o755);
2059         }
2060
2061         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2062
2063         // Copy the include directory as well; needed mostly to build
2064         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2065         // just broadly useful to be able to link against the bundled LLVM.
2066         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2067
2068         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2069         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2070         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2071         // compiler libraries.
2072         let dst_libdir = tarball.image_dir().join("lib");
2073         maybe_install_llvm(builder, target, &dst_libdir);
2074         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2075         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2076
2077         Some(tarball.generate())
2078     }
2079 }
2080
2081 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2082 /// release process to avoid cloning the monorepo and building stuff.
2083 ///
2084 /// Should not be considered stable by end users.
2085 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2086 pub struct BuildManifest {
2087     pub target: TargetSelection,
2088 }
2089
2090 impl Step for BuildManifest {
2091     type Output = GeneratedTarball;
2092     const DEFAULT: bool = false;
2093     const ONLY_HOSTS: bool = true;
2094
2095     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2096         run.alias("build-manifest")
2097     }
2098
2099     fn make_run(run: RunConfig<'_>) {
2100         run.builder.ensure(BuildManifest { target: run.target });
2101     }
2102
2103     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2104         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2105
2106         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2107         tarball.add_file(&build_manifest, "bin", 0o755);
2108         tarball.generate()
2109     }
2110 }
2111
2112 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2113 ///
2114 /// Currently this is the PGO profile data.
2115 ///
2116 /// Should not be considered stable by end users.
2117 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2118 pub struct ReproducibleArtifacts {
2119     pub target: TargetSelection,
2120 }
2121
2122 impl Step for ReproducibleArtifacts {
2123     type Output = Option<GeneratedTarball>;
2124     const DEFAULT: bool = true;
2125     const ONLY_HOSTS: bool = true;
2126
2127     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2128         run.alias("reproducible-artifacts")
2129     }
2130
2131     fn make_run(run: RunConfig<'_>) {
2132         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2133     }
2134
2135     fn run(self, builder: &Builder<'_>) -> Self::Output {
2136         let mut added_anything = false;
2137         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2138         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2139             tarball.add_file(path, ".", 0o644);
2140             added_anything = true;
2141         }
2142         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2143             tarball.add_file(path, ".", 0o644);
2144             added_anything = true;
2145         }
2146         if added_anything { Some(tarball.generate()) } else { None }
2147     }
2148 }