src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
  43         panic!(
  44             "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
  45             tool_name, help
  46         )
  47     }
  48 }
  49
  50 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  51     if !builder.config.extended {
  52         return false;
  53     }
  54     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  55 }
  56
  57 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  58 pub struct Docs {
  59     pub host: TargetSelection,
  60 }
  61
  62 impl Step for Docs {
  63     type Output = Option<GeneratedTarball>;
  64     const DEFAULT: bool = true;
  65
  66     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  67         let default = run.builder.config.docs;
  68         run.alias("rust-docs").default_condition(default)
  69     }
  70
  71     fn make_run(run: RunConfig<'_>) {
  72         run.builder.ensure(Docs { host: run.target });
  73     }
  74
  75     /// Builds the `rust-docs` installer component.
  76     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  77         let host = self.host;
  78         builder.default_doc(&[]);
  79
  80         let dest = "share/doc/rust/html";
  81
  82         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  83         tarball.set_product_name("Rust Documentation");
  84         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  85         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  86         Some(tarball.generate())
  87     }
  88 }
  89
  90 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  91 pub struct RustcDocs {
  92     pub host: TargetSelection,
  93 }
  94
  95 impl Step for RustcDocs {
  96     type Output = Option<GeneratedTarball>;
  97     const DEFAULT: bool = true;
  98
  99     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 100         let builder = run.builder;
 101         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 102     }
 103
 104     fn make_run(run: RunConfig<'_>) {
 105         run.builder.ensure(RustcDocs { host: run.target });
 106     }
 107
 108     /// Builds the `rustc-docs` installer component.
 109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 110         let host = self.host;
 111         builder.default_doc(&[]);
 112
 113         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 114         tarball.set_product_name("Rustc Documentation");
 115         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 116         Some(tarball.generate())
 117     }
 118 }
 119
 120 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 121     let mut found = Vec::with_capacity(files.len());
 122
 123     for file in files {
 124         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 125
 126         if let Some(file_path) = file_path {
 127             found.push(file_path);
 128         } else {
 129             panic!("Could not find '{}' in {:?}", file, path);
 130         }
 131     }
 132
 133     found
 134 }
 135
 136 fn make_win_dist(
 137     rust_root: &Path,
 138     plat_root: &Path,
 139     target: TargetSelection,
 140     builder: &Builder<'_>,
 141 ) {
 142     //Ask gcc where it keeps its stuff
 143     let mut cmd = Command::new(builder.cc(target));
 144     cmd.arg("-print-search-dirs");
 145     let gcc_out = output(&mut cmd);
 146
 147     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 148     let mut lib_path = Vec::new();
 149
 150     for line in gcc_out.lines() {
 151         let idx = line.find(':').unwrap();
 152         let key = &line[..idx];
 153         let trim_chars: &[_] = &[' ', '='];
 154         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 155
 156         if key == "programs" {
 157             bin_path.extend(value);
 158         } else if key == "libraries" {
 159             lib_path.extend(value);
 160         }
 161     }
 162
 163     let compiler = if target == "i686-pc-windows-gnu" {
 164         "i686-w64-mingw32-gcc.exe"
 165     } else if target == "x86_64-pc-windows-gnu" {
 166         "x86_64-w64-mingw32-gcc.exe"
 167     } else {
 168         "gcc.exe"
 169     };
 170     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 171     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 172     if target.starts_with("i686-") {
 173         rustc_dlls.push("libgcc_s_dw2-1.dll");
 174     } else {
 175         rustc_dlls.push("libgcc_s_seh-1.dll");
 176     }
 177
 178     let target_libs = [
 179         //MinGW libs
 180         "libgcc.a",
 181         "libgcc_eh.a",
 182         "libgcc_s.a",
 183         "libm.a",
 184         "libmingw32.a",
 185         "libmingwex.a",
 186         "libstdc++.a",
 187         "libiconv.a",
 188         "libmoldname.a",
 189         "libpthread.a",
 190         //Windows import libs
 191         "libadvapi32.a",
 192         "libbcrypt.a",
 193         "libcomctl32.a",
 194         "libcomdlg32.a",
 195         "libcredui.a",
 196         "libcrypt32.a",
 197         "libdbghelp.a",
 198         "libgdi32.a",
 199         "libimagehlp.a",
 200         "libiphlpapi.a",
 201         "libkernel32.a",
 202         "libmsimg32.a",
 203         "libmsvcrt.a",
 204         "libodbc32.a",
 205         "libole32.a",
 206         "liboleaut32.a",
 207         "libopengl32.a",
 208         "libpsapi.a",
 209         "librpcrt4.a",
 210         "libsecur32.a",
 211         "libsetupapi.a",
 212         "libshell32.a",
 213         "libsynchronization.a",
 214         "libuser32.a",
 215         "libuserenv.a",
 216         "libuuid.a",
 217         "libwinhttp.a",
 218         "libwinmm.a",
 219         "libwinspool.a",
 220         "libws2_32.a",
 221         "libwsock32.a",
 222     ];
 223
 224     //Find mingw artifacts we want to bundle
 225     let target_tools = find_files(&target_tools, &bin_path);
 226     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 227     let target_libs = find_files(&target_libs, &lib_path);
 228
 229     // Copy runtime dlls next to rustc.exe
 230     let dist_bin_dir = rust_root.join("bin/");
 231     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 232     for src in rustc_dlls {
 233         builder.copy_to_folder(&src, &dist_bin_dir);
 234     }
 235
 236     //Copy platform tools to platform-specific bin directory
 237     let target_bin_dir = plat_root
 238         .join("lib")
 239         .join("rustlib")
 240         .join(target.triple)
 241         .join("bin")
 242         .join("self-contained");
 243     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 244     for src in target_tools {
 245         builder.copy_to_folder(&src, &target_bin_dir);
 246     }
 247
 248     // Warn windows-gnu users that the bundled GCC cannot compile C files
 249     builder.create(
 250         &target_bin_dir.join("GCC-WARNING.txt"),
 251         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 252          used as a linker. In order to be able to compile projects containing C code use \
 253          the GCC provided by MinGW or Cygwin.",
 254     );
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root
 258         .join("lib")
 259         .join("rustlib")
 260         .join(target.triple)
 261         .join("lib")
 262         .join("self-contained");
 263     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 264     for src in target_libs {
 265         builder.copy_to_folder(&src, &target_lib_dir);
 266     }
 267 }
 268
 269 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 270 pub struct Mingw {
 271     pub host: TargetSelection,
 272 }
 273
 274 impl Step for Mingw {
 275     type Output = Option<GeneratedTarball>;
 276     const DEFAULT: bool = true;
 277
 278     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 279         run.alias("rust-mingw")
 280     }
 281
 282     fn make_run(run: RunConfig<'_>) {
 283         run.builder.ensure(Mingw { host: run.target });
 284     }
 285
 286     /// Builds the `rust-mingw` installer component.
 287     ///
 288     /// This contains all the bits and pieces to run the MinGW Windows targets
 289     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 290     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 291         let host = self.host;
 292         if !host.ends_with("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 297         tarball.set_product_name("Rust MinGW");
 298
 299         // The first argument is a "temporary directory" which is just
 300         // thrown away (this contains the runtime DLLs included in the rustc package
 301         // above) and the second argument is where to place all the MinGW components
 302         // (which is what we want).
 303         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 304
 305         Some(tarball.generate())
 306     }
 307 }
 308
 309 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 310 pub struct Rustc {
 311     pub compiler: Compiler,
 312 }
 313
 314 impl Step for Rustc {
 315     type Output = GeneratedTarball;
 316     const DEFAULT: bool = true;
 317     const ONLY_HOSTS: bool = true;
 318
 319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 320         run.alias("rustc")
 321     }
 322
 323     fn make_run(run: RunConfig<'_>) {
 324         run.builder
 325             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 326     }
 327
 328     /// Creates the `rustc` installer component.
 329     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 330         let compiler = self.compiler;
 331         let host = self.compiler.host;
 332
 333         let tarball = Tarball::new(builder, "rustc", &host.triple);
 334
 335         // Prepare the rustc "image", what will actually end up getting installed
 336         prepare_image(builder, compiler, tarball.image_dir());
 337
 338         // On MinGW we've got a few runtime DLL dependencies that we need to
 339         // include. The first argument to this script is where to put these DLLs
 340         // (the image we're creating), and the second argument is a junk directory
 341         // to ignore all other MinGW stuff the script creates.
 342         //
 343         // On 32-bit MinGW we're always including a DLL which needs some extra
 344         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 345         // anything requiring us to distribute a license, but it's likely the
 346         // install will *also* include the rust-mingw package, which also needs
 347         // licenses, so to be safe we just include it here in all MinGW packages.
 348         if host.ends_with("pc-windows-gnu") {
 349             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 350             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 351         }
 352
 353         return tarball.generate();
 354
 355         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 356             let host = compiler.host;
 357             let src = builder.sysroot(compiler);
 358
 359             // Copy rustc/rustdoc binaries
 360             t!(fs::create_dir_all(image.join("bin")));
 361             builder.cp_r(&src.join("bin"), &image.join("bin"));
 362
 363             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 364
 365             let libdir_relative = builder.libdir_relative(compiler);
 366
 367             // Copy runtime DLLs needed by the compiler
 368             if libdir_relative.to_str() != Some("bin") {
 369                 let libdir = builder.rustc_libdir(compiler);
 370                 for entry in builder.read_dir(&libdir) {
 371                     let name = entry.file_name();
 372                     if let Some(s) = name.to_str() {
 373                         if is_dylib(s) {
 374                             // Don't use custom libdir here because ^lib/ will be resolved again
 375                             // with installer
 376                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 377                         }
 378                     }
 379                 }
 380             }
 381
 382             // Copy over the codegen backends
 383             let backends_src = builder.sysroot_codegen_backends(compiler);
 384             let backends_rel = backends_src
 385                 .strip_prefix(&src)
 386                 .unwrap()
 387                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 388                 .unwrap();
 389             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 390             let backends_dst = image.join("lib").join(&backends_rel);
 391
 392             t!(fs::create_dir_all(&backends_dst));
 393             builder.cp_r(&backends_src, &backends_dst);
 394
 395             // Copy libLLVM.so to the lib dir as well, if needed. While not
 396             // technically needed by rustc itself it's needed by lots of other
 397             // components like the llvm tools and LLD. LLD is included below and
 398             // tools/LLDB come later, so let's just throw it in the rustc
 399             // component for now.
 400             maybe_install_llvm_runtime(builder, host, image);
 401
 402             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 403             t!(fs::create_dir_all(&dst_dir));
 404
 405             // Copy over lld if it's there
 406             if builder.config.lld_enabled {
 407                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 408                 let rust_lld = exe("rust-lld", compiler.host);
 409                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 410                 // for `-Z gcc-ld=lld`
 411                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 412                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 413                 t!(fs::create_dir(&gcc_lld_dst_dir));
 414                 let exe_name = exe("ld", compiler.host);
 415                 builder.copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 416             }
 417
 418             // Man pages
 419             t!(fs::create_dir_all(image.join("share/man/man1")));
 420             let man_src = builder.src.join("src/doc/man");
 421             let man_dst = image.join("share/man/man1");
 422
 423             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 424             // to hardlink, and we don't want to edit the source templates
 425             for file_entry in builder.read_dir(&man_src) {
 426                 let page_src = file_entry.path();
 427                 let page_dst = man_dst.join(file_entry.file_name());
 428                 let src_text = t!(std::fs::read_to_string(&page_src));
 429                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 430                 t!(std::fs::write(&page_dst, &new_text));
 431                 t!(fs::copy(&page_src, &page_dst));
 432             }
 433
 434             // Debugger scripts
 435             builder
 436                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 437
 438             // Misc license info
 439             let cp = |file: &str| {
 440                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 441             };
 442             cp("COPYRIGHT");
 443             cp("LICENSE-APACHE");
 444             cp("LICENSE-MIT");
 445             cp("README.md");
 446         }
 447     }
 448 }
 449
 450 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 451 pub struct DebuggerScripts {
 452     pub sysroot: Interned<PathBuf>,
 453     pub host: TargetSelection,
 454 }
 455
 456 impl Step for DebuggerScripts {
 457     type Output = ();
 458
 459     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 460         run.never()
 461     }
 462
 463     /// Copies debugger scripts for `target` into the `sysroot` specified.
 464     fn run(self, builder: &Builder<'_>) {
 465         let host = self.host;
 466         let sysroot = self.sysroot;
 467         let dst = sysroot.join("lib/rustlib/etc");
 468         t!(fs::create_dir_all(&dst));
 469         let cp_debugger_script = |file: &str| {
 470             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 471         };
 472         if host.contains("windows-msvc") {
 473             // windbg debugger scripts
 474             builder.install(
 475                 &builder.src.join("src/etc/rust-windbg.cmd"),
 476                 &sysroot.join("bin"),
 477                 0o755,
 478             );
 479
 480             cp_debugger_script("natvis/intrinsic.natvis");
 481             cp_debugger_script("natvis/liballoc.natvis");
 482             cp_debugger_script("natvis/libcore.natvis");
 483             cp_debugger_script("natvis/libstd.natvis");
 484         } else {
 485             cp_debugger_script("rust_types.py");
 486
 487             // gdb debugger scripts
 488             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 489             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 490
 491             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 492             cp_debugger_script("gdb_lookup.py");
 493             cp_debugger_script("gdb_providers.py");
 494
 495             // lldb debugger scripts
 496             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 497
 498             cp_debugger_script("lldb_lookup.py");
 499             cp_debugger_script("lldb_providers.py");
 500             cp_debugger_script("lldb_commands")
 501         }
 502     }
 503 }
 504
 505 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 506     // The only true set of target libraries came from the build triple, so
 507     // let's reduce redundant work by only producing archives from that host.
 508     if compiler.host != builder.config.build {
 509         builder.info("\tskipping, not a build host");
 510         true
 511     } else {
 512         false
 513     }
 514 }
 515
 516 /// Copy stamped files into an image's `target/lib` directory.
 517 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 518     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 519     let self_contained_dst = dst.join("self-contained");
 520     t!(fs::create_dir_all(&dst));
 521     t!(fs::create_dir_all(&self_contained_dst));
 522     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 523         if dependency_type == DependencyType::TargetSelfContained {
 524             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 525         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 526             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 527         }
 528     }
 529 }
 530
 531 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 532 pub struct Std {
 533     pub compiler: Compiler,
 534     pub target: TargetSelection,
 535 }
 536
 537 impl Step for Std {
 538     type Output = Option<GeneratedTarball>;
 539     const DEFAULT: bool = true;
 540
 541     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 542         run.alias("rust-std")
 543     }
 544
 545     fn make_run(run: RunConfig<'_>) {
 546         run.builder.ensure(Std {
 547             compiler: run.builder.compiler_for(
 548                 run.builder.top_stage,
 549                 run.builder.config.build,
 550                 run.target,
 551             ),
 552             target: run.target,
 553         });
 554     }
 555
 556     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 557         let compiler = self.compiler;
 558         let target = self.target;
 559
 560         if skip_host_target_lib(builder, compiler) {
 561             return None;
 562         }
 563
 564         builder.ensure(compile::Std::new(compiler, target));
 565
 566         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 567         tarball.include_target_in_component_name(true);
 568
 569         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 570         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 571         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 572
 573         Some(tarball.generate())
 574     }
 575 }
 576
 577 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 578 pub struct RustcDev {
 579     pub compiler: Compiler,
 580     pub target: TargetSelection,
 581 }
 582
 583 impl Step for RustcDev {
 584     type Output = Option<GeneratedTarball>;
 585     const DEFAULT: bool = true;
 586     const ONLY_HOSTS: bool = true;
 587
 588     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 589         run.alias("rustc-dev")
 590     }
 591
 592     fn make_run(run: RunConfig<'_>) {
 593         run.builder.ensure(RustcDev {
 594             compiler: run.builder.compiler_for(
 595                 run.builder.top_stage,
 596                 run.builder.config.build,
 597                 run.target,
 598             ),
 599             target: run.target,
 600         });
 601     }
 602
 603     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 604         let compiler = self.compiler;
 605         let target = self.target;
 606         if skip_host_target_lib(builder, compiler) {
 607             return None;
 608         }
 609
 610         builder.ensure(compile::Rustc::new(compiler, target));
 611
 612         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 613
 614         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 615         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 616         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 617
 618         let src_files = &["Cargo.lock"];
 619         // This is the reduced set of paths which will become the rustc-dev component
 620         // (essentially the compiler crates and all of their path dependencies).
 621         copy_src_dirs(
 622             builder,
 623             &builder.src,
 624             &["compiler"],
 625             &[],
 626             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 627         );
 628         for file in src_files {
 629             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 630         }
 631
 632         Some(tarball.generate())
 633     }
 634 }
 635
 636 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 637 pub struct Analysis {
 638     pub compiler: Compiler,
 639     pub target: TargetSelection,
 640 }
 641
 642 impl Step for Analysis {
 643     type Output = Option<GeneratedTarball>;
 644     const DEFAULT: bool = true;
 645
 646     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 647         let default = should_build_extended_tool(&run.builder, "analysis");
 648         run.alias("rust-analysis").default_condition(default)
 649     }
 650
 651     fn make_run(run: RunConfig<'_>) {
 652         run.builder.ensure(Analysis {
 653             // Find the actual compiler (handling the full bootstrap option) which
 654             // produced the save-analysis data because that data isn't copied
 655             // through the sysroot uplifting.
 656             compiler: run.builder.compiler_for(
 657                 run.builder.top_stage,
 658                 run.builder.config.build,
 659                 run.target,
 660             ),
 661             target: run.target,
 662         });
 663     }
 664
 665     /// Creates a tarball of save-analysis metadata, if available.
 666     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 667         let compiler = self.compiler;
 668         let target = self.target;
 669         if compiler.host != builder.config.build {
 670             return None;
 671         }
 672
 673         builder.ensure(compile::Std::new(compiler, target));
 674         let src = builder
 675             .stage_out(compiler, Mode::Std)
 676             .join(target.triple)
 677             .join(builder.cargo_dir())
 678             .join("deps")
 679             .join("save-analysis");
 680
 681         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 682         tarball.include_target_in_component_name(true);
 683         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 684         Some(tarball.generate())
 685     }
 686 }
 687
 688 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 689 /// `dst_dir`.
 690 fn copy_src_dirs(
 691     builder: &Builder<'_>,
 692     base: &Path,
 693     src_dirs: &[&str],
 694     exclude_dirs: &[&str],
 695     dst_dir: &Path,
 696 ) {
 697     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 698         let spath = match path.to_str() {
 699             Some(path) => path,
 700             None => return false,
 701         };
 702         if spath.ends_with('~') || spath.ends_with(".pyc") {
 703             return false;
 704         }
 705
 706         const LLVM_PROJECTS: &[&str] = &[
 707             "llvm-project/clang",
 708             "llvm-project\\clang",
 709             "llvm-project/libunwind",
 710             "llvm-project\\libunwind",
 711             "llvm-project/lld",
 712             "llvm-project\\lld",
 713             "llvm-project/lldb",
 714             "llvm-project\\lldb",
 715             "llvm-project/llvm",
 716             "llvm-project\\llvm",
 717             "llvm-project/compiler-rt",
 718             "llvm-project\\compiler-rt",
 719             "llvm-project/cmake",
 720             "llvm-project\\cmake",
 721         ];
 722         if spath.contains("llvm-project")
 723             && !spath.ends_with("llvm-project")
 724             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 725         {
 726             return false;
 727         }
 728
 729         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 730         if LLVM_TEST.iter().any(|path| spath.contains(path))
 731             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 732         {
 733             return false;
 734         }
 735
 736         let full_path = Path::new(dir).join(path);
 737         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 738             return false;
 739         }
 740
 741         let excludes = [
 742             "CVS",
 743             "RCS",
 744             "SCCS",
 745             ".git",
 746             ".gitignore",
 747             ".gitmodules",
 748             ".gitattributes",
 749             ".cvsignore",
 750             ".svn",
 751             ".arch-ids",
 752             "{arch}",
 753             "=RELEASE-ID",
 754             "=meta-update",
 755             "=update",
 756             ".bzr",
 757             ".bzrignore",
 758             ".bzrtags",
 759             ".hg",
 760             ".hgignore",
 761             ".hgrags",
 762             "_darcs",
 763         ];
 764         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 765     }
 766
 767     // Copy the directories using our filter
 768     for item in src_dirs {
 769         let dst = &dst_dir.join(item);
 770         t!(fs::create_dir_all(dst));
 771         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 772     }
 773 }
 774
 775 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 776 pub struct Src;
 777
 778 impl Step for Src {
 779     /// The output path of the src installer tarball
 780     type Output = GeneratedTarball;
 781     const DEFAULT: bool = true;
 782     const ONLY_HOSTS: bool = true;
 783
 784     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 785         run.alias("rust-src")
 786     }
 787
 788     fn make_run(run: RunConfig<'_>) {
 789         run.builder.ensure(Src);
 790     }
 791
 792     /// Creates the `rust-src` installer component
 793     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 794         let tarball = Tarball::new_targetless(builder, "rust-src");
 795
 796         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 797         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 798         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 799         // and fix them...
 800         //
 801         // NOTE: if you update the paths here, you also should update the "virtual" path
 802         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 803         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 804
 805         let src_files = ["Cargo.lock"];
 806         // This is the reduced set of paths which will become the rust-src component
 807         // (essentially libstd and all of its path dependencies).
 808         copy_src_dirs(
 809             builder,
 810             &builder.src,
 811             &["library", "src/llvm-project/libunwind"],
 812             &[
 813                 // not needed and contains symlinks which rustup currently
 814                 // chokes on when unpacking.
 815                 "library/backtrace/crates",
 816                 // these are 30MB combined and aren't necessary for building
 817                 // the standard library.
 818                 "library/stdarch/Cargo.toml",
 819                 "library/stdarch/crates/stdarch-verify",
 820                 "library/stdarch/crates/intrinsic-test",
 821             ],
 822             &dst_src,
 823         );
 824         for file in src_files.iter() {
 825             builder.copy(&builder.src.join(file), &dst_src.join(file));
 826         }
 827
 828         tarball.generate()
 829     }
 830 }
 831
 832 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 833 pub struct PlainSourceTarball;
 834
 835 impl Step for PlainSourceTarball {
 836     /// Produces the location of the tarball generated
 837     type Output = GeneratedTarball;
 838     const DEFAULT: bool = true;
 839     const ONLY_HOSTS: bool = true;
 840
 841     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 842         let builder = run.builder;
 843         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 844     }
 845
 846     fn make_run(run: RunConfig<'_>) {
 847         run.builder.ensure(PlainSourceTarball);
 848     }
 849
 850     /// Creates the plain source tarball
 851     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 852         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 853         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 854         // It also contains symbolic links, unlike other any other dist tarball.
 855         // It's used for distros building rustc from source in a pre-vendored environment.
 856         let mut tarball = Tarball::new(builder, "rustc", "src");
 857         tarball.permit_symlinks(true);
 858         let plain_dst_src = tarball.image_dir();
 859
 860         // This is the set of root paths which will become part of the source package
 861         let src_files = [
 862             "COPYRIGHT",
 863             "LICENSE-APACHE",
 864             "LICENSE-MIT",
 865             "CONTRIBUTING.md",
 866             "README.md",
 867             "RELEASES.md",
 868             "configure",
 869             "x.py",
 870             "config.toml.example",
 871             "Cargo.toml",
 872             "Cargo.lock",
 873         ];
 874         let src_dirs = ["src", "compiler", "library"];
 875
 876         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 877
 878         // Copy the files normally
 879         for item in &src_files {
 880             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 881         }
 882
 883         // Create the version file
 884         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 885         if let Some(sha) = builder.rust_sha() {
 886             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 887         }
 888
 889         // If we're building from git sources, we need to vendor a complete distribution.
 890         if builder.rust_info.is_git() {
 891             // Ensure we have the submodules checked out.
 892             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 893
 894             // Vendor all Cargo dependencies
 895             let mut cmd = Command::new(&builder.initial_cargo);
 896             cmd.arg("vendor")
 897                 .arg("--sync")
 898                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 899                 .arg("--sync")
 900                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 901                 .arg("--sync")
 902                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 903                 .current_dir(&plain_dst_src);
 904
 905             let config = if !builder.config.dry_run {
 906                 t!(String::from_utf8(t!(cmd.output()).stdout))
 907             } else {
 908                 String::new()
 909             };
 910
 911             let cargo_config_dir = plain_dst_src.join(".cargo");
 912             builder.create_dir(&cargo_config_dir);
 913             builder.create(&cargo_config_dir.join("config.toml"), &config);
 914         }
 915
 916         tarball.bare()
 917     }
 918 }
 919
 920 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 921 pub struct Cargo {
 922     pub compiler: Compiler,
 923     pub target: TargetSelection,
 924 }
 925
 926 impl Step for Cargo {
 927     type Output = Option<GeneratedTarball>;
 928     const DEFAULT: bool = true;
 929     const ONLY_HOSTS: bool = true;
 930
 931     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 932         let default = should_build_extended_tool(&run.builder, "cargo");
 933         run.alias("cargo").default_condition(default)
 934     }
 935
 936     fn make_run(run: RunConfig<'_>) {
 937         run.builder.ensure(Cargo {
 938             compiler: run.builder.compiler_for(
 939                 run.builder.top_stage,
 940                 run.builder.config.build,
 941                 run.target,
 942             ),
 943             target: run.target,
 944         });
 945     }
 946
 947     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 948         let compiler = self.compiler;
 949         let target = self.target;
 950
 951         let cargo = builder.ensure(tool::Cargo { compiler, target });
 952         let src = builder.src.join("src/tools/cargo");
 953         let etc = src.join("src/etc");
 954
 955         // Prepare the image directory
 956         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 957         tarball.set_overlay(OverlayKind::Cargo);
 958
 959         tarball.add_file(&cargo, "bin", 0o755);
 960         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 961         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 962         tarball.add_dir(etc.join("man"), "share/man/man1");
 963         tarball.add_legal_and_readme_to("share/doc/cargo");
 964
 965         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 966             let dirent = dirent.expect("read dir entry");
 967             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 968                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 969             }
 970         }
 971
 972         Some(tarball.generate())
 973     }
 974 }
 975
 976 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 977 pub struct Rls {
 978     pub compiler: Compiler,
 979     pub target: TargetSelection,
 980 }
 981
 982 impl Step for Rls {
 983     type Output = Option<GeneratedTarball>;
 984     const ONLY_HOSTS: bool = true;
 985     const DEFAULT: bool = true;
 986
 987     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 988         let default = should_build_extended_tool(&run.builder, "rls");
 989         run.alias("rls").default_condition(default)
 990     }
 991
 992     fn make_run(run: RunConfig<'_>) {
 993         run.builder.ensure(Rls {
 994             compiler: run.builder.compiler_for(
 995                 run.builder.top_stage,
 996                 run.builder.config.build,
 997                 run.target,
 998             ),
 999             target: run.target,
1000         });
1001     }
1002
1003     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1004         let compiler = self.compiler;
1005         let target = self.target;
1006
1007         let rls = builder
1008             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1009             .or_else(|| {
1010                 missing_tool("RLS", builder.build.config.missing_tools);
1011                 None
1012             })?;
1013
1014         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1015         tarball.set_overlay(OverlayKind::RLS);
1016         tarball.is_preview(true);
1017         tarball.add_file(rls, "bin", 0o755);
1018         tarball.add_legal_and_readme_to("share/doc/rls");
1019         Some(tarball.generate())
1020     }
1021 }
1022
1023 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1024 pub struct RustAnalyzer {
1025     pub compiler: Compiler,
1026     pub target: TargetSelection,
1027 }
1028
1029 impl Step for RustAnalyzer {
1030     type Output = Option<GeneratedTarball>;
1031     const DEFAULT: bool = true;
1032     const ONLY_HOSTS: bool = true;
1033
1034     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1035         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1036         run.alias("rust-analyzer").default_condition(default)
1037     }
1038
1039     fn make_run(run: RunConfig<'_>) {
1040         run.builder.ensure(RustAnalyzer {
1041             compiler: run.builder.compiler_for(
1042                 run.builder.top_stage,
1043                 run.builder.config.build,
1044                 run.target,
1045             ),
1046             target: run.target,
1047         });
1048     }
1049
1050     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1051         let compiler = self.compiler;
1052         let target = self.target;
1053
1054         if target.contains("riscv64") {
1055             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1056             // to build. See #74813 for details.
1057             return None;
1058         }
1059
1060         let rust_analyzer = builder
1061             .ensure(tool::RustAnalyzer { compiler, target })
1062             .expect("rust-analyzer always builds");
1063
1064         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1065         tarball.set_overlay(OverlayKind::RustAnalyzer);
1066         tarball.is_preview(true);
1067         tarball.add_file(rust_analyzer, "bin", 0o755);
1068         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1069         Some(tarball.generate())
1070     }
1071 }
1072
1073 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1074 pub struct Clippy {
1075     pub compiler: Compiler,
1076     pub target: TargetSelection,
1077 }
1078
1079 impl Step for Clippy {
1080     type Output = Option<GeneratedTarball>;
1081     const DEFAULT: bool = true;
1082     const ONLY_HOSTS: bool = true;
1083
1084     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1085         let default = should_build_extended_tool(&run.builder, "clippy");
1086         run.alias("clippy").default_condition(default)
1087     }
1088
1089     fn make_run(run: RunConfig<'_>) {
1090         run.builder.ensure(Clippy {
1091             compiler: run.builder.compiler_for(
1092                 run.builder.top_stage,
1093                 run.builder.config.build,
1094                 run.target,
1095             ),
1096             target: run.target,
1097         });
1098     }
1099
1100     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1101         let compiler = self.compiler;
1102         let target = self.target;
1103
1104         // Prepare the image directory
1105         // We expect clippy to build, because we've exited this step above if tool
1106         // state for clippy isn't testing.
1107         let clippy = builder
1108             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1109             .expect("clippy expected to build - essential tool");
1110         let cargoclippy = builder
1111             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1112             .expect("clippy expected to build - essential tool");
1113
1114         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1115         tarball.set_overlay(OverlayKind::Clippy);
1116         tarball.is_preview(true);
1117         tarball.add_file(clippy, "bin", 0o755);
1118         tarball.add_file(cargoclippy, "bin", 0o755);
1119         tarball.add_legal_and_readme_to("share/doc/clippy");
1120         Some(tarball.generate())
1121     }
1122 }
1123
1124 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1125 pub struct Miri {
1126     pub compiler: Compiler,
1127     pub target: TargetSelection,
1128 }
1129
1130 impl Step for Miri {
1131     type Output = Option<GeneratedTarball>;
1132     const DEFAULT: bool = true;
1133     const ONLY_HOSTS: bool = true;
1134
1135     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1136         let default = should_build_extended_tool(&run.builder, "miri");
1137         run.alias("miri").default_condition(default)
1138     }
1139
1140     fn make_run(run: RunConfig<'_>) {
1141         run.builder.ensure(Miri {
1142             compiler: run.builder.compiler_for(
1143                 run.builder.top_stage,
1144                 run.builder.config.build,
1145                 run.target,
1146             ),
1147             target: run.target,
1148         });
1149     }
1150
1151     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1152         // This prevents miri from being built for "dist" or "install"
1153         // on the stable/beta channels. It is a nightly-only tool and should
1154         // not be included.
1155         if !builder.build.unstable_features() {
1156             return None;
1157         }
1158         let compiler = self.compiler;
1159         let target = self.target;
1160
1161         let miri = builder
1162             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1163             .or_else(|| {
1164                 missing_tool("miri", builder.build.config.missing_tools);
1165                 None
1166             })?;
1167         let cargomiri = builder
1168             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1169             .or_else(|| {
1170                 missing_tool("cargo miri", builder.build.config.missing_tools);
1171                 None
1172             })?;
1173
1174         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1175         tarball.set_overlay(OverlayKind::Miri);
1176         tarball.is_preview(true);
1177         tarball.add_file(miri, "bin", 0o755);
1178         tarball.add_file(cargomiri, "bin", 0o755);
1179         tarball.add_legal_and_readme_to("share/doc/miri");
1180         Some(tarball.generate())
1181     }
1182 }
1183
1184 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1185 pub struct Rustfmt {
1186     pub compiler: Compiler,
1187     pub target: TargetSelection,
1188 }
1189
1190 impl Step for Rustfmt {
1191     type Output = Option<GeneratedTarball>;
1192     const DEFAULT: bool = true;
1193     const ONLY_HOSTS: bool = true;
1194
1195     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1196         let default = should_build_extended_tool(&run.builder, "rustfmt");
1197         run.alias("rustfmt").default_condition(default)
1198     }
1199
1200     fn make_run(run: RunConfig<'_>) {
1201         run.builder.ensure(Rustfmt {
1202             compiler: run.builder.compiler_for(
1203                 run.builder.top_stage,
1204                 run.builder.config.build,
1205                 run.target,
1206             ),
1207             target: run.target,
1208         });
1209     }
1210
1211     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1212         let compiler = self.compiler;
1213         let target = self.target;
1214
1215         let rustfmt = builder
1216             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1217             .or_else(|| {
1218                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1219                 None
1220             })?;
1221         let cargofmt = builder
1222             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1223             .or_else(|| {
1224                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1225                 None
1226             })?;
1227
1228         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1229         tarball.set_overlay(OverlayKind::Rustfmt);
1230         tarball.is_preview(true);
1231         tarball.add_file(rustfmt, "bin", 0o755);
1232         tarball.add_file(cargofmt, "bin", 0o755);
1233         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1234         Some(tarball.generate())
1235     }
1236 }
1237
1238 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1239 pub struct RustDemangler {
1240     pub compiler: Compiler,
1241     pub target: TargetSelection,
1242 }
1243
1244 impl Step for RustDemangler {
1245     type Output = Option<GeneratedTarball>;
1246     const DEFAULT: bool = true;
1247     const ONLY_HOSTS: bool = true;
1248
1249     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1250         // While other tools use `should_build_extended_tool` to decide whether to be run by
1251         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1252         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1253         // we run the step by default when only `extended = true`, and decide whether to actually
1254         // run it or not later.
1255         let default = run.builder.config.extended;
1256         run.alias("rust-demangler").default_condition(default)
1257     }
1258
1259     fn make_run(run: RunConfig<'_>) {
1260         run.builder.ensure(RustDemangler {
1261             compiler: run.builder.compiler_for(
1262                 run.builder.top_stage,
1263                 run.builder.config.build,
1264                 run.target,
1265             ),
1266             target: run.target,
1267         });
1268     }
1269
1270     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1271         let compiler = self.compiler;
1272         let target = self.target;
1273
1274         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1275         let condition = should_build_extended_tool(builder, "rust-demangler")
1276             || builder.config.profiler_enabled(target);
1277         if builder.config.extended && !condition {
1278             return None;
1279         }
1280
1281         let rust_demangler = builder
1282             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1283             .expect("rust-demangler expected to build - in-tree tool");
1284
1285         // Prepare the image directory
1286         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1287         tarball.set_overlay(OverlayKind::RustDemangler);
1288         tarball.is_preview(true);
1289         tarball.add_file(&rust_demangler, "bin", 0o755);
1290         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1291         Some(tarball.generate())
1292     }
1293 }
1294
1295 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1296 pub struct Extended {
1297     stage: u32,
1298     host: TargetSelection,
1299     target: TargetSelection,
1300 }
1301
1302 impl Step for Extended {
1303     type Output = ();
1304     const DEFAULT: bool = true;
1305     const ONLY_HOSTS: bool = true;
1306
1307     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1308         let builder = run.builder;
1309         run.alias("extended").default_condition(builder.config.extended)
1310     }
1311
1312     fn make_run(run: RunConfig<'_>) {
1313         run.builder.ensure(Extended {
1314             stage: run.builder.top_stage,
1315             host: run.builder.config.build,
1316             target: run.target,
1317         });
1318     }
1319
1320     /// Creates a combined installer for the specified target in the provided stage.
1321     fn run(self, builder: &Builder<'_>) {
1322         let target = self.target;
1323         let stage = self.stage;
1324         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1325
1326         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1327
1328         let mut tarballs = Vec::new();
1329         let mut built_tools = HashSet::new();
1330         macro_rules! add_component {
1331             ($name:expr => $step:expr) => {
1332                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1333                     tarballs.push(tarball);
1334                     built_tools.insert($name);
1335                 }
1336             };
1337         }
1338
1339         // When rust-std package split from rustc, we needed to ensure that during
1340         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1341         // the std files during uninstall. To do this ensure that rustc comes
1342         // before rust-std in the list below.
1343         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1344         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1345
1346         if target.ends_with("windows-gnu") {
1347             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1348         }
1349
1350         add_component!("rust-docs" => Docs { host: target });
1351         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1352         add_component!("cargo" => Cargo { compiler, target });
1353         add_component!("rustfmt" => Rustfmt { compiler, target });
1354         add_component!("rls" => Rls { compiler, target });
1355         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1356         add_component!("llvm-components" => LlvmTools { target });
1357         add_component!("clippy" => Clippy { compiler, target });
1358         add_component!("miri" => Miri { compiler, target });
1359         add_component!("analysis" => Analysis { compiler, target });
1360
1361         let etc = builder.src.join("src/etc/installer");
1362
1363         // Avoid producing tarballs during a dry run.
1364         if builder.config.dry_run {
1365             return;
1366         }
1367
1368         let tarball = Tarball::new(builder, "rust", &target.triple);
1369         let generated = tarball.combine(&tarballs);
1370
1371         let tmp = tmpdir(builder).join("combined-tarball");
1372         let work = generated.work_dir();
1373
1374         let mut license = String::new();
1375         license += &builder.read(&builder.src.join("COPYRIGHT"));
1376         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1377         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1378         license.push('\n');
1379         license.push('\n');
1380
1381         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1382         let mut rtf = rtf.to_string();
1383         rtf.push('\n');
1384         for line in license.lines() {
1385             rtf.push_str(line);
1386             rtf.push_str("\\line ");
1387         }
1388         rtf.push('}');
1389
1390         fn filter(contents: &str, marker: &str) -> String {
1391             let start = format!("tool-{}-start", marker);
1392             let end = format!("tool-{}-end", marker);
1393             let mut lines = Vec::new();
1394             let mut omitted = false;
1395             for line in contents.lines() {
1396                 if line.contains(&start) {
1397                     omitted = true;
1398                 } else if line.contains(&end) {
1399                     omitted = false;
1400                 } else if !omitted {
1401                     lines.push(line);
1402                 }
1403             }
1404
1405             lines.join("\n")
1406         }
1407
1408         let xform = |p: &Path| {
1409             let mut contents = t!(fs::read_to_string(p));
1410             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1411                 if !built_tools.contains(tool) {
1412                     contents = filter(&contents, tool);
1413                 }
1414             }
1415             let ret = tmp.join(p.file_name().unwrap());
1416             t!(fs::write(&ret, &contents));
1417             ret
1418         };
1419
1420         if target.contains("apple-darwin") {
1421             builder.info("building pkg installer");
1422             let pkg = tmp.join("pkg");
1423             let _ = fs::remove_dir_all(&pkg);
1424
1425             let pkgbuild = |component: &str| {
1426                 let mut cmd = Command::new("pkgbuild");
1427                 cmd.arg("--identifier")
1428                     .arg(format!("org.rust-lang.{}", component))
1429                     .arg("--scripts")
1430                     .arg(pkg.join(component))
1431                     .arg("--nopayload")
1432                     .arg(pkg.join(component).with_extension("pkg"));
1433                 builder.run(&mut cmd);
1434             };
1435
1436             let prepare = |name: &str| {
1437                 builder.create_dir(&pkg.join(name));
1438                 builder.cp_r(
1439                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1440                     &pkg.join(name),
1441                 );
1442                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1443                 pkgbuild(name);
1444             };
1445             prepare("rustc");
1446             prepare("cargo");
1447             prepare("rust-std");
1448             prepare("rust-analysis");
1449             prepare("clippy");
1450             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1451                 if built_tools.contains(tool) {
1452                     prepare(tool);
1453                 }
1454             }
1455             // create an 'uninstall' package
1456             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1457             pkgbuild("uninstall");
1458
1459             builder.create_dir(&pkg.join("res"));
1460             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1461             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1462             let mut cmd = Command::new("productbuild");
1463             cmd.arg("--distribution")
1464                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1465                 .arg("--resources")
1466                 .arg(pkg.join("res"))
1467                 .arg(distdir(builder).join(format!(
1468                     "{}-{}.pkg",
1469                     pkgname(builder, "rust"),
1470                     target.triple
1471                 )))
1472                 .arg("--package-path")
1473                 .arg(&pkg);
1474             let _time = timeit(builder);
1475             builder.run(&mut cmd);
1476         }
1477
1478         if target.contains("windows") {
1479             let exe = tmp.join("exe");
1480             let _ = fs::remove_dir_all(&exe);
1481
1482             let prepare = |name: &str| {
1483                 builder.create_dir(&exe.join(name));
1484                 let dir = if name == "rust-std" || name == "rust-analysis" {
1485                     format!("{}-{}", name, target.triple)
1486                 } else if name == "rls" {
1487                     "rls-preview".to_string()
1488                 } else if name == "rust-analyzer" {
1489                     "rust-analyzer-preview".to_string()
1490                 } else if name == "clippy" {
1491                     "clippy-preview".to_string()
1492                 } else if name == "rust-demangler" {
1493                     "rust-demangler-preview".to_string()
1494                 } else if name == "miri" {
1495                     "miri-preview".to_string()
1496                 } else {
1497                     name.to_string()
1498                 };
1499                 builder.cp_r(
1500                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1501                     &exe.join(name),
1502                 );
1503                 builder.remove(&exe.join(name).join("manifest.in"));
1504             };
1505             prepare("rustc");
1506             prepare("cargo");
1507             prepare("rust-analysis");
1508             prepare("rust-docs");
1509             prepare("rust-std");
1510             prepare("clippy");
1511             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1512                 if built_tools.contains(tool) {
1513                     prepare(tool);
1514                 }
1515             }
1516             if target.ends_with("windows-gnu") {
1517                 prepare("rust-mingw");
1518             }
1519
1520             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1521
1522             // Generate msi installer
1523             let wix_path = env::var_os("WIX")
1524                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1525             let wix = PathBuf::from(wix_path);
1526             let heat = wix.join("bin/heat.exe");
1527             let candle = wix.join("bin/candle.exe");
1528             let light = wix.join("bin/light.exe");
1529
1530             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1531             builder.run(
1532                 Command::new(&heat)
1533                     .current_dir(&exe)
1534                     .arg("dir")
1535                     .arg("rustc")
1536                     .args(&heat_flags)
1537                     .arg("-cg")
1538                     .arg("RustcGroup")
1539                     .arg("-dr")
1540                     .arg("Rustc")
1541                     .arg("-var")
1542                     .arg("var.RustcDir")
1543                     .arg("-out")
1544                     .arg(exe.join("RustcGroup.wxs")),
1545             );
1546             builder.run(
1547                 Command::new(&heat)
1548                     .current_dir(&exe)
1549                     .arg("dir")
1550                     .arg("rust-docs")
1551                     .args(&heat_flags)
1552                     .arg("-cg")
1553                     .arg("DocsGroup")
1554                     .arg("-dr")
1555                     .arg("Docs")
1556                     .arg("-var")
1557                     .arg("var.DocsDir")
1558                     .arg("-out")
1559                     .arg(exe.join("DocsGroup.wxs"))
1560                     .arg("-t")
1561                     .arg(etc.join("msi/squash-components.xsl")),
1562             );
1563             builder.run(
1564                 Command::new(&heat)
1565                     .current_dir(&exe)
1566                     .arg("dir")
1567                     .arg("cargo")
1568                     .args(&heat_flags)
1569                     .arg("-cg")
1570                     .arg("CargoGroup")
1571                     .arg("-dr")
1572                     .arg("Cargo")
1573                     .arg("-var")
1574                     .arg("var.CargoDir")
1575                     .arg("-out")
1576                     .arg(exe.join("CargoGroup.wxs"))
1577                     .arg("-t")
1578                     .arg(etc.join("msi/remove-duplicates.xsl")),
1579             );
1580             builder.run(
1581                 Command::new(&heat)
1582                     .current_dir(&exe)
1583                     .arg("dir")
1584                     .arg("rust-std")
1585                     .args(&heat_flags)
1586                     .arg("-cg")
1587                     .arg("StdGroup")
1588                     .arg("-dr")
1589                     .arg("Std")
1590                     .arg("-var")
1591                     .arg("var.StdDir")
1592                     .arg("-out")
1593                     .arg(exe.join("StdGroup.wxs")),
1594             );
1595             if built_tools.contains("rls") {
1596                 builder.run(
1597                     Command::new(&heat)
1598                         .current_dir(&exe)
1599                         .arg("dir")
1600                         .arg("rls")
1601                         .args(&heat_flags)
1602                         .arg("-cg")
1603                         .arg("RlsGroup")
1604                         .arg("-dr")
1605                         .arg("Rls")
1606                         .arg("-var")
1607                         .arg("var.RlsDir")
1608                         .arg("-out")
1609                         .arg(exe.join("RlsGroup.wxs"))
1610                         .arg("-t")
1611                         .arg(etc.join("msi/remove-duplicates.xsl")),
1612                 );
1613             }
1614             if built_tools.contains("rust-analyzer") {
1615                 builder.run(
1616                     Command::new(&heat)
1617                         .current_dir(&exe)
1618                         .arg("dir")
1619                         .arg("rust-analyzer")
1620                         .args(&heat_flags)
1621                         .arg("-cg")
1622                         .arg("RustAnalyzerGroup")
1623                         .arg("-dr")
1624                         .arg("RustAnalyzer")
1625                         .arg("-var")
1626                         .arg("var.RustAnalyzerDir")
1627                         .arg("-out")
1628                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1629                         .arg("-t")
1630                         .arg(etc.join("msi/remove-duplicates.xsl")),
1631                 );
1632             }
1633             builder.run(
1634                 Command::new(&heat)
1635                     .current_dir(&exe)
1636                     .arg("dir")
1637                     .arg("clippy")
1638                     .args(&heat_flags)
1639                     .arg("-cg")
1640                     .arg("ClippyGroup")
1641                     .arg("-dr")
1642                     .arg("Clippy")
1643                     .arg("-var")
1644                     .arg("var.ClippyDir")
1645                     .arg("-out")
1646                     .arg(exe.join("ClippyGroup.wxs"))
1647                     .arg("-t")
1648                     .arg(etc.join("msi/remove-duplicates.xsl")),
1649             );
1650             if built_tools.contains("rust-demangler") {
1651                 builder.run(
1652                     Command::new(&heat)
1653                         .current_dir(&exe)
1654                         .arg("dir")
1655                         .arg("rust-demangler")
1656                         .args(&heat_flags)
1657                         .arg("-cg")
1658                         .arg("RustDemanglerGroup")
1659                         .arg("-dr")
1660                         .arg("RustDemangler")
1661                         .arg("-var")
1662                         .arg("var.RustDemanglerDir")
1663                         .arg("-out")
1664                         .arg(exe.join("RustDemanglerGroup.wxs"))
1665                         .arg("-t")
1666                         .arg(etc.join("msi/remove-duplicates.xsl")),
1667                 );
1668             }
1669             if built_tools.contains("miri") {
1670                 builder.run(
1671                     Command::new(&heat)
1672                         .current_dir(&exe)
1673                         .arg("dir")
1674                         .arg("miri")
1675                         .args(&heat_flags)
1676                         .arg("-cg")
1677                         .arg("MiriGroup")
1678                         .arg("-dr")
1679                         .arg("Miri")
1680                         .arg("-var")
1681                         .arg("var.MiriDir")
1682                         .arg("-out")
1683                         .arg(exe.join("MiriGroup.wxs"))
1684                         .arg("-t")
1685                         .arg(etc.join("msi/remove-duplicates.xsl")),
1686                 );
1687             }
1688             builder.run(
1689                 Command::new(&heat)
1690                     .current_dir(&exe)
1691                     .arg("dir")
1692                     .arg("rust-analysis")
1693                     .args(&heat_flags)
1694                     .arg("-cg")
1695                     .arg("AnalysisGroup")
1696                     .arg("-dr")
1697                     .arg("Analysis")
1698                     .arg("-var")
1699                     .arg("var.AnalysisDir")
1700                     .arg("-out")
1701                     .arg(exe.join("AnalysisGroup.wxs"))
1702                     .arg("-t")
1703                     .arg(etc.join("msi/remove-duplicates.xsl")),
1704             );
1705             if target.ends_with("windows-gnu") {
1706                 builder.run(
1707                     Command::new(&heat)
1708                         .current_dir(&exe)
1709                         .arg("dir")
1710                         .arg("rust-mingw")
1711                         .args(&heat_flags)
1712                         .arg("-cg")
1713                         .arg("GccGroup")
1714                         .arg("-dr")
1715                         .arg("Gcc")
1716                         .arg("-var")
1717                         .arg("var.GccDir")
1718                         .arg("-out")
1719                         .arg(exe.join("GccGroup.wxs")),
1720                 );
1721             }
1722
1723             let candle = |input: &Path| {
1724                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1725                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1726                 let mut cmd = Command::new(&candle);
1727                 cmd.current_dir(&exe)
1728                     .arg("-nologo")
1729                     .arg("-dRustcDir=rustc")
1730                     .arg("-dDocsDir=rust-docs")
1731                     .arg("-dCargoDir=cargo")
1732                     .arg("-dStdDir=rust-std")
1733                     .arg("-dAnalysisDir=rust-analysis")
1734                     .arg("-dClippyDir=clippy")
1735                     .arg("-arch")
1736                     .arg(&arch)
1737                     .arg("-out")
1738                     .arg(&output)
1739                     .arg(&input);
1740                 add_env(builder, &mut cmd, target);
1741
1742                 if built_tools.contains("rust-demangler") {
1743                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1744                 }
1745                 if built_tools.contains("rls") {
1746                     cmd.arg("-dRlsDir=rls");
1747                 }
1748                 if built_tools.contains("rust-analyzer") {
1749                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1750                 }
1751                 if built_tools.contains("miri") {
1752                     cmd.arg("-dMiriDir=miri");
1753                 }
1754                 if target.ends_with("windows-gnu") {
1755                     cmd.arg("-dGccDir=rust-mingw");
1756                 }
1757                 builder.run(&mut cmd);
1758             };
1759             candle(&xform(&etc.join("msi/rust.wxs")));
1760             candle(&etc.join("msi/ui.wxs"));
1761             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1762             candle("RustcGroup.wxs".as_ref());
1763             candle("DocsGroup.wxs".as_ref());
1764             candle("CargoGroup.wxs".as_ref());
1765             candle("StdGroup.wxs".as_ref());
1766             candle("ClippyGroup.wxs".as_ref());
1767             if built_tools.contains("rust-demangler") {
1768                 candle("RustDemanglerGroup.wxs".as_ref());
1769             }
1770             if built_tools.contains("rls") {
1771                 candle("RlsGroup.wxs".as_ref());
1772             }
1773             if built_tools.contains("rust-analyzer") {
1774                 candle("RustAnalyzerGroup.wxs".as_ref());
1775             }
1776             if built_tools.contains("miri") {
1777                 candle("MiriGroup.wxs".as_ref());
1778             }
1779             candle("AnalysisGroup.wxs".as_ref());
1780
1781             if target.ends_with("windows-gnu") {
1782                 candle("GccGroup.wxs".as_ref());
1783             }
1784
1785             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1786             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1787             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1788
1789             builder.info(&format!("building `msi` installer with {:?}", light));
1790             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1791             let mut cmd = Command::new(&light);
1792             cmd.arg("-nologo")
1793                 .arg("-ext")
1794                 .arg("WixUIExtension")
1795                 .arg("-ext")
1796                 .arg("WixUtilExtension")
1797                 .arg("-out")
1798                 .arg(exe.join(&filename))
1799                 .arg("rust.wixobj")
1800                 .arg("ui.wixobj")
1801                 .arg("rustwelcomedlg.wixobj")
1802                 .arg("RustcGroup.wixobj")
1803                 .arg("DocsGroup.wixobj")
1804                 .arg("CargoGroup.wixobj")
1805                 .arg("StdGroup.wixobj")
1806                 .arg("AnalysisGroup.wixobj")
1807                 .arg("ClippyGroup.wixobj")
1808                 .current_dir(&exe);
1809
1810             if built_tools.contains("rls") {
1811                 cmd.arg("RlsGroup.wixobj");
1812             }
1813             if built_tools.contains("rust-analyzer") {
1814                 cmd.arg("RustAnalyzerGroup.wixobj");
1815             }
1816             if built_tools.contains("rust-demangler") {
1817                 cmd.arg("RustDemanglerGroup.wixobj");
1818             }
1819             if built_tools.contains("miri") {
1820                 cmd.arg("MiriGroup.wixobj");
1821             }
1822
1823             if target.ends_with("windows-gnu") {
1824                 cmd.arg("GccGroup.wixobj");
1825             }
1826             // ICE57 wrongly complains about the shortcuts
1827             cmd.arg("-sice:ICE57");
1828
1829             let _time = timeit(builder);
1830             builder.run(&mut cmd);
1831
1832             if !builder.config.dry_run {
1833                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1834             }
1835         }
1836     }
1837 }
1838
1839 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1840     let mut parts = builder.version.split('.');
1841     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1842         .env("CFG_RELEASE_NUM", &builder.version)
1843         .env("CFG_RELEASE", builder.rust_release())
1844         .env("CFG_VER_MAJOR", parts.next().unwrap())
1845         .env("CFG_VER_MINOR", parts.next().unwrap())
1846         .env("CFG_VER_PATCH", parts.next().unwrap())
1847         .env("CFG_VER_BUILD", "0") // just needed to build
1848         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1849         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1850         .env("CFG_BUILD", target.triple)
1851         .env("CFG_CHANNEL", &builder.config.channel);
1852
1853     if target.contains("windows-gnullvm") {
1854         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1855     } else if target.contains("windows-gnu") {
1856         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1857     } else {
1858         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1859     }
1860 }
1861
1862 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1863 ///
1864 /// Returns whether the files were actually copied.
1865 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1866     if let Some(config) = builder.config.target_config.get(&target) {
1867         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1868             // If the LLVM was externally provided, then we don't currently copy
1869             // artifacts into the sysroot. This is not necessarily the right
1870             // choice (in particular, it will require the LLVM dylib to be in
1871             // the linker's load path at runtime), but the common use case for
1872             // external LLVMs is distribution provided LLVMs, and in that case
1873             // they're usually in the standard search path (e.g., /usr/lib) and
1874             // copying them here is going to cause problems as we may end up
1875             // with the wrong files and isn't what distributions want.
1876             //
1877             // This behavior may be revisited in the future though.
1878             //
1879             // If the LLVM is coming from ourselves (just from CI) though, we
1880             // still want to install it, as it otherwise won't be available.
1881             return false;
1882         }
1883     }
1884
1885     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1886     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1887     // clear why this is the case, though. llvm-config will emit the versioned
1888     // paths and we don't want those in the sysroot (as we're expecting
1889     // unversioned paths).
1890     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1891         let src_libdir = builder.llvm_out(target).join("lib");
1892         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1893         if llvm_dylib_path.exists() {
1894             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1895         }
1896         !builder.config.dry_run
1897     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1898         let mut cmd = Command::new(llvm_config);
1899         cmd.arg("--libfiles");
1900         builder.verbose(&format!("running {:?}", cmd));
1901         let files = output(&mut cmd);
1902         let build_llvm_out = &builder.llvm_out(builder.config.build);
1903         let target_llvm_out = &builder.llvm_out(target);
1904         for file in files.trim_end().split(' ') {
1905             // If we're not using a custom LLVM, make sure we package for the target.
1906             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1907                 target_llvm_out.join(relative_path)
1908             } else {
1909                 PathBuf::from(file)
1910             };
1911             builder.install(&file, dst_libdir, 0o644);
1912         }
1913         !builder.config.dry_run
1914     } else {
1915         false
1916     }
1917 }
1918
1919 /// Maybe add libLLVM.so to the target lib-dir for linking.
1920 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1921     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1922     // We do not need to copy LLVM files into the sysroot if it is not
1923     // dynamically linked; it is already included into librustc_llvm
1924     // statically.
1925     if builder.llvm_link_shared() {
1926         maybe_install_llvm(builder, target, &dst_libdir);
1927     }
1928 }
1929
1930 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1931 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1932     let dst_libdir =
1933         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1934     // We do not need to copy LLVM files into the sysroot if it is not
1935     // dynamically linked; it is already included into librustc_llvm
1936     // statically.
1937     if builder.llvm_link_shared() {
1938         maybe_install_llvm(builder, target, &dst_libdir);
1939     }
1940 }
1941
1942 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1943 pub struct LlvmTools {
1944     pub target: TargetSelection,
1945 }
1946
1947 impl Step for LlvmTools {
1948     type Output = Option<GeneratedTarball>;
1949     const ONLY_HOSTS: bool = true;
1950     const DEFAULT: bool = true;
1951
1952     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1953         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1954         // FIXME: allow using the names of the tools themselves?
1955         run.alias("llvm-tools").default_condition(default)
1956     }
1957
1958     fn make_run(run: RunConfig<'_>) {
1959         run.builder.ensure(LlvmTools { target: run.target });
1960     }
1961
1962     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1963         let target = self.target;
1964
1965         /* run only if llvm-config isn't used */
1966         if let Some(config) = builder.config.target_config.get(&target) {
1967             if let Some(ref _s) = config.llvm_config {
1968                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1969                 return None;
1970             }
1971         }
1972
1973         builder.ensure(crate::native::Llvm { target });
1974
1975         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1976         tarball.set_overlay(OverlayKind::LLVM);
1977         tarball.is_preview(true);
1978
1979         // Prepare the image directory
1980         let src_bindir = builder.llvm_out(target).join("bin");
1981         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1982         for tool in LLVM_TOOLS {
1983             let exe = src_bindir.join(exe(tool, target));
1984             tarball.add_file(&exe, &dst_bindir, 0o755);
1985         }
1986
1987         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1988         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1989         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1990         // compiler libraries.
1991         maybe_install_llvm_target(builder, target, tarball.image_dir());
1992
1993         Some(tarball.generate())
1994     }
1995 }
1996
1997 // Tarball intended for internal consumption to ease rustc/std development.
1998 //
1999 // Should not be considered stable by end users.
2000 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2001 pub struct RustDev {
2002     pub target: TargetSelection,
2003 }
2004
2005 impl Step for RustDev {
2006     type Output = Option<GeneratedTarball>;
2007     const DEFAULT: bool = true;
2008     const ONLY_HOSTS: bool = true;
2009
2010     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2011         run.alias("rust-dev")
2012     }
2013
2014     fn make_run(run: RunConfig<'_>) {
2015         run.builder.ensure(RustDev { target: run.target });
2016     }
2017
2018     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2019         let target = self.target;
2020
2021         /* run only if llvm-config isn't used */
2022         if let Some(config) = builder.config.target_config.get(&target) {
2023             if let Some(ref _s) = config.llvm_config {
2024                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2025                 return None;
2026             }
2027         }
2028
2029         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2030         tarball.set_overlay(OverlayKind::LLVM);
2031
2032         builder.ensure(crate::native::Llvm { target });
2033
2034         let src_bindir = builder.llvm_out(target).join("bin");
2035         // If updating this list, you likely want to change
2036         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2037         // will not pick up the extra file until LLVM gets bumped.
2038         for bin in &[
2039             "llvm-config",
2040             "llvm-ar",
2041             "llvm-objdump",
2042             "llvm-profdata",
2043             "llvm-bcanalyzer",
2044             "llvm-cov",
2045             "llvm-dwp",
2046             "llvm-nm",
2047             "llvm-dwarfdump",
2048         ] {
2049             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2050         }
2051
2052         // We don't build LLD on some platforms, so only add it if it exists
2053         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2054         if lld_path.exists() {
2055             tarball.add_file(lld_path, "bin", 0o755);
2056         }
2057
2058         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2059
2060         // Copy the include directory as well; needed mostly to build
2061         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2062         // just broadly useful to be able to link against the bundled LLVM.
2063         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2064
2065         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2066         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2067         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2068         // compiler libraries.
2069         let dst_libdir = tarball.image_dir().join("lib");
2070         maybe_install_llvm(builder, target, &dst_libdir);
2071         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2072         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2073
2074         Some(tarball.generate())
2075     }
2076 }
2077
2078 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2079 /// release process to avoid cloning the monorepo and building stuff.
2080 ///
2081 /// Should not be considered stable by end users.
2082 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2083 pub struct BuildManifest {
2084     pub target: TargetSelection,
2085 }
2086
2087 impl Step for BuildManifest {
2088     type Output = GeneratedTarball;
2089     const DEFAULT: bool = false;
2090     const ONLY_HOSTS: bool = true;
2091
2092     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2093         run.alias("build-manifest")
2094     }
2095
2096     fn make_run(run: RunConfig<'_>) {
2097         run.builder.ensure(BuildManifest { target: run.target });
2098     }
2099
2100     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2101         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2102
2103         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2104         tarball.add_file(&build_manifest, "bin", 0o755);
2105         tarball.generate()
2106     }
2107 }
2108
2109 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2110 ///
2111 /// Currently this is the PGO profile data.
2112 ///
2113 /// Should not be considered stable by end users.
2114 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2115 pub struct ReproducibleArtifacts {
2116     pub target: TargetSelection,
2117 }
2118
2119 impl Step for ReproducibleArtifacts {
2120     type Output = Option<GeneratedTarball>;
2121     const DEFAULT: bool = true;
2122     const ONLY_HOSTS: bool = true;
2123
2124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2125         run.alias("reproducible-artifacts")
2126     }
2127
2128     fn make_run(run: RunConfig<'_>) {
2129         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2130     }
2131
2132     fn run(self, builder: &Builder<'_>) -> Self::Output {
2133         let mut added_anything = false;
2134         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2135         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2136             tarball.add_file(path, ".", 0o644);
2137             added_anything = true;
2138         }
2139         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2140             tarball.add_file(path, ".", 0o644);
2141             added_anything = true;
2142         }
2143         if added_anything { Some(tarball.generate()) } else { None }
2144     }
2145 }