src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
  43         panic!(
  44             "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
  45             tool_name, help
  46         )
  47     }
  48 }
  49
  50 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  51     if !builder.config.extended {
  52         return false;
  53     }
  54     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  55 }
  56
  57 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  58 pub struct Docs {
  59     pub host: TargetSelection,
  60 }
  61
  62 impl Step for Docs {
  63     type Output = Option<GeneratedTarball>;
  64     const DEFAULT: bool = true;
  65
  66     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  67         let default = run.builder.config.docs;
  68         run.alias("rust-docs").default_condition(default)
  69     }
  70
  71     fn make_run(run: RunConfig<'_>) {
  72         run.builder.ensure(Docs { host: run.target });
  73     }
  74
  75     /// Builds the `rust-docs` installer component.
  76     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  77         let host = self.host;
  78         builder.default_doc(&[]);
  79
  80         let dest = "share/doc/rust/html";
  81
  82         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  83         tarball.set_product_name("Rust Documentation");
  84         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  85         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  86         Some(tarball.generate())
  87     }
  88 }
  89
  90 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  91 pub struct RustcDocs {
  92     pub host: TargetSelection,
  93 }
  94
  95 impl Step for RustcDocs {
  96     type Output = Option<GeneratedTarball>;
  97     const DEFAULT: bool = true;
  98
  99     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 100         let builder = run.builder;
 101         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 102     }
 103
 104     fn make_run(run: RunConfig<'_>) {
 105         run.builder.ensure(RustcDocs { host: run.target });
 106     }
 107
 108     /// Builds the `rustc-docs` installer component.
 109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 110         let host = self.host;
 111         builder.default_doc(&[]);
 112
 113         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 114         tarball.set_product_name("Rustc Documentation");
 115         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 116         Some(tarball.generate())
 117     }
 118 }
 119
 120 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 121     let mut found = Vec::with_capacity(files.len());
 122
 123     for file in files {
 124         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 125
 126         if let Some(file_path) = file_path {
 127             found.push(file_path);
 128         } else {
 129             panic!("Could not find '{}' in {:?}", file, path);
 130         }
 131     }
 132
 133     found
 134 }
 135
 136 fn make_win_dist(
 137     rust_root: &Path,
 138     plat_root: &Path,
 139     target: TargetSelection,
 140     builder: &Builder<'_>,
 141 ) {
 142     //Ask gcc where it keeps its stuff
 143     let mut cmd = Command::new(builder.cc(target));
 144     cmd.arg("-print-search-dirs");
 145     let gcc_out = output(&mut cmd);
 146
 147     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 148     let mut lib_path = Vec::new();
 149
 150     for line in gcc_out.lines() {
 151         let idx = line.find(':').unwrap();
 152         let key = &line[..idx];
 153         let trim_chars: &[_] = &[' ', '='];
 154         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 155
 156         if key == "programs" {
 157             bin_path.extend(value);
 158         } else if key == "libraries" {
 159             lib_path.extend(value);
 160         }
 161     }
 162
 163     let compiler = if target == "i686-pc-windows-gnu" {
 164         "i686-w64-mingw32-gcc.exe"
 165     } else if target == "x86_64-pc-windows-gnu" {
 166         "x86_64-w64-mingw32-gcc.exe"
 167     } else {
 168         "gcc.exe"
 169     };
 170     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 171     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 172     if target.starts_with("i686-") {
 173         rustc_dlls.push("libgcc_s_dw2-1.dll");
 174     } else {
 175         rustc_dlls.push("libgcc_s_seh-1.dll");
 176     }
 177
 178     let target_libs = [
 179         //MinGW libs
 180         "libgcc.a",
 181         "libgcc_eh.a",
 182         "libgcc_s.a",
 183         "libm.a",
 184         "libmingw32.a",
 185         "libmingwex.a",
 186         "libstdc++.a",
 187         "libiconv.a",
 188         "libmoldname.a",
 189         "libpthread.a",
 190         //Windows import libs
 191         "libadvapi32.a",
 192         "libbcrypt.a",
 193         "libcomctl32.a",
 194         "libcomdlg32.a",
 195         "libcredui.a",
 196         "libcrypt32.a",
 197         "libdbghelp.a",
 198         "libgdi32.a",
 199         "libimagehlp.a",
 200         "libiphlpapi.a",
 201         "libkernel32.a",
 202         "libmsimg32.a",
 203         "libmsvcrt.a",
 204         "libodbc32.a",
 205         "libole32.a",
 206         "liboleaut32.a",
 207         "libopengl32.a",
 208         "libpsapi.a",
 209         "librpcrt4.a",
 210         "libsecur32.a",
 211         "libsetupapi.a",
 212         "libshell32.a",
 213         "libsynchronization.a",
 214         "libuser32.a",
 215         "libuserenv.a",
 216         "libuuid.a",
 217         "libwinhttp.a",
 218         "libwinmm.a",
 219         "libwinspool.a",
 220         "libws2_32.a",
 221         "libwsock32.a",
 222     ];
 223
 224     //Find mingw artifacts we want to bundle
 225     let target_tools = find_files(&target_tools, &bin_path);
 226     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 227     let target_libs = find_files(&target_libs, &lib_path);
 228
 229     // Copy runtime dlls next to rustc.exe
 230     let dist_bin_dir = rust_root.join("bin/");
 231     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 232     for src in rustc_dlls {
 233         builder.copy_to_folder(&src, &dist_bin_dir);
 234     }
 235
 236     //Copy platform tools to platform-specific bin directory
 237     let target_bin_dir = plat_root
 238         .join("lib")
 239         .join("rustlib")
 240         .join(target.triple)
 241         .join("bin")
 242         .join("self-contained");
 243     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 244     for src in target_tools {
 245         builder.copy_to_folder(&src, &target_bin_dir);
 246     }
 247
 248     // Warn windows-gnu users that the bundled GCC cannot compile C files
 249     builder.create(
 250         &target_bin_dir.join("GCC-WARNING.txt"),
 251         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 252          used as a linker. In order to be able to compile projects containing C code use \
 253          the GCC provided by MinGW or Cygwin.",
 254     );
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root
 258         .join("lib")
 259         .join("rustlib")
 260         .join(target.triple)
 261         .join("lib")
 262         .join("self-contained");
 263     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 264     for src in target_libs {
 265         builder.copy_to_folder(&src, &target_lib_dir);
 266     }
 267 }
 268
 269 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 270 pub struct Mingw {
 271     pub host: TargetSelection,
 272 }
 273
 274 impl Step for Mingw {
 275     type Output = Option<GeneratedTarball>;
 276     const DEFAULT: bool = true;
 277
 278     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 279         run.alias("rust-mingw")
 280     }
 281
 282     fn make_run(run: RunConfig<'_>) {
 283         run.builder.ensure(Mingw { host: run.target });
 284     }
 285
 286     /// Builds the `rust-mingw` installer component.
 287     ///
 288     /// This contains all the bits and pieces to run the MinGW Windows targets
 289     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 290     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 291         let host = self.host;
 292         if !host.ends_with("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 297         tarball.set_product_name("Rust MinGW");
 298
 299         // The first argument is a "temporary directory" which is just
 300         // thrown away (this contains the runtime DLLs included in the rustc package
 301         // above) and the second argument is where to place all the MinGW components
 302         // (which is what we want).
 303         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 304
 305         Some(tarball.generate())
 306     }
 307 }
 308
 309 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 310 pub struct Rustc {
 311     pub compiler: Compiler,
 312 }
 313
 314 impl Step for Rustc {
 315     type Output = GeneratedTarball;
 316     const DEFAULT: bool = true;
 317     const ONLY_HOSTS: bool = true;
 318
 319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 320         run.alias("rustc")
 321     }
 322
 323     fn make_run(run: RunConfig<'_>) {
 324         run.builder
 325             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 326     }
 327
 328     /// Creates the `rustc` installer component.
 329     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 330         let compiler = self.compiler;
 331         let host = self.compiler.host;
 332
 333         let tarball = Tarball::new(builder, "rustc", &host.triple);
 334
 335         // Prepare the rustc "image", what will actually end up getting installed
 336         prepare_image(builder, compiler, tarball.image_dir());
 337
 338         // On MinGW we've got a few runtime DLL dependencies that we need to
 339         // include. The first argument to this script is where to put these DLLs
 340         // (the image we're creating), and the second argument is a junk directory
 341         // to ignore all other MinGW stuff the script creates.
 342         //
 343         // On 32-bit MinGW we're always including a DLL which needs some extra
 344         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 345         // anything requiring us to distribute a license, but it's likely the
 346         // install will *also* include the rust-mingw package, which also needs
 347         // licenses, so to be safe we just include it here in all MinGW packages.
 348         if host.ends_with("pc-windows-gnu") {
 349             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 350             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 351         }
 352
 353         return tarball.generate();
 354
 355         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 356             let host = compiler.host;
 357             let src = builder.sysroot(compiler);
 358
 359             // Copy rustc/rustdoc binaries
 360             t!(fs::create_dir_all(image.join("bin")));
 361             builder.cp_r(&src.join("bin"), &image.join("bin"));
 362
 363             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 364
 365             let ra_proc_macro_srv = builder
 366                 .ensure(tool::RustAnalyzerProcMacroSrv {
 367                     compiler: builder.compiler_for(
 368                         compiler.stage,
 369                         builder.config.build,
 370                         compiler.host,
 371                     ),
 372                     target: compiler.host,
 373                 })
 374                 .expect("rust-analyzer-proc-macro-server always builds");
 375             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 376
 377             let libdir_relative = builder.libdir_relative(compiler);
 378
 379             // Copy runtime DLLs needed by the compiler
 380             if libdir_relative.to_str() != Some("bin") {
 381                 let libdir = builder.rustc_libdir(compiler);
 382                 for entry in builder.read_dir(&libdir) {
 383                     let name = entry.file_name();
 384                     if let Some(s) = name.to_str() {
 385                         if is_dylib(s) {
 386                             // Don't use custom libdir here because ^lib/ will be resolved again
 387                             // with installer
 388                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 389                         }
 390                     }
 391                 }
 392             }
 393
 394             // Copy over the codegen backends
 395             let backends_src = builder.sysroot_codegen_backends(compiler);
 396             let backends_rel = backends_src
 397                 .strip_prefix(&src)
 398                 .unwrap()
 399                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 400                 .unwrap();
 401             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 402             let backends_dst = image.join("lib").join(&backends_rel);
 403
 404             t!(fs::create_dir_all(&backends_dst));
 405             builder.cp_r(&backends_src, &backends_dst);
 406
 407             // Copy libLLVM.so to the lib dir as well, if needed. While not
 408             // technically needed by rustc itself it's needed by lots of other
 409             // components like the llvm tools and LLD. LLD is included below and
 410             // tools/LLDB come later, so let's just throw it in the rustc
 411             // component for now.
 412             maybe_install_llvm_runtime(builder, host, image);
 413
 414             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 415             t!(fs::create_dir_all(&dst_dir));
 416
 417             // Copy over lld if it's there
 418             if builder.config.lld_enabled {
 419                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 420                 let rust_lld = exe("rust-lld", compiler.host);
 421                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 422                 // for `-Z gcc-ld=lld`
 423                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 424                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 425                 t!(fs::create_dir(&gcc_lld_dst_dir));
 426                 for name in crate::LLD_FILE_NAMES {
 427                     let exe_name = exe(name, compiler.host);
 428                     builder
 429                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 430                 }
 431             }
 432
 433             // Man pages
 434             t!(fs::create_dir_all(image.join("share/man/man1")));
 435             let man_src = builder.src.join("src/doc/man");
 436             let man_dst = image.join("share/man/man1");
 437
 438             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 439             // to hardlink, and we don't want to edit the source templates
 440             for file_entry in builder.read_dir(&man_src) {
 441                 let page_src = file_entry.path();
 442                 let page_dst = man_dst.join(file_entry.file_name());
 443                 let src_text = t!(std::fs::read_to_string(&page_src));
 444                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 445                 t!(std::fs::write(&page_dst, &new_text));
 446                 t!(fs::copy(&page_src, &page_dst));
 447             }
 448
 449             // Debugger scripts
 450             builder
 451                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 452
 453             // Misc license info
 454             let cp = |file: &str| {
 455                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 456             };
 457             cp("COPYRIGHT");
 458             cp("LICENSE-APACHE");
 459             cp("LICENSE-MIT");
 460             cp("README.md");
 461         }
 462     }
 463 }
 464
 465 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 466 pub struct DebuggerScripts {
 467     pub sysroot: Interned<PathBuf>,
 468     pub host: TargetSelection,
 469 }
 470
 471 impl Step for DebuggerScripts {
 472     type Output = ();
 473
 474     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 475         run.never()
 476     }
 477
 478     /// Copies debugger scripts for `target` into the `sysroot` specified.
 479     fn run(self, builder: &Builder<'_>) {
 480         let host = self.host;
 481         let sysroot = self.sysroot;
 482         let dst = sysroot.join("lib/rustlib/etc");
 483         t!(fs::create_dir_all(&dst));
 484         let cp_debugger_script = |file: &str| {
 485             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 486         };
 487         if host.contains("windows-msvc") {
 488             // windbg debugger scripts
 489             builder.install(
 490                 &builder.src.join("src/etc/rust-windbg.cmd"),
 491                 &sysroot.join("bin"),
 492                 0o755,
 493             );
 494
 495             cp_debugger_script("natvis/intrinsic.natvis");
 496             cp_debugger_script("natvis/liballoc.natvis");
 497             cp_debugger_script("natvis/libcore.natvis");
 498             cp_debugger_script("natvis/libstd.natvis");
 499         } else {
 500             cp_debugger_script("rust_types.py");
 501
 502             // gdb debugger scripts
 503             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 504             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 505
 506             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 507             cp_debugger_script("gdb_lookup.py");
 508             cp_debugger_script("gdb_providers.py");
 509
 510             // lldb debugger scripts
 511             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 512
 513             cp_debugger_script("lldb_lookup.py");
 514             cp_debugger_script("lldb_providers.py");
 515             cp_debugger_script("lldb_commands")
 516         }
 517     }
 518 }
 519
 520 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 521     // The only true set of target libraries came from the build triple, so
 522     // let's reduce redundant work by only producing archives from that host.
 523     if compiler.host != builder.config.build {
 524         builder.info("\tskipping, not a build host");
 525         true
 526     } else {
 527         false
 528     }
 529 }
 530
 531 /// Copy stamped files into an image's `target/lib` directory.
 532 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 533     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 534     let self_contained_dst = dst.join("self-contained");
 535     t!(fs::create_dir_all(&dst));
 536     t!(fs::create_dir_all(&self_contained_dst));
 537     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 538         if dependency_type == DependencyType::TargetSelfContained {
 539             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 540         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 541             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 542         }
 543     }
 544 }
 545
 546 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 547 pub struct Std {
 548     pub compiler: Compiler,
 549     pub target: TargetSelection,
 550 }
 551
 552 impl Step for Std {
 553     type Output = Option<GeneratedTarball>;
 554     const DEFAULT: bool = true;
 555
 556     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 557         run.alias("rust-std")
 558     }
 559
 560     fn make_run(run: RunConfig<'_>) {
 561         run.builder.ensure(Std {
 562             compiler: run.builder.compiler_for(
 563                 run.builder.top_stage,
 564                 run.builder.config.build,
 565                 run.target,
 566             ),
 567             target: run.target,
 568         });
 569     }
 570
 571     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 572         let compiler = self.compiler;
 573         let target = self.target;
 574
 575         if skip_host_target_lib(builder, compiler) {
 576             return None;
 577         }
 578
 579         builder.ensure(compile::Std::new(compiler, target));
 580
 581         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 582         tarball.include_target_in_component_name(true);
 583
 584         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 585         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 586         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 587
 588         Some(tarball.generate())
 589     }
 590 }
 591
 592 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 593 pub struct RustcDev {
 594     pub compiler: Compiler,
 595     pub target: TargetSelection,
 596 }
 597
 598 impl Step for RustcDev {
 599     type Output = Option<GeneratedTarball>;
 600     const DEFAULT: bool = true;
 601     const ONLY_HOSTS: bool = true;
 602
 603     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 604         run.alias("rustc-dev")
 605     }
 606
 607     fn make_run(run: RunConfig<'_>) {
 608         run.builder.ensure(RustcDev {
 609             compiler: run.builder.compiler_for(
 610                 run.builder.top_stage,
 611                 run.builder.config.build,
 612                 run.target,
 613             ),
 614             target: run.target,
 615         });
 616     }
 617
 618     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 619         let compiler = self.compiler;
 620         let target = self.target;
 621         if skip_host_target_lib(builder, compiler) {
 622             return None;
 623         }
 624
 625         builder.ensure(compile::Rustc::new(compiler, target));
 626
 627         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 628
 629         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 630         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 631         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 632
 633         let src_files = &["Cargo.lock"];
 634         // This is the reduced set of paths which will become the rustc-dev component
 635         // (essentially the compiler crates and all of their path dependencies).
 636         copy_src_dirs(
 637             builder,
 638             &builder.src,
 639             &["compiler"],
 640             &[],
 641             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 642         );
 643         for file in src_files {
 644             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 645         }
 646
 647         Some(tarball.generate())
 648     }
 649 }
 650
 651 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 652 pub struct Analysis {
 653     pub compiler: Compiler,
 654     pub target: TargetSelection,
 655 }
 656
 657 impl Step for Analysis {
 658     type Output = Option<GeneratedTarball>;
 659     const DEFAULT: bool = true;
 660
 661     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 662         let default = should_build_extended_tool(&run.builder, "analysis");
 663         run.alias("rust-analysis").default_condition(default)
 664     }
 665
 666     fn make_run(run: RunConfig<'_>) {
 667         run.builder.ensure(Analysis {
 668             // Find the actual compiler (handling the full bootstrap option) which
 669             // produced the save-analysis data because that data isn't copied
 670             // through the sysroot uplifting.
 671             compiler: run.builder.compiler_for(
 672                 run.builder.top_stage,
 673                 run.builder.config.build,
 674                 run.target,
 675             ),
 676             target: run.target,
 677         });
 678     }
 679
 680     /// Creates a tarball of save-analysis metadata, if available.
 681     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 682         let compiler = self.compiler;
 683         let target = self.target;
 684         if compiler.host != builder.config.build {
 685             return None;
 686         }
 687
 688         builder.ensure(compile::Std::new(compiler, target));
 689         let src = builder
 690             .stage_out(compiler, Mode::Std)
 691             .join(target.triple)
 692             .join(builder.cargo_dir())
 693             .join("deps")
 694             .join("save-analysis");
 695
 696         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 697         tarball.include_target_in_component_name(true);
 698         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 699         Some(tarball.generate())
 700     }
 701 }
 702
 703 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 704 /// `dst_dir`.
 705 fn copy_src_dirs(
 706     builder: &Builder<'_>,
 707     base: &Path,
 708     src_dirs: &[&str],
 709     exclude_dirs: &[&str],
 710     dst_dir: &Path,
 711 ) {
 712     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 713         let spath = match path.to_str() {
 714             Some(path) => path,
 715             None => return false,
 716         };
 717         if spath.ends_with('~') || spath.ends_with(".pyc") {
 718             return false;
 719         }
 720
 721         const LLVM_PROJECTS: &[&str] = &[
 722             "llvm-project/clang",
 723             "llvm-project\\clang",
 724             "llvm-project/libunwind",
 725             "llvm-project\\libunwind",
 726             "llvm-project/lld",
 727             "llvm-project\\lld",
 728             "llvm-project/lldb",
 729             "llvm-project\\lldb",
 730             "llvm-project/llvm",
 731             "llvm-project\\llvm",
 732             "llvm-project/compiler-rt",
 733             "llvm-project\\compiler-rt",
 734             "llvm-project/cmake",
 735             "llvm-project\\cmake",
 736         ];
 737         if spath.contains("llvm-project")
 738             && !spath.ends_with("llvm-project")
 739             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 740         {
 741             return false;
 742         }
 743
 744         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 745         if LLVM_TEST.iter().any(|path| spath.contains(path))
 746             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 747         {
 748             return false;
 749         }
 750
 751         let full_path = Path::new(dir).join(path);
 752         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 753             return false;
 754         }
 755
 756         let excludes = [
 757             "CVS",
 758             "RCS",
 759             "SCCS",
 760             ".git",
 761             ".gitignore",
 762             ".gitmodules",
 763             ".gitattributes",
 764             ".cvsignore",
 765             ".svn",
 766             ".arch-ids",
 767             "{arch}",
 768             "=RELEASE-ID",
 769             "=meta-update",
 770             "=update",
 771             ".bzr",
 772             ".bzrignore",
 773             ".bzrtags",
 774             ".hg",
 775             ".hgignore",
 776             ".hgrags",
 777             "_darcs",
 778         ];
 779         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 780     }
 781
 782     // Copy the directories using our filter
 783     for item in src_dirs {
 784         let dst = &dst_dir.join(item);
 785         t!(fs::create_dir_all(dst));
 786         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 787     }
 788 }
 789
 790 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 791 pub struct Src;
 792
 793 impl Step for Src {
 794     /// The output path of the src installer tarball
 795     type Output = GeneratedTarball;
 796     const DEFAULT: bool = true;
 797     const ONLY_HOSTS: bool = true;
 798
 799     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 800         run.alias("rust-src")
 801     }
 802
 803     fn make_run(run: RunConfig<'_>) {
 804         run.builder.ensure(Src);
 805     }
 806
 807     /// Creates the `rust-src` installer component
 808     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 809         let tarball = Tarball::new_targetless(builder, "rust-src");
 810
 811         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 812         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 813         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 814         // and fix them...
 815         //
 816         // NOTE: if you update the paths here, you also should update the "virtual" path
 817         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 818         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 819
 820         let src_files = ["Cargo.lock"];
 821         // This is the reduced set of paths which will become the rust-src component
 822         // (essentially libstd and all of its path dependencies).
 823         copy_src_dirs(
 824             builder,
 825             &builder.src,
 826             &["library", "src/llvm-project/libunwind"],
 827             &[
 828                 // not needed and contains symlinks which rustup currently
 829                 // chokes on when unpacking.
 830                 "library/backtrace/crates",
 831                 // these are 30MB combined and aren't necessary for building
 832                 // the standard library.
 833                 "library/stdarch/Cargo.toml",
 834                 "library/stdarch/crates/stdarch-verify",
 835                 "library/stdarch/crates/intrinsic-test",
 836             ],
 837             &dst_src,
 838         );
 839         for file in src_files.iter() {
 840             builder.copy(&builder.src.join(file), &dst_src.join(file));
 841         }
 842
 843         tarball.generate()
 844     }
 845 }
 846
 847 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 848 pub struct PlainSourceTarball;
 849
 850 impl Step for PlainSourceTarball {
 851     /// Produces the location of the tarball generated
 852     type Output = GeneratedTarball;
 853     const DEFAULT: bool = true;
 854     const ONLY_HOSTS: bool = true;
 855
 856     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 857         let builder = run.builder;
 858         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 859     }
 860
 861     fn make_run(run: RunConfig<'_>) {
 862         run.builder.ensure(PlainSourceTarball);
 863     }
 864
 865     /// Creates the plain source tarball
 866     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 867         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 868         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 869         // It also contains symbolic links, unlike other any other dist tarball.
 870         // It's used for distros building rustc from source in a pre-vendored environment.
 871         let mut tarball = Tarball::new(builder, "rustc", "src");
 872         tarball.permit_symlinks(true);
 873         let plain_dst_src = tarball.image_dir();
 874
 875         // This is the set of root paths which will become part of the source package
 876         let src_files = [
 877             "COPYRIGHT",
 878             "LICENSE-APACHE",
 879             "LICENSE-MIT",
 880             "CONTRIBUTING.md",
 881             "README.md",
 882             "RELEASES.md",
 883             "configure",
 884             "x.py",
 885             "config.toml.example",
 886             "Cargo.toml",
 887             "Cargo.lock",
 888         ];
 889         let src_dirs = ["src", "compiler", "library"];
 890
 891         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 892
 893         // Copy the files normally
 894         for item in &src_files {
 895             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 896         }
 897
 898         // Create the version file
 899         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 900         if let Some(sha) = builder.rust_sha() {
 901             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 902         }
 903
 904         // If we're building from git sources, we need to vendor a complete distribution.
 905         if builder.rust_info.is_git() {
 906             // Ensure we have the submodules checked out.
 907             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 908
 909             // Vendor all Cargo dependencies
 910             let mut cmd = Command::new(&builder.initial_cargo);
 911             cmd.arg("vendor")
 912                 .arg("--sync")
 913                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 914                 .arg("--sync")
 915                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 916                 .arg("--sync")
 917                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 918                 .current_dir(&plain_dst_src);
 919
 920             let config = if !builder.config.dry_run {
 921                 t!(String::from_utf8(t!(cmd.output()).stdout))
 922             } else {
 923                 String::new()
 924             };
 925
 926             let cargo_config_dir = plain_dst_src.join(".cargo");
 927             builder.create_dir(&cargo_config_dir);
 928             builder.create(&cargo_config_dir.join("config.toml"), &config);
 929         }
 930
 931         tarball.bare()
 932     }
 933 }
 934
 935 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 936 pub struct Cargo {
 937     pub compiler: Compiler,
 938     pub target: TargetSelection,
 939 }
 940
 941 impl Step for Cargo {
 942     type Output = Option<GeneratedTarball>;
 943     const DEFAULT: bool = true;
 944     const ONLY_HOSTS: bool = true;
 945
 946     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 947         let default = should_build_extended_tool(&run.builder, "cargo");
 948         run.alias("cargo").default_condition(default)
 949     }
 950
 951     fn make_run(run: RunConfig<'_>) {
 952         run.builder.ensure(Cargo {
 953             compiler: run.builder.compiler_for(
 954                 run.builder.top_stage,
 955                 run.builder.config.build,
 956                 run.target,
 957             ),
 958             target: run.target,
 959         });
 960     }
 961
 962     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 963         let compiler = self.compiler;
 964         let target = self.target;
 965
 966         let cargo = builder.ensure(tool::Cargo { compiler, target });
 967         let src = builder.src.join("src/tools/cargo");
 968         let etc = src.join("src/etc");
 969
 970         // Prepare the image directory
 971         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 972         tarball.set_overlay(OverlayKind::Cargo);
 973
 974         tarball.add_file(&cargo, "bin", 0o755);
 975         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 976         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 977         tarball.add_dir(etc.join("man"), "share/man/man1");
 978         tarball.add_legal_and_readme_to("share/doc/cargo");
 979
 980         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 981             let dirent = dirent.expect("read dir entry");
 982             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 983                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 984             }
 985         }
 986
 987         Some(tarball.generate())
 988     }
 989 }
 990
 991 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 992 pub struct Rls {
 993     pub compiler: Compiler,
 994     pub target: TargetSelection,
 995 }
 996
 997 impl Step for Rls {
 998     type Output = Option<GeneratedTarball>;
 999     const ONLY_HOSTS: bool = true;
1000     const DEFAULT: bool = true;
1001
1002     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1003         let default = should_build_extended_tool(&run.builder, "rls");
1004         run.alias("rls").default_condition(default)
1005     }
1006
1007     fn make_run(run: RunConfig<'_>) {
1008         run.builder.ensure(Rls {
1009             compiler: run.builder.compiler_for(
1010                 run.builder.top_stage,
1011                 run.builder.config.build,
1012                 run.target,
1013             ),
1014             target: run.target,
1015         });
1016     }
1017
1018     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1019         let compiler = self.compiler;
1020         let target = self.target;
1021
1022         let rls = builder
1023             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1024             .expect("rls expected to build");
1025
1026         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1027         tarball.set_overlay(OverlayKind::RLS);
1028         tarball.is_preview(true);
1029         tarball.add_file(rls, "bin", 0o755);
1030         tarball.add_legal_and_readme_to("share/doc/rls");
1031         Some(tarball.generate())
1032     }
1033 }
1034
1035 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1036 pub struct RustAnalyzer {
1037     pub compiler: Compiler,
1038     pub target: TargetSelection,
1039 }
1040
1041 impl Step for RustAnalyzer {
1042     type Output = Option<GeneratedTarball>;
1043     const DEFAULT: bool = true;
1044     const ONLY_HOSTS: bool = true;
1045
1046     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1047         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1048         run.alias("rust-analyzer").default_condition(default)
1049     }
1050
1051     fn make_run(run: RunConfig<'_>) {
1052         run.builder.ensure(RustAnalyzer {
1053             compiler: run.builder.compiler_for(
1054                 run.builder.top_stage,
1055                 run.builder.config.build,
1056                 run.target,
1057             ),
1058             target: run.target,
1059         });
1060     }
1061
1062     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1063         let compiler = self.compiler;
1064         let target = self.target;
1065
1066         if target.contains("riscv64") {
1067             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1068             // to build. See #74813 for details.
1069             return None;
1070         }
1071
1072         let rust_analyzer = builder
1073             .ensure(tool::RustAnalyzer { compiler, target })
1074             .expect("rust-analyzer always builds");
1075
1076         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1077         tarball.set_overlay(OverlayKind::RustAnalyzer);
1078         tarball.is_preview(true);
1079         tarball.add_file(rust_analyzer, "bin", 0o755);
1080         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1081         Some(tarball.generate())
1082     }
1083 }
1084
1085 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1086 pub struct Clippy {
1087     pub compiler: Compiler,
1088     pub target: TargetSelection,
1089 }
1090
1091 impl Step for Clippy {
1092     type Output = Option<GeneratedTarball>;
1093     const DEFAULT: bool = true;
1094     const ONLY_HOSTS: bool = true;
1095
1096     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1097         let default = should_build_extended_tool(&run.builder, "clippy");
1098         run.alias("clippy").default_condition(default)
1099     }
1100
1101     fn make_run(run: RunConfig<'_>) {
1102         run.builder.ensure(Clippy {
1103             compiler: run.builder.compiler_for(
1104                 run.builder.top_stage,
1105                 run.builder.config.build,
1106                 run.target,
1107             ),
1108             target: run.target,
1109         });
1110     }
1111
1112     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1113         let compiler = self.compiler;
1114         let target = self.target;
1115
1116         // Prepare the image directory
1117         // We expect clippy to build, because we've exited this step above if tool
1118         // state for clippy isn't testing.
1119         let clippy = builder
1120             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1121             .expect("clippy expected to build - essential tool");
1122         let cargoclippy = builder
1123             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1124             .expect("clippy expected to build - essential tool");
1125
1126         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1127         tarball.set_overlay(OverlayKind::Clippy);
1128         tarball.is_preview(true);
1129         tarball.add_file(clippy, "bin", 0o755);
1130         tarball.add_file(cargoclippy, "bin", 0o755);
1131         tarball.add_legal_and_readme_to("share/doc/clippy");
1132         Some(tarball.generate())
1133     }
1134 }
1135
1136 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1137 pub struct Miri {
1138     pub compiler: Compiler,
1139     pub target: TargetSelection,
1140 }
1141
1142 impl Step for Miri {
1143     type Output = Option<GeneratedTarball>;
1144     const DEFAULT: bool = true;
1145     const ONLY_HOSTS: bool = true;
1146
1147     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1148         let default = should_build_extended_tool(&run.builder, "miri");
1149         run.alias("miri").default_condition(default)
1150     }
1151
1152     fn make_run(run: RunConfig<'_>) {
1153         run.builder.ensure(Miri {
1154             compiler: run.builder.compiler_for(
1155                 run.builder.top_stage,
1156                 run.builder.config.build,
1157                 run.target,
1158             ),
1159             target: run.target,
1160         });
1161     }
1162
1163     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1164         // This prevents miri from being built for "dist" or "install"
1165         // on the stable/beta channels. It is a nightly-only tool and should
1166         // not be included.
1167         if !builder.build.unstable_features() {
1168             return None;
1169         }
1170         let compiler = self.compiler;
1171         let target = self.target;
1172
1173         let miri = builder
1174             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1175             .or_else(|| {
1176                 missing_tool("miri", builder.build.config.missing_tools);
1177                 None
1178             })?;
1179         let cargomiri = builder
1180             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1181             .or_else(|| {
1182                 missing_tool("cargo miri", builder.build.config.missing_tools);
1183                 None
1184             })?;
1185
1186         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1187         tarball.set_overlay(OverlayKind::Miri);
1188         tarball.is_preview(true);
1189         tarball.add_file(miri, "bin", 0o755);
1190         tarball.add_file(cargomiri, "bin", 0o755);
1191         tarball.add_legal_and_readme_to("share/doc/miri");
1192         Some(tarball.generate())
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Rustfmt {
1198     pub compiler: Compiler,
1199     pub target: TargetSelection,
1200 }
1201
1202 impl Step for Rustfmt {
1203     type Output = Option<GeneratedTarball>;
1204     const DEFAULT: bool = true;
1205     const ONLY_HOSTS: bool = true;
1206
1207     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1208         let default = should_build_extended_tool(&run.builder, "rustfmt");
1209         run.alias("rustfmt").default_condition(default)
1210     }
1211
1212     fn make_run(run: RunConfig<'_>) {
1213         run.builder.ensure(Rustfmt {
1214             compiler: run.builder.compiler_for(
1215                 run.builder.top_stage,
1216                 run.builder.config.build,
1217                 run.target,
1218             ),
1219             target: run.target,
1220         });
1221     }
1222
1223     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1224         let compiler = self.compiler;
1225         let target = self.target;
1226
1227         let rustfmt = builder
1228             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1229             .expect("rustfmt expected to build - essential tool");
1230         let cargofmt = builder
1231             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1232             .expect("cargo fmt expected to build - essential tool");
1233         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1234         tarball.set_overlay(OverlayKind::Rustfmt);
1235         tarball.is_preview(true);
1236         tarball.add_file(rustfmt, "bin", 0o755);
1237         tarball.add_file(cargofmt, "bin", 0o755);
1238         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1239         Some(tarball.generate())
1240     }
1241 }
1242
1243 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1244 pub struct RustDemangler {
1245     pub compiler: Compiler,
1246     pub target: TargetSelection,
1247 }
1248
1249 impl Step for RustDemangler {
1250     type Output = Option<GeneratedTarball>;
1251     const DEFAULT: bool = true;
1252     const ONLY_HOSTS: bool = true;
1253
1254     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1255         // While other tools use `should_build_extended_tool` to decide whether to be run by
1256         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1257         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1258         // we run the step by default when only `extended = true`, and decide whether to actually
1259         // run it or not later.
1260         let default = run.builder.config.extended;
1261         run.alias("rust-demangler").default_condition(default)
1262     }
1263
1264     fn make_run(run: RunConfig<'_>) {
1265         run.builder.ensure(RustDemangler {
1266             compiler: run.builder.compiler_for(
1267                 run.builder.top_stage,
1268                 run.builder.config.build,
1269                 run.target,
1270             ),
1271             target: run.target,
1272         });
1273     }
1274
1275     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1276         let compiler = self.compiler;
1277         let target = self.target;
1278
1279         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1280         let condition = should_build_extended_tool(builder, "rust-demangler")
1281             || builder.config.profiler_enabled(target);
1282         if builder.config.extended && !condition {
1283             return None;
1284         }
1285
1286         let rust_demangler = builder
1287             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1288             .expect("rust-demangler expected to build - in-tree tool");
1289
1290         // Prepare the image directory
1291         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1292         tarball.set_overlay(OverlayKind::RustDemangler);
1293         tarball.is_preview(true);
1294         tarball.add_file(&rust_demangler, "bin", 0o755);
1295         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1296         Some(tarball.generate())
1297     }
1298 }
1299
1300 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1301 pub struct Extended {
1302     stage: u32,
1303     host: TargetSelection,
1304     target: TargetSelection,
1305 }
1306
1307 impl Step for Extended {
1308     type Output = ();
1309     const DEFAULT: bool = true;
1310     const ONLY_HOSTS: bool = true;
1311
1312     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1313         let builder = run.builder;
1314         run.alias("extended").default_condition(builder.config.extended)
1315     }
1316
1317     fn make_run(run: RunConfig<'_>) {
1318         run.builder.ensure(Extended {
1319             stage: run.builder.top_stage,
1320             host: run.builder.config.build,
1321             target: run.target,
1322         });
1323     }
1324
1325     /// Creates a combined installer for the specified target in the provided stage.
1326     fn run(self, builder: &Builder<'_>) {
1327         let target = self.target;
1328         let stage = self.stage;
1329         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1330
1331         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1332
1333         let mut tarballs = Vec::new();
1334         let mut built_tools = HashSet::new();
1335         macro_rules! add_component {
1336             ($name:expr => $step:expr) => {
1337                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1338                     tarballs.push(tarball);
1339                     built_tools.insert($name);
1340                 }
1341             };
1342         }
1343
1344         // When rust-std package split from rustc, we needed to ensure that during
1345         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1346         // the std files during uninstall. To do this ensure that rustc comes
1347         // before rust-std in the list below.
1348         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1349         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1350
1351         if target.ends_with("windows-gnu") {
1352             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1353         }
1354
1355         add_component!("rust-docs" => Docs { host: target });
1356         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1357         add_component!("cargo" => Cargo { compiler, target });
1358         add_component!("rustfmt" => Rustfmt { compiler, target });
1359         add_component!("rls" => Rls { compiler, target });
1360         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1361         add_component!("llvm-components" => LlvmTools { target });
1362         add_component!("clippy" => Clippy { compiler, target });
1363         add_component!("miri" => Miri { compiler, target });
1364         add_component!("analysis" => Analysis { compiler, target });
1365
1366         let etc = builder.src.join("src/etc/installer");
1367
1368         // Avoid producing tarballs during a dry run.
1369         if builder.config.dry_run {
1370             return;
1371         }
1372
1373         let tarball = Tarball::new(builder, "rust", &target.triple);
1374         let generated = tarball.combine(&tarballs);
1375
1376         let tmp = tmpdir(builder).join("combined-tarball");
1377         let work = generated.work_dir();
1378
1379         let mut license = String::new();
1380         license += &builder.read(&builder.src.join("COPYRIGHT"));
1381         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1382         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1383         license.push('\n');
1384         license.push('\n');
1385
1386         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1387         let mut rtf = rtf.to_string();
1388         rtf.push('\n');
1389         for line in license.lines() {
1390             rtf.push_str(line);
1391             rtf.push_str("\\line ");
1392         }
1393         rtf.push('}');
1394
1395         fn filter(contents: &str, marker: &str) -> String {
1396             let start = format!("tool-{}-start", marker);
1397             let end = format!("tool-{}-end", marker);
1398             let mut lines = Vec::new();
1399             let mut omitted = false;
1400             for line in contents.lines() {
1401                 if line.contains(&start) {
1402                     omitted = true;
1403                 } else if line.contains(&end) {
1404                     omitted = false;
1405                 } else if !omitted {
1406                     lines.push(line);
1407                 }
1408             }
1409
1410             lines.join("\n")
1411         }
1412
1413         let xform = |p: &Path| {
1414             let mut contents = t!(fs::read_to_string(p));
1415             for tool in &["rust-demangler", "rust-analyzer", "miri", "rustfmt"] {
1416                 if !built_tools.contains(tool) {
1417                     contents = filter(&contents, tool);
1418                 }
1419             }
1420             let ret = tmp.join(p.file_name().unwrap());
1421             t!(fs::write(&ret, &contents));
1422             ret
1423         };
1424
1425         if target.contains("apple-darwin") {
1426             builder.info("building pkg installer");
1427             let pkg = tmp.join("pkg");
1428             let _ = fs::remove_dir_all(&pkg);
1429
1430             let pkgbuild = |component: &str| {
1431                 let mut cmd = Command::new("pkgbuild");
1432                 cmd.arg("--identifier")
1433                     .arg(format!("org.rust-lang.{}", component))
1434                     .arg("--scripts")
1435                     .arg(pkg.join(component))
1436                     .arg("--nopayload")
1437                     .arg(pkg.join(component).with_extension("pkg"));
1438                 builder.run(&mut cmd);
1439             };
1440
1441             let prepare = |name: &str| {
1442                 builder.create_dir(&pkg.join(name));
1443                 builder.cp_r(
1444                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1445                     &pkg.join(name),
1446                 );
1447                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1448                 pkgbuild(name);
1449             };
1450             prepare("rustc");
1451             prepare("cargo");
1452             prepare("rust-std");
1453             prepare("rust-analysis");
1454             prepare("clippy");
1455             for tool in &["rust-docs", "rust-demangler", "rust-analyzer", "miri"] {
1456                 if built_tools.contains(tool) {
1457                     prepare(tool);
1458                 }
1459             }
1460             // create an 'uninstall' package
1461             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1462             pkgbuild("uninstall");
1463
1464             builder.create_dir(&pkg.join("res"));
1465             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1466             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1467             let mut cmd = Command::new("productbuild");
1468             cmd.arg("--distribution")
1469                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1470                 .arg("--resources")
1471                 .arg(pkg.join("res"))
1472                 .arg(distdir(builder).join(format!(
1473                     "{}-{}.pkg",
1474                     pkgname(builder, "rust"),
1475                     target.triple
1476                 )))
1477                 .arg("--package-path")
1478                 .arg(&pkg);
1479             let _time = timeit(builder);
1480             builder.run(&mut cmd);
1481         }
1482
1483         if target.contains("windows") {
1484             let exe = tmp.join("exe");
1485             let _ = fs::remove_dir_all(&exe);
1486
1487             let prepare = |name: &str| {
1488                 builder.create_dir(&exe.join(name));
1489                 let dir = if name == "rust-std" || name == "rust-analysis" {
1490                     format!("{}-{}", name, target.triple)
1491                 } else if name == "rust-analyzer" {
1492                     "rust-analyzer-preview".to_string()
1493                 } else if name == "clippy" {
1494                     "clippy-preview".to_string()
1495                 } else if name == "rust-demangler" {
1496                     "rust-demangler-preview".to_string()
1497                 } else if name == "miri" {
1498                     "miri-preview".to_string()
1499                 } else {
1500                     name.to_string()
1501                 };
1502                 builder.cp_r(
1503                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1504                     &exe.join(name),
1505                 );
1506                 builder.remove(&exe.join(name).join("manifest.in"));
1507             };
1508             prepare("rustc");
1509             prepare("cargo");
1510             prepare("rust-analysis");
1511             prepare("rust-docs");
1512             prepare("rust-std");
1513             prepare("clippy");
1514             for tool in &["rust-demangler", "rust-analyzer", "miri"] {
1515                 if built_tools.contains(tool) {
1516                     prepare(tool);
1517                 }
1518             }
1519             if target.ends_with("windows-gnu") {
1520                 prepare("rust-mingw");
1521             }
1522
1523             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1524
1525             // Generate msi installer
1526             let wix_path = env::var_os("WIX")
1527                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1528             let wix = PathBuf::from(wix_path);
1529             let heat = wix.join("bin/heat.exe");
1530             let candle = wix.join("bin/candle.exe");
1531             let light = wix.join("bin/light.exe");
1532
1533             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1534             builder.run(
1535                 Command::new(&heat)
1536                     .current_dir(&exe)
1537                     .arg("dir")
1538                     .arg("rustc")
1539                     .args(&heat_flags)
1540                     .arg("-cg")
1541                     .arg("RustcGroup")
1542                     .arg("-dr")
1543                     .arg("Rustc")
1544                     .arg("-var")
1545                     .arg("var.RustcDir")
1546                     .arg("-out")
1547                     .arg(exe.join("RustcGroup.wxs")),
1548             );
1549             builder.run(
1550                 Command::new(&heat)
1551                     .current_dir(&exe)
1552                     .arg("dir")
1553                     .arg("rust-docs")
1554                     .args(&heat_flags)
1555                     .arg("-cg")
1556                     .arg("DocsGroup")
1557                     .arg("-dr")
1558                     .arg("Docs")
1559                     .arg("-var")
1560                     .arg("var.DocsDir")
1561                     .arg("-out")
1562                     .arg(exe.join("DocsGroup.wxs"))
1563                     .arg("-t")
1564                     .arg(etc.join("msi/squash-components.xsl")),
1565             );
1566             builder.run(
1567                 Command::new(&heat)
1568                     .current_dir(&exe)
1569                     .arg("dir")
1570                     .arg("cargo")
1571                     .args(&heat_flags)
1572                     .arg("-cg")
1573                     .arg("CargoGroup")
1574                     .arg("-dr")
1575                     .arg("Cargo")
1576                     .arg("-var")
1577                     .arg("var.CargoDir")
1578                     .arg("-out")
1579                     .arg(exe.join("CargoGroup.wxs"))
1580                     .arg("-t")
1581                     .arg(etc.join("msi/remove-duplicates.xsl")),
1582             );
1583             builder.run(
1584                 Command::new(&heat)
1585                     .current_dir(&exe)
1586                     .arg("dir")
1587                     .arg("rust-std")
1588                     .args(&heat_flags)
1589                     .arg("-cg")
1590                     .arg("StdGroup")
1591                     .arg("-dr")
1592                     .arg("Std")
1593                     .arg("-var")
1594                     .arg("var.StdDir")
1595                     .arg("-out")
1596                     .arg(exe.join("StdGroup.wxs")),
1597             );
1598             if built_tools.contains("rust-analyzer") {
1599                 builder.run(
1600                     Command::new(&heat)
1601                         .current_dir(&exe)
1602                         .arg("dir")
1603                         .arg("rust-analyzer")
1604                         .args(&heat_flags)
1605                         .arg("-cg")
1606                         .arg("RustAnalyzerGroup")
1607                         .arg("-dr")
1608                         .arg("RustAnalyzer")
1609                         .arg("-var")
1610                         .arg("var.RustAnalyzerDir")
1611                         .arg("-out")
1612                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1613                         .arg("-t")
1614                         .arg(etc.join("msi/remove-duplicates.xsl")),
1615                 );
1616             }
1617             builder.run(
1618                 Command::new(&heat)
1619                     .current_dir(&exe)
1620                     .arg("dir")
1621                     .arg("clippy")
1622                     .args(&heat_flags)
1623                     .arg("-cg")
1624                     .arg("ClippyGroup")
1625                     .arg("-dr")
1626                     .arg("Clippy")
1627                     .arg("-var")
1628                     .arg("var.ClippyDir")
1629                     .arg("-out")
1630                     .arg(exe.join("ClippyGroup.wxs"))
1631                     .arg("-t")
1632                     .arg(etc.join("msi/remove-duplicates.xsl")),
1633             );
1634             if built_tools.contains("rust-demangler") {
1635                 builder.run(
1636                     Command::new(&heat)
1637                         .current_dir(&exe)
1638                         .arg("dir")
1639                         .arg("rust-demangler")
1640                         .args(&heat_flags)
1641                         .arg("-cg")
1642                         .arg("RustDemanglerGroup")
1643                         .arg("-dr")
1644                         .arg("RustDemangler")
1645                         .arg("-var")
1646                         .arg("var.RustDemanglerDir")
1647                         .arg("-out")
1648                         .arg(exe.join("RustDemanglerGroup.wxs"))
1649                         .arg("-t")
1650                         .arg(etc.join("msi/remove-duplicates.xsl")),
1651                 );
1652             }
1653             if built_tools.contains("miri") {
1654                 builder.run(
1655                     Command::new(&heat)
1656                         .current_dir(&exe)
1657                         .arg("dir")
1658                         .arg("miri")
1659                         .args(&heat_flags)
1660                         .arg("-cg")
1661                         .arg("MiriGroup")
1662                         .arg("-dr")
1663                         .arg("Miri")
1664                         .arg("-var")
1665                         .arg("var.MiriDir")
1666                         .arg("-out")
1667                         .arg(exe.join("MiriGroup.wxs"))
1668                         .arg("-t")
1669                         .arg(etc.join("msi/remove-duplicates.xsl")),
1670                 );
1671             }
1672             builder.run(
1673                 Command::new(&heat)
1674                     .current_dir(&exe)
1675                     .arg("dir")
1676                     .arg("rust-analysis")
1677                     .args(&heat_flags)
1678                     .arg("-cg")
1679                     .arg("AnalysisGroup")
1680                     .arg("-dr")
1681                     .arg("Analysis")
1682                     .arg("-var")
1683                     .arg("var.AnalysisDir")
1684                     .arg("-out")
1685                     .arg(exe.join("AnalysisGroup.wxs"))
1686                     .arg("-t")
1687                     .arg(etc.join("msi/remove-duplicates.xsl")),
1688             );
1689             if target.ends_with("windows-gnu") {
1690                 builder.run(
1691                     Command::new(&heat)
1692                         .current_dir(&exe)
1693                         .arg("dir")
1694                         .arg("rust-mingw")
1695                         .args(&heat_flags)
1696                         .arg("-cg")
1697                         .arg("GccGroup")
1698                         .arg("-dr")
1699                         .arg("Gcc")
1700                         .arg("-var")
1701                         .arg("var.GccDir")
1702                         .arg("-out")
1703                         .arg(exe.join("GccGroup.wxs")),
1704                 );
1705             }
1706
1707             let candle = |input: &Path| {
1708                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1709                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1710                 let mut cmd = Command::new(&candle);
1711                 cmd.current_dir(&exe)
1712                     .arg("-nologo")
1713                     .arg("-dRustcDir=rustc")
1714                     .arg("-dDocsDir=rust-docs")
1715                     .arg("-dCargoDir=cargo")
1716                     .arg("-dStdDir=rust-std")
1717                     .arg("-dAnalysisDir=rust-analysis")
1718                     .arg("-dClippyDir=clippy")
1719                     .arg("-arch")
1720                     .arg(&arch)
1721                     .arg("-out")
1722                     .arg(&output)
1723                     .arg(&input);
1724                 add_env(builder, &mut cmd, target);
1725
1726                 if built_tools.contains("rust-demangler") {
1727                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1728                 }
1729                 if built_tools.contains("rust-analyzer") {
1730                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1731                 }
1732                 if built_tools.contains("miri") {
1733                     cmd.arg("-dMiriDir=miri");
1734                 }
1735                 if target.ends_with("windows-gnu") {
1736                     cmd.arg("-dGccDir=rust-mingw");
1737                 }
1738                 builder.run(&mut cmd);
1739             };
1740             candle(&xform(&etc.join("msi/rust.wxs")));
1741             candle(&etc.join("msi/ui.wxs"));
1742             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1743             candle("RustcGroup.wxs".as_ref());
1744             candle("DocsGroup.wxs".as_ref());
1745             candle("CargoGroup.wxs".as_ref());
1746             candle("StdGroup.wxs".as_ref());
1747             candle("ClippyGroup.wxs".as_ref());
1748             if built_tools.contains("rust-demangler") {
1749                 candle("RustDemanglerGroup.wxs".as_ref());
1750             }
1751             if built_tools.contains("rust-analyzer") {
1752                 candle("RustAnalyzerGroup.wxs".as_ref());
1753             }
1754             if built_tools.contains("miri") {
1755                 candle("MiriGroup.wxs".as_ref());
1756             }
1757             candle("AnalysisGroup.wxs".as_ref());
1758
1759             if target.ends_with("windows-gnu") {
1760                 candle("GccGroup.wxs".as_ref());
1761             }
1762
1763             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1764             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1765             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1766
1767             builder.info(&format!("building `msi` installer with {:?}", light));
1768             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1769             let mut cmd = Command::new(&light);
1770             cmd.arg("-nologo")
1771                 .arg("-ext")
1772                 .arg("WixUIExtension")
1773                 .arg("-ext")
1774                 .arg("WixUtilExtension")
1775                 .arg("-out")
1776                 .arg(exe.join(&filename))
1777                 .arg("rust.wixobj")
1778                 .arg("ui.wixobj")
1779                 .arg("rustwelcomedlg.wixobj")
1780                 .arg("RustcGroup.wixobj")
1781                 .arg("DocsGroup.wixobj")
1782                 .arg("CargoGroup.wixobj")
1783                 .arg("StdGroup.wixobj")
1784                 .arg("AnalysisGroup.wixobj")
1785                 .arg("ClippyGroup.wixobj")
1786                 .current_dir(&exe);
1787
1788             if built_tools.contains("rust-analyzer") {
1789                 cmd.arg("RustAnalyzerGroup.wixobj");
1790             }
1791             if built_tools.contains("rust-demangler") {
1792                 cmd.arg("RustDemanglerGroup.wixobj");
1793             }
1794             if built_tools.contains("miri") {
1795                 cmd.arg("MiriGroup.wixobj");
1796             }
1797
1798             if target.ends_with("windows-gnu") {
1799                 cmd.arg("GccGroup.wixobj");
1800             }
1801             // ICE57 wrongly complains about the shortcuts
1802             cmd.arg("-sice:ICE57");
1803
1804             let _time = timeit(builder);
1805             builder.run(&mut cmd);
1806
1807             if !builder.config.dry_run {
1808                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1809             }
1810         }
1811     }
1812 }
1813
1814 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1815     let mut parts = builder.version.split('.');
1816     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1817         .env("CFG_RELEASE_NUM", &builder.version)
1818         .env("CFG_RELEASE", builder.rust_release())
1819         .env("CFG_VER_MAJOR", parts.next().unwrap())
1820         .env("CFG_VER_MINOR", parts.next().unwrap())
1821         .env("CFG_VER_PATCH", parts.next().unwrap())
1822         .env("CFG_VER_BUILD", "0") // just needed to build
1823         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1824         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1825         .env("CFG_BUILD", target.triple)
1826         .env("CFG_CHANNEL", &builder.config.channel);
1827
1828     if target.contains("windows-gnullvm") {
1829         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1830     } else if target.contains("windows-gnu") {
1831         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1832     } else {
1833         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1834     }
1835 }
1836
1837 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1838 ///
1839 /// Returns whether the files were actually copied.
1840 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1841     if let Some(config) = builder.config.target_config.get(&target) {
1842         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1843             // If the LLVM was externally provided, then we don't currently copy
1844             // artifacts into the sysroot. This is not necessarily the right
1845             // choice (in particular, it will require the LLVM dylib to be in
1846             // the linker's load path at runtime), but the common use case for
1847             // external LLVMs is distribution provided LLVMs, and in that case
1848             // they're usually in the standard search path (e.g., /usr/lib) and
1849             // copying them here is going to cause problems as we may end up
1850             // with the wrong files and isn't what distributions want.
1851             //
1852             // This behavior may be revisited in the future though.
1853             //
1854             // If the LLVM is coming from ourselves (just from CI) though, we
1855             // still want to install it, as it otherwise won't be available.
1856             return false;
1857         }
1858     }
1859
1860     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1861     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1862     // clear why this is the case, though. llvm-config will emit the versioned
1863     // paths and we don't want those in the sysroot (as we're expecting
1864     // unversioned paths).
1865     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1866         let src_libdir = builder.llvm_out(target).join("lib");
1867         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1868         if llvm_dylib_path.exists() {
1869             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1870         }
1871         !builder.config.dry_run
1872     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1873         let mut cmd = Command::new(llvm_config);
1874         cmd.arg("--libfiles");
1875         builder.verbose(&format!("running {:?}", cmd));
1876         let files = output(&mut cmd);
1877         let build_llvm_out = &builder.llvm_out(builder.config.build);
1878         let target_llvm_out = &builder.llvm_out(target);
1879         for file in files.trim_end().split(' ') {
1880             // If we're not using a custom LLVM, make sure we package for the target.
1881             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1882                 target_llvm_out.join(relative_path)
1883             } else {
1884                 PathBuf::from(file)
1885             };
1886             builder.install(&file, dst_libdir, 0o644);
1887         }
1888         !builder.config.dry_run
1889     } else {
1890         false
1891     }
1892 }
1893
1894 /// Maybe add libLLVM.so to the target lib-dir for linking.
1895 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1896     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1897     // We do not need to copy LLVM files into the sysroot if it is not
1898     // dynamically linked; it is already included into librustc_llvm
1899     // statically.
1900     if builder.llvm_link_shared() {
1901         maybe_install_llvm(builder, target, &dst_libdir);
1902     }
1903 }
1904
1905 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1906 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1907     let dst_libdir =
1908         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1909     // We do not need to copy LLVM files into the sysroot if it is not
1910     // dynamically linked; it is already included into librustc_llvm
1911     // statically.
1912     if builder.llvm_link_shared() {
1913         maybe_install_llvm(builder, target, &dst_libdir);
1914     }
1915 }
1916
1917 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1918 pub struct LlvmTools {
1919     pub target: TargetSelection,
1920 }
1921
1922 impl Step for LlvmTools {
1923     type Output = Option<GeneratedTarball>;
1924     const ONLY_HOSTS: bool = true;
1925     const DEFAULT: bool = true;
1926
1927     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1928         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1929         // FIXME: allow using the names of the tools themselves?
1930         run.alias("llvm-tools").default_condition(default)
1931     }
1932
1933     fn make_run(run: RunConfig<'_>) {
1934         run.builder.ensure(LlvmTools { target: run.target });
1935     }
1936
1937     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1938         let target = self.target;
1939
1940         /* run only if llvm-config isn't used */
1941         if let Some(config) = builder.config.target_config.get(&target) {
1942             if let Some(ref _s) = config.llvm_config {
1943                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1944                 return None;
1945             }
1946         }
1947
1948         builder.ensure(crate::native::Llvm { target });
1949
1950         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1951         tarball.set_overlay(OverlayKind::LLVM);
1952         tarball.is_preview(true);
1953
1954         // Prepare the image directory
1955         let src_bindir = builder.llvm_out(target).join("bin");
1956         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1957         for tool in LLVM_TOOLS {
1958             let exe = src_bindir.join(exe(tool, target));
1959             tarball.add_file(&exe, &dst_bindir, 0o755);
1960         }
1961
1962         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1963         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1964         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1965         // compiler libraries.
1966         maybe_install_llvm_target(builder, target, tarball.image_dir());
1967
1968         Some(tarball.generate())
1969     }
1970 }
1971
1972 // Tarball intended for internal consumption to ease rustc/std development.
1973 //
1974 // Should not be considered stable by end users.
1975 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1976 pub struct RustDev {
1977     pub target: TargetSelection,
1978 }
1979
1980 impl Step for RustDev {
1981     type Output = Option<GeneratedTarball>;
1982     const DEFAULT: bool = true;
1983     const ONLY_HOSTS: bool = true;
1984
1985     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1986         run.alias("rust-dev")
1987     }
1988
1989     fn make_run(run: RunConfig<'_>) {
1990         run.builder.ensure(RustDev { target: run.target });
1991     }
1992
1993     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1994         let target = self.target;
1995
1996         /* run only if llvm-config isn't used */
1997         if let Some(config) = builder.config.target_config.get(&target) {
1998             if let Some(ref _s) = config.llvm_config {
1999                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2000                 return None;
2001             }
2002         }
2003
2004         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2005         tarball.set_overlay(OverlayKind::LLVM);
2006
2007         builder.ensure(crate::native::Llvm { target });
2008
2009         let src_bindir = builder.llvm_out(target).join("bin");
2010         // If updating this list, you likely want to change
2011         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2012         // will not pick up the extra file until LLVM gets bumped.
2013         for bin in &[
2014             "llvm-config",
2015             "llvm-ar",
2016             "llvm-objdump",
2017             "llvm-profdata",
2018             "llvm-bcanalyzer",
2019             "llvm-cov",
2020             "llvm-dwp",
2021             "llvm-nm",
2022             "llvm-dwarfdump",
2023         ] {
2024             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2025         }
2026
2027         // We don't build LLD on some platforms, so only add it if it exists
2028         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2029         if lld_path.exists() {
2030             tarball.add_file(lld_path, "bin", 0o755);
2031         }
2032
2033         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2034
2035         // Copy the include directory as well; needed mostly to build
2036         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2037         // just broadly useful to be able to link against the bundled LLVM.
2038         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2039
2040         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2041         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2042         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2043         // compiler libraries.
2044         let dst_libdir = tarball.image_dir().join("lib");
2045         maybe_install_llvm(builder, target, &dst_libdir);
2046         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2047         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2048
2049         Some(tarball.generate())
2050     }
2051 }
2052
2053 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2054 /// release process to avoid cloning the monorepo and building stuff.
2055 ///
2056 /// Should not be considered stable by end users.
2057 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2058 pub struct BuildManifest {
2059     pub target: TargetSelection,
2060 }
2061
2062 impl Step for BuildManifest {
2063     type Output = GeneratedTarball;
2064     const DEFAULT: bool = false;
2065     const ONLY_HOSTS: bool = true;
2066
2067     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2068         run.alias("build-manifest")
2069     }
2070
2071     fn make_run(run: RunConfig<'_>) {
2072         run.builder.ensure(BuildManifest { target: run.target });
2073     }
2074
2075     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2076         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2077
2078         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2079         tarball.add_file(&build_manifest, "bin", 0o755);
2080         tarball.generate()
2081     }
2082 }
2083
2084 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2085 ///
2086 /// Currently this is the PGO profile data.
2087 ///
2088 /// Should not be considered stable by end users.
2089 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2090 pub struct ReproducibleArtifacts {
2091     pub target: TargetSelection,
2092 }
2093
2094 impl Step for ReproducibleArtifacts {
2095     type Output = Option<GeneratedTarball>;
2096     const DEFAULT: bool = true;
2097     const ONLY_HOSTS: bool = true;
2098
2099     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2100         run.alias("reproducible-artifacts")
2101     }
2102
2103     fn make_run(run: RunConfig<'_>) {
2104         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2105     }
2106
2107     fn run(self, builder: &Builder<'_>) -> Self::Output {
2108         let mut added_anything = false;
2109         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2110         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2111             tarball.add_file(path, ".", 0o644);
2112             added_anything = true;
2113         }
2114         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2115             tarball.add_file(path, ".", 0o644);
2116             added_anything = true;
2117         }
2118         if added_anything { Some(tarball.generate()) } else { None }
2119     }
2120 }