src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         let help = "note: not all tools are available on all nightlies\nhelp: see https://forge.rust-lang.org/infra/toolstate.html for more information";
  43         panic!(
  44             "Unable to build submodule tool {} (use `missing-tools = true` to ignore this failure)\n{}",
  45             tool_name, help
  46         )
  47     }
  48 }
  49
  50 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  51     if !builder.config.extended {
  52         return false;
  53     }
  54     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  55 }
  56
  57 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  58 pub struct Docs {
  59     pub host: TargetSelection,
  60 }
  61
  62 impl Step for Docs {
  63     type Output = Option<GeneratedTarball>;
  64     const DEFAULT: bool = true;
  65
  66     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  67         let default = run.builder.config.docs;
  68         run.alias("rust-docs").default_condition(default)
  69     }
  70
  71     fn make_run(run: RunConfig<'_>) {
  72         run.builder.ensure(Docs { host: run.target });
  73     }
  74
  75     /// Builds the `rust-docs` installer component.
  76     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  77         let host = self.host;
  78         builder.default_doc(&[]);
  79
  80         let dest = "share/doc/rust/html";
  81
  82         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  83         tarball.set_product_name("Rust Documentation");
  84         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  85         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  86         Some(tarball.generate())
  87     }
  88 }
  89
  90 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  91 pub struct JsonDocs {
  92     pub host: TargetSelection,
  93 }
  94
  95 impl Step for JsonDocs {
  96     type Output = Option<GeneratedTarball>;
  97     const DEFAULT: bool = true;
  98
  99     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 100         let default = run.builder.config.docs;
 101         run.alias("rust-json-docs").default_condition(default)
 102     }
 103
 104     fn make_run(run: RunConfig<'_>) {
 105         run.builder.ensure(JsonDocs { host: run.target });
 106     }
 107
 108     /// Builds the `rust-json-docs` installer component.
 109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 110         // This prevents JSON docs from being built for "dist" or "install"
 111         // on the stable/beta channels. The JSON format is not stable yet and
 112         // should not be included in stable/beta toolchains.
 113         if !builder.build.unstable_features() {
 114             return None;
 115         }
 116
 117         let host = self.host;
 118         builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
 119
 120         let dest = "share/doc/rust/json";
 121
 122         let mut tarball = Tarball::new(builder, "rust-json-docs", &host.triple);
 123         tarball.set_product_name("Rust Documentation In JSON Format");
 124         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 125         Some(tarball.generate())
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: TargetSelection,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = Option<GeneratedTarball>;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         let builder = run.builder;
 140         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 141     }
 142
 143     fn make_run(run: RunConfig<'_>) {
 144         run.builder.ensure(RustcDocs { host: run.target });
 145     }
 146
 147     /// Builds the `rustc-docs` installer component.
 148     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 149         let host = self.host;
 150         builder.default_doc(&[]);
 151
 152         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 153         tarball.set_product_name("Rustc Documentation");
 154         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 155         Some(tarball.generate())
 156     }
 157 }
 158
 159 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 160     let mut found = Vec::with_capacity(files.len());
 161
 162     for file in files {
 163         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 164
 165         if let Some(file_path) = file_path {
 166             found.push(file_path);
 167         } else {
 168             panic!("Could not find '{}' in {:?}", file, path);
 169         }
 170     }
 171
 172     found
 173 }
 174
 175 fn make_win_dist(
 176     rust_root: &Path,
 177     plat_root: &Path,
 178     target: TargetSelection,
 179     builder: &Builder<'_>,
 180 ) {
 181     //Ask gcc where it keeps its stuff
 182     let mut cmd = Command::new(builder.cc(target));
 183     cmd.arg("-print-search-dirs");
 184     let gcc_out = output(&mut cmd);
 185
 186     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 187     let mut lib_path = Vec::new();
 188
 189     for line in gcc_out.lines() {
 190         let idx = line.find(':').unwrap();
 191         let key = &line[..idx];
 192         let trim_chars: &[_] = &[' ', '='];
 193         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 194
 195         if key == "programs" {
 196             bin_path.extend(value);
 197         } else if key == "libraries" {
 198             lib_path.extend(value);
 199         }
 200     }
 201
 202     let compiler = if target == "i686-pc-windows-gnu" {
 203         "i686-w64-mingw32-gcc.exe"
 204     } else if target == "x86_64-pc-windows-gnu" {
 205         "x86_64-w64-mingw32-gcc.exe"
 206     } else {
 207         "gcc.exe"
 208     };
 209     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 210     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 211     if target.starts_with("i686-") {
 212         rustc_dlls.push("libgcc_s_dw2-1.dll");
 213     } else {
 214         rustc_dlls.push("libgcc_s_seh-1.dll");
 215     }
 216
 217     let target_libs = [
 218         //MinGW libs
 219         "libgcc.a",
 220         "libgcc_eh.a",
 221         "libgcc_s.a",
 222         "libm.a",
 223         "libmingw32.a",
 224         "libmingwex.a",
 225         "libstdc++.a",
 226         "libiconv.a",
 227         "libmoldname.a",
 228         "libpthread.a",
 229         //Windows import libs
 230         "libadvapi32.a",
 231         "libbcrypt.a",
 232         "libcomctl32.a",
 233         "libcomdlg32.a",
 234         "libcredui.a",
 235         "libcrypt32.a",
 236         "libdbghelp.a",
 237         "libgdi32.a",
 238         "libimagehlp.a",
 239         "libiphlpapi.a",
 240         "libkernel32.a",
 241         "libmsimg32.a",
 242         "libmsvcrt.a",
 243         "libodbc32.a",
 244         "libole32.a",
 245         "liboleaut32.a",
 246         "libopengl32.a",
 247         "libpsapi.a",
 248         "librpcrt4.a",
 249         "libsecur32.a",
 250         "libsetupapi.a",
 251         "libshell32.a",
 252         "libsynchronization.a",
 253         "libuser32.a",
 254         "libuserenv.a",
 255         "libuuid.a",
 256         "libwinhttp.a",
 257         "libwinmm.a",
 258         "libwinspool.a",
 259         "libws2_32.a",
 260         "libwsock32.a",
 261     ];
 262
 263     //Find mingw artifacts we want to bundle
 264     let target_tools = find_files(&target_tools, &bin_path);
 265     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 266     let target_libs = find_files(&target_libs, &lib_path);
 267
 268     // Copy runtime dlls next to rustc.exe
 269     let dist_bin_dir = rust_root.join("bin/");
 270     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 271     for src in rustc_dlls {
 272         builder.copy_to_folder(&src, &dist_bin_dir);
 273     }
 274
 275     //Copy platform tools to platform-specific bin directory
 276     let target_bin_dir = plat_root
 277         .join("lib")
 278         .join("rustlib")
 279         .join(target.triple)
 280         .join("bin")
 281         .join("self-contained");
 282     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 283     for src in target_tools {
 284         builder.copy_to_folder(&src, &target_bin_dir);
 285     }
 286
 287     // Warn windows-gnu users that the bundled GCC cannot compile C files
 288     builder.create(
 289         &target_bin_dir.join("GCC-WARNING.txt"),
 290         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 291          used as a linker. In order to be able to compile projects containing C code use \
 292          the GCC provided by MinGW or Cygwin.",
 293     );
 294
 295     //Copy platform libs to platform-specific lib directory
 296     let target_lib_dir = plat_root
 297         .join("lib")
 298         .join("rustlib")
 299         .join(target.triple)
 300         .join("lib")
 301         .join("self-contained");
 302     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 303     for src in target_libs {
 304         builder.copy_to_folder(&src, &target_lib_dir);
 305     }
 306 }
 307
 308 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 309 pub struct Mingw {
 310     pub host: TargetSelection,
 311 }
 312
 313 impl Step for Mingw {
 314     type Output = Option<GeneratedTarball>;
 315     const DEFAULT: bool = true;
 316
 317     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 318         run.alias("rust-mingw")
 319     }
 320
 321     fn make_run(run: RunConfig<'_>) {
 322         run.builder.ensure(Mingw { host: run.target });
 323     }
 324
 325     /// Builds the `rust-mingw` installer component.
 326     ///
 327     /// This contains all the bits and pieces to run the MinGW Windows targets
 328     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 329     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 330         let host = self.host;
 331         if !host.ends_with("pc-windows-gnu") {
 332             return None;
 333         }
 334
 335         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 336         tarball.set_product_name("Rust MinGW");
 337
 338         // The first argument is a "temporary directory" which is just
 339         // thrown away (this contains the runtime DLLs included in the rustc package
 340         // above) and the second argument is where to place all the MinGW components
 341         // (which is what we want).
 342         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 343
 344         Some(tarball.generate())
 345     }
 346 }
 347
 348 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 349 pub struct Rustc {
 350     pub compiler: Compiler,
 351 }
 352
 353 impl Step for Rustc {
 354     type Output = GeneratedTarball;
 355     const DEFAULT: bool = true;
 356     const ONLY_HOSTS: bool = true;
 357
 358     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 359         run.alias("rustc")
 360     }
 361
 362     fn make_run(run: RunConfig<'_>) {
 363         run.builder
 364             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 365     }
 366
 367     /// Creates the `rustc` installer component.
 368     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 369         let compiler = self.compiler;
 370         let host = self.compiler.host;
 371
 372         let tarball = Tarball::new(builder, "rustc", &host.triple);
 373
 374         // Prepare the rustc "image", what will actually end up getting installed
 375         prepare_image(builder, compiler, tarball.image_dir());
 376
 377         // On MinGW we've got a few runtime DLL dependencies that we need to
 378         // include. The first argument to this script is where to put these DLLs
 379         // (the image we're creating), and the second argument is a junk directory
 380         // to ignore all other MinGW stuff the script creates.
 381         //
 382         // On 32-bit MinGW we're always including a DLL which needs some extra
 383         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 384         // anything requiring us to distribute a license, but it's likely the
 385         // install will *also* include the rust-mingw package, which also needs
 386         // licenses, so to be safe we just include it here in all MinGW packages.
 387         if host.ends_with("pc-windows-gnu") {
 388             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 389             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 390         }
 391
 392         return tarball.generate();
 393
 394         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 395             let host = compiler.host;
 396             let src = builder.sysroot(compiler);
 397
 398             // Copy rustc/rustdoc binaries
 399             t!(fs::create_dir_all(image.join("bin")));
 400             builder.cp_r(&src.join("bin"), &image.join("bin"));
 401
 402             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 403
 404             let ra_proc_macro_srv = builder
 405                 .ensure(tool::RustAnalyzerProcMacroSrv {
 406                     compiler: builder.compiler_for(
 407                         compiler.stage,
 408                         builder.config.build,
 409                         compiler.host,
 410                     ),
 411                     target: compiler.host,
 412                 })
 413                 .expect("rust-analyzer-proc-macro-server always builds");
 414             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 415
 416             let libdir_relative = builder.libdir_relative(compiler);
 417
 418             // Copy runtime DLLs needed by the compiler
 419             if libdir_relative.to_str() != Some("bin") {
 420                 let libdir = builder.rustc_libdir(compiler);
 421                 for entry in builder.read_dir(&libdir) {
 422                     let name = entry.file_name();
 423                     if let Some(s) = name.to_str() {
 424                         if is_dylib(s) {
 425                             // Don't use custom libdir here because ^lib/ will be resolved again
 426                             // with installer
 427                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 428                         }
 429                     }
 430                 }
 431             }
 432
 433             // Copy over the codegen backends
 434             let backends_src = builder.sysroot_codegen_backends(compiler);
 435             let backends_rel = backends_src
 436                 .strip_prefix(&src)
 437                 .unwrap()
 438                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 439                 .unwrap();
 440             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 441             let backends_dst = image.join("lib").join(&backends_rel);
 442
 443             t!(fs::create_dir_all(&backends_dst));
 444             builder.cp_r(&backends_src, &backends_dst);
 445
 446             // Copy libLLVM.so to the lib dir as well, if needed. While not
 447             // technically needed by rustc itself it's needed by lots of other
 448             // components like the llvm tools and LLD. LLD is included below and
 449             // tools/LLDB come later, so let's just throw it in the rustc
 450             // component for now.
 451             maybe_install_llvm_runtime(builder, host, image);
 452
 453             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 454             t!(fs::create_dir_all(&dst_dir));
 455
 456             // Copy over lld if it's there
 457             if builder.config.lld_enabled {
 458                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 459                 let rust_lld = exe("rust-lld", compiler.host);
 460                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 461                 // for `-Z gcc-ld=lld`
 462                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 463                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 464                 t!(fs::create_dir(&gcc_lld_dst_dir));
 465                 for name in crate::LLD_FILE_NAMES {
 466                     let exe_name = exe(name, compiler.host);
 467                     builder
 468                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 469                 }
 470             }
 471
 472             // Man pages
 473             t!(fs::create_dir_all(image.join("share/man/man1")));
 474             let man_src = builder.src.join("src/doc/man");
 475             let man_dst = image.join("share/man/man1");
 476
 477             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 478             // to hardlink, and we don't want to edit the source templates
 479             for file_entry in builder.read_dir(&man_src) {
 480                 let page_src = file_entry.path();
 481                 let page_dst = man_dst.join(file_entry.file_name());
 482                 let src_text = t!(std::fs::read_to_string(&page_src));
 483                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 484                 t!(std::fs::write(&page_dst, &new_text));
 485                 t!(fs::copy(&page_src, &page_dst));
 486             }
 487
 488             // Debugger scripts
 489             builder
 490                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 491
 492             // Misc license info
 493             let cp = |file: &str| {
 494                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 495             };
 496             cp("COPYRIGHT");
 497             cp("LICENSE-APACHE");
 498             cp("LICENSE-MIT");
 499             cp("README.md");
 500         }
 501     }
 502 }
 503
 504 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 505 pub struct DebuggerScripts {
 506     pub sysroot: Interned<PathBuf>,
 507     pub host: TargetSelection,
 508 }
 509
 510 impl Step for DebuggerScripts {
 511     type Output = ();
 512
 513     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 514         run.never()
 515     }
 516
 517     /// Copies debugger scripts for `target` into the `sysroot` specified.
 518     fn run(self, builder: &Builder<'_>) {
 519         let host = self.host;
 520         let sysroot = self.sysroot;
 521         let dst = sysroot.join("lib/rustlib/etc");
 522         t!(fs::create_dir_all(&dst));
 523         let cp_debugger_script = |file: &str| {
 524             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 525         };
 526         if host.contains("windows-msvc") {
 527             // windbg debugger scripts
 528             builder.install(
 529                 &builder.src.join("src/etc/rust-windbg.cmd"),
 530                 &sysroot.join("bin"),
 531                 0o755,
 532             );
 533
 534             cp_debugger_script("natvis/intrinsic.natvis");
 535             cp_debugger_script("natvis/liballoc.natvis");
 536             cp_debugger_script("natvis/libcore.natvis");
 537             cp_debugger_script("natvis/libstd.natvis");
 538         } else {
 539             cp_debugger_script("rust_types.py");
 540
 541             // gdb debugger scripts
 542             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 543             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 544
 545             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 546             cp_debugger_script("gdb_lookup.py");
 547             cp_debugger_script("gdb_providers.py");
 548
 549             // lldb debugger scripts
 550             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 551
 552             cp_debugger_script("lldb_lookup.py");
 553             cp_debugger_script("lldb_providers.py");
 554             cp_debugger_script("lldb_commands")
 555         }
 556     }
 557 }
 558
 559 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 560     // The only true set of target libraries came from the build triple, so
 561     // let's reduce redundant work by only producing archives from that host.
 562     if compiler.host != builder.config.build {
 563         builder.info("\tskipping, not a build host");
 564         true
 565     } else {
 566         false
 567     }
 568 }
 569
 570 /// Copy stamped files into an image's `target/lib` directory.
 571 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 572     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 573     let self_contained_dst = dst.join("self-contained");
 574     t!(fs::create_dir_all(&dst));
 575     t!(fs::create_dir_all(&self_contained_dst));
 576     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 577         if dependency_type == DependencyType::TargetSelfContained {
 578             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 579         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 580             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 581         }
 582     }
 583 }
 584
 585 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 586 pub struct Std {
 587     pub compiler: Compiler,
 588     pub target: TargetSelection,
 589 }
 590
 591 impl Step for Std {
 592     type Output = Option<GeneratedTarball>;
 593     const DEFAULT: bool = true;
 594
 595     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 596         run.alias("rust-std")
 597     }
 598
 599     fn make_run(run: RunConfig<'_>) {
 600         run.builder.ensure(Std {
 601             compiler: run.builder.compiler_for(
 602                 run.builder.top_stage,
 603                 run.builder.config.build,
 604                 run.target,
 605             ),
 606             target: run.target,
 607         });
 608     }
 609
 610     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 611         let compiler = self.compiler;
 612         let target = self.target;
 613
 614         if skip_host_target_lib(builder, compiler) {
 615             return None;
 616         }
 617
 618         builder.ensure(compile::Std::new(compiler, target));
 619
 620         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 621         tarball.include_target_in_component_name(true);
 622
 623         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 624         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 625         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 626
 627         Some(tarball.generate())
 628     }
 629 }
 630
 631 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 632 pub struct RustcDev {
 633     pub compiler: Compiler,
 634     pub target: TargetSelection,
 635 }
 636
 637 impl Step for RustcDev {
 638     type Output = Option<GeneratedTarball>;
 639     const DEFAULT: bool = true;
 640     const ONLY_HOSTS: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         run.alias("rustc-dev")
 644     }
 645
 646     fn make_run(run: RunConfig<'_>) {
 647         run.builder.ensure(RustcDev {
 648             compiler: run.builder.compiler_for(
 649                 run.builder.top_stage,
 650                 run.builder.config.build,
 651                 run.target,
 652             ),
 653             target: run.target,
 654         });
 655     }
 656
 657     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 658         let compiler = self.compiler;
 659         let target = self.target;
 660         if skip_host_target_lib(builder, compiler) {
 661             return None;
 662         }
 663
 664         builder.ensure(compile::Rustc::new(compiler, target));
 665
 666         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 667
 668         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 669         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 670         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 671
 672         let src_files = &["Cargo.lock"];
 673         // This is the reduced set of paths which will become the rustc-dev component
 674         // (essentially the compiler crates and all of their path dependencies).
 675         copy_src_dirs(
 676             builder,
 677             &builder.src,
 678             &["compiler"],
 679             &[],
 680             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 681         );
 682         for file in src_files {
 683             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 684         }
 685
 686         Some(tarball.generate())
 687     }
 688 }
 689
 690 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 691 pub struct Analysis {
 692     pub compiler: Compiler,
 693     pub target: TargetSelection,
 694 }
 695
 696 impl Step for Analysis {
 697     type Output = Option<GeneratedTarball>;
 698     const DEFAULT: bool = true;
 699
 700     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 701         let default = should_build_extended_tool(&run.builder, "analysis");
 702         run.alias("rust-analysis").default_condition(default)
 703     }
 704
 705     fn make_run(run: RunConfig<'_>) {
 706         run.builder.ensure(Analysis {
 707             // Find the actual compiler (handling the full bootstrap option) which
 708             // produced the save-analysis data because that data isn't copied
 709             // through the sysroot uplifting.
 710             compiler: run.builder.compiler_for(
 711                 run.builder.top_stage,
 712                 run.builder.config.build,
 713                 run.target,
 714             ),
 715             target: run.target,
 716         });
 717     }
 718
 719     /// Creates a tarball of save-analysis metadata, if available.
 720     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 721         let compiler = self.compiler;
 722         let target = self.target;
 723         if compiler.host != builder.config.build {
 724             return None;
 725         }
 726
 727         builder.ensure(compile::Std::new(compiler, target));
 728         let src = builder
 729             .stage_out(compiler, Mode::Std)
 730             .join(target.triple)
 731             .join(builder.cargo_dir())
 732             .join("deps")
 733             .join("save-analysis");
 734
 735         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 736         tarball.include_target_in_component_name(true);
 737         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 738         Some(tarball.generate())
 739     }
 740 }
 741
 742 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 743 /// `dst_dir`.
 744 fn copy_src_dirs(
 745     builder: &Builder<'_>,
 746     base: &Path,
 747     src_dirs: &[&str],
 748     exclude_dirs: &[&str],
 749     dst_dir: &Path,
 750 ) {
 751     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 752         let spath = match path.to_str() {
 753             Some(path) => path,
 754             None => return false,
 755         };
 756         if spath.ends_with('~') || spath.ends_with(".pyc") {
 757             return false;
 758         }
 759
 760         const LLVM_PROJECTS: &[&str] = &[
 761             "llvm-project/clang",
 762             "llvm-project\\clang",
 763             "llvm-project/libunwind",
 764             "llvm-project\\libunwind",
 765             "llvm-project/lld",
 766             "llvm-project\\lld",
 767             "llvm-project/lldb",
 768             "llvm-project\\lldb",
 769             "llvm-project/llvm",
 770             "llvm-project\\llvm",
 771             "llvm-project/compiler-rt",
 772             "llvm-project\\compiler-rt",
 773             "llvm-project/cmake",
 774             "llvm-project\\cmake",
 775         ];
 776         if spath.contains("llvm-project")
 777             && !spath.ends_with("llvm-project")
 778             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 779         {
 780             return false;
 781         }
 782
 783         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 784         if LLVM_TEST.iter().any(|path| spath.contains(path))
 785             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 786         {
 787             return false;
 788         }
 789
 790         let full_path = Path::new(dir).join(path);
 791         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 792             return false;
 793         }
 794
 795         let excludes = [
 796             "CVS",
 797             "RCS",
 798             "SCCS",
 799             ".git",
 800             ".gitignore",
 801             ".gitmodules",
 802             ".gitattributes",
 803             ".cvsignore",
 804             ".svn",
 805             ".arch-ids",
 806             "{arch}",
 807             "=RELEASE-ID",
 808             "=meta-update",
 809             "=update",
 810             ".bzr",
 811             ".bzrignore",
 812             ".bzrtags",
 813             ".hg",
 814             ".hgignore",
 815             ".hgrags",
 816             "_darcs",
 817         ];
 818         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 819     }
 820
 821     // Copy the directories using our filter
 822     for item in src_dirs {
 823         let dst = &dst_dir.join(item);
 824         t!(fs::create_dir_all(dst));
 825         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 826     }
 827 }
 828
 829 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 830 pub struct Src;
 831
 832 impl Step for Src {
 833     /// The output path of the src installer tarball
 834     type Output = GeneratedTarball;
 835     const DEFAULT: bool = true;
 836     const ONLY_HOSTS: bool = true;
 837
 838     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 839         run.alias("rust-src")
 840     }
 841
 842     fn make_run(run: RunConfig<'_>) {
 843         run.builder.ensure(Src);
 844     }
 845
 846     /// Creates the `rust-src` installer component
 847     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 848         let tarball = Tarball::new_targetless(builder, "rust-src");
 849
 850         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 851         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 852         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 853         // and fix them...
 854         //
 855         // NOTE: if you update the paths here, you also should update the "virtual" path
 856         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 857         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 858
 859         let src_files = ["Cargo.lock"];
 860         // This is the reduced set of paths which will become the rust-src component
 861         // (essentially libstd and all of its path dependencies).
 862         copy_src_dirs(
 863             builder,
 864             &builder.src,
 865             &["library", "src/llvm-project/libunwind"],
 866             &[
 867                 // not needed and contains symlinks which rustup currently
 868                 // chokes on when unpacking.
 869                 "library/backtrace/crates",
 870                 // these are 30MB combined and aren't necessary for building
 871                 // the standard library.
 872                 "library/stdarch/Cargo.toml",
 873                 "library/stdarch/crates/stdarch-verify",
 874                 "library/stdarch/crates/intrinsic-test",
 875             ],
 876             &dst_src,
 877         );
 878         for file in src_files.iter() {
 879             builder.copy(&builder.src.join(file), &dst_src.join(file));
 880         }
 881
 882         tarball.generate()
 883     }
 884 }
 885
 886 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 887 pub struct PlainSourceTarball;
 888
 889 impl Step for PlainSourceTarball {
 890     /// Produces the location of the tarball generated
 891     type Output = GeneratedTarball;
 892     const DEFAULT: bool = true;
 893     const ONLY_HOSTS: bool = true;
 894
 895     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 896         let builder = run.builder;
 897         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 898     }
 899
 900     fn make_run(run: RunConfig<'_>) {
 901         run.builder.ensure(PlainSourceTarball);
 902     }
 903
 904     /// Creates the plain source tarball
 905     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 906         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 907         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 908         // It also contains symbolic links, unlike other any other dist tarball.
 909         // It's used for distros building rustc from source in a pre-vendored environment.
 910         let mut tarball = Tarball::new(builder, "rustc", "src");
 911         tarball.permit_symlinks(true);
 912         let plain_dst_src = tarball.image_dir();
 913
 914         // This is the set of root paths which will become part of the source package
 915         let src_files = [
 916             "COPYRIGHT",
 917             "LICENSE-APACHE",
 918             "LICENSE-MIT",
 919             "CONTRIBUTING.md",
 920             "README.md",
 921             "RELEASES.md",
 922             "configure",
 923             "x.py",
 924             "config.toml.example",
 925             "Cargo.toml",
 926             "Cargo.lock",
 927         ];
 928         let src_dirs = ["src", "compiler", "library"];
 929
 930         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 931
 932         // Copy the files normally
 933         for item in &src_files {
 934             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 935         }
 936
 937         // Create the version file
 938         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 939         if let Some(sha) = builder.rust_sha() {
 940             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 941         }
 942
 943         // If we're building from git sources, we need to vendor a complete distribution.
 944         if builder.rust_info.is_git() {
 945             // Ensure we have the submodules checked out.
 946             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 947
 948             // Vendor all Cargo dependencies
 949             let mut cmd = Command::new(&builder.initial_cargo);
 950             cmd.arg("vendor")
 951                 .arg("--sync")
 952                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 953                 .arg("--sync")
 954                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 955                 .arg("--sync")
 956                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 957                 .current_dir(&plain_dst_src);
 958
 959             let config = if !builder.config.dry_run {
 960                 t!(String::from_utf8(t!(cmd.output()).stdout))
 961             } else {
 962                 String::new()
 963             };
 964
 965             let cargo_config_dir = plain_dst_src.join(".cargo");
 966             builder.create_dir(&cargo_config_dir);
 967             builder.create(&cargo_config_dir.join("config.toml"), &config);
 968         }
 969
 970         tarball.bare()
 971     }
 972 }
 973
 974 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 975 pub struct Cargo {
 976     pub compiler: Compiler,
 977     pub target: TargetSelection,
 978 }
 979
 980 impl Step for Cargo {
 981     type Output = Option<GeneratedTarball>;
 982     const DEFAULT: bool = true;
 983     const ONLY_HOSTS: bool = true;
 984
 985     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 986         let default = should_build_extended_tool(&run.builder, "cargo");
 987         run.alias("cargo").default_condition(default)
 988     }
 989
 990     fn make_run(run: RunConfig<'_>) {
 991         run.builder.ensure(Cargo {
 992             compiler: run.builder.compiler_for(
 993                 run.builder.top_stage,
 994                 run.builder.config.build,
 995                 run.target,
 996             ),
 997             target: run.target,
 998         });
 999     }
1000
1001     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1002         let compiler = self.compiler;
1003         let target = self.target;
1004
1005         let cargo = builder.ensure(tool::Cargo { compiler, target });
1006         let src = builder.src.join("src/tools/cargo");
1007         let etc = src.join("src/etc");
1008
1009         // Prepare the image directory
1010         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
1011         tarball.set_overlay(OverlayKind::Cargo);
1012
1013         tarball.add_file(&cargo, "bin", 0o755);
1014         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
1015         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1016         tarball.add_dir(etc.join("man"), "share/man/man1");
1017         tarball.add_legal_and_readme_to("share/doc/cargo");
1018
1019         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1020             let dirent = dirent.expect("read dir entry");
1021             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1022                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1023             }
1024         }
1025
1026         Some(tarball.generate())
1027     }
1028 }
1029
1030 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1031 pub struct Rls {
1032     pub compiler: Compiler,
1033     pub target: TargetSelection,
1034 }
1035
1036 impl Step for Rls {
1037     type Output = Option<GeneratedTarball>;
1038     const ONLY_HOSTS: bool = true;
1039     const DEFAULT: bool = true;
1040
1041     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1042         let default = should_build_extended_tool(&run.builder, "rls");
1043         run.alias("rls").default_condition(default)
1044     }
1045
1046     fn make_run(run: RunConfig<'_>) {
1047         run.builder.ensure(Rls {
1048             compiler: run.builder.compiler_for(
1049                 run.builder.top_stage,
1050                 run.builder.config.build,
1051                 run.target,
1052             ),
1053             target: run.target,
1054         });
1055     }
1056
1057     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1058         let compiler = self.compiler;
1059         let target = self.target;
1060
1061         let rls = builder
1062             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1063             .expect("rls expected to build");
1064
1065         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1066         tarball.set_overlay(OverlayKind::RLS);
1067         tarball.is_preview(true);
1068         tarball.add_file(rls, "bin", 0o755);
1069         tarball.add_legal_and_readme_to("share/doc/rls");
1070         Some(tarball.generate())
1071     }
1072 }
1073
1074 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1075 pub struct RustAnalyzer {
1076     pub compiler: Compiler,
1077     pub target: TargetSelection,
1078 }
1079
1080 impl Step for RustAnalyzer {
1081     type Output = Option<GeneratedTarball>;
1082     const DEFAULT: bool = true;
1083     const ONLY_HOSTS: bool = true;
1084
1085     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1086         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1087         run.alias("rust-analyzer").default_condition(default)
1088     }
1089
1090     fn make_run(run: RunConfig<'_>) {
1091         run.builder.ensure(RustAnalyzer {
1092             compiler: run.builder.compiler_for(
1093                 run.builder.top_stage,
1094                 run.builder.config.build,
1095                 run.target,
1096             ),
1097             target: run.target,
1098         });
1099     }
1100
1101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1102         let compiler = self.compiler;
1103         let target = self.target;
1104
1105         if target.contains("riscv64") {
1106             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1107             // to build. See #74813 for details.
1108             return None;
1109         }
1110
1111         let rust_analyzer = builder
1112             .ensure(tool::RustAnalyzer { compiler, target })
1113             .expect("rust-analyzer always builds");
1114
1115         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1116         tarball.set_overlay(OverlayKind::RustAnalyzer);
1117         tarball.is_preview(true);
1118         tarball.add_file(rust_analyzer, "bin", 0o755);
1119         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1120         Some(tarball.generate())
1121     }
1122 }
1123
1124 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1125 pub struct Clippy {
1126     pub compiler: Compiler,
1127     pub target: TargetSelection,
1128 }
1129
1130 impl Step for Clippy {
1131     type Output = Option<GeneratedTarball>;
1132     const DEFAULT: bool = true;
1133     const ONLY_HOSTS: bool = true;
1134
1135     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1136         let default = should_build_extended_tool(&run.builder, "clippy");
1137         run.alias("clippy").default_condition(default)
1138     }
1139
1140     fn make_run(run: RunConfig<'_>) {
1141         run.builder.ensure(Clippy {
1142             compiler: run.builder.compiler_for(
1143                 run.builder.top_stage,
1144                 run.builder.config.build,
1145                 run.target,
1146             ),
1147             target: run.target,
1148         });
1149     }
1150
1151     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1152         let compiler = self.compiler;
1153         let target = self.target;
1154
1155         // Prepare the image directory
1156         // We expect clippy to build, because we've exited this step above if tool
1157         // state for clippy isn't testing.
1158         let clippy = builder
1159             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1160             .expect("clippy expected to build - essential tool");
1161         let cargoclippy = builder
1162             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1163             .expect("clippy expected to build - essential tool");
1164
1165         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1166         tarball.set_overlay(OverlayKind::Clippy);
1167         tarball.is_preview(true);
1168         tarball.add_file(clippy, "bin", 0o755);
1169         tarball.add_file(cargoclippy, "bin", 0o755);
1170         tarball.add_legal_and_readme_to("share/doc/clippy");
1171         Some(tarball.generate())
1172     }
1173 }
1174
1175 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1176 pub struct Miri {
1177     pub compiler: Compiler,
1178     pub target: TargetSelection,
1179 }
1180
1181 impl Step for Miri {
1182     type Output = Option<GeneratedTarball>;
1183     const DEFAULT: bool = true;
1184     const ONLY_HOSTS: bool = true;
1185
1186     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1187         let default = should_build_extended_tool(&run.builder, "miri");
1188         run.alias("miri").default_condition(default)
1189     }
1190
1191     fn make_run(run: RunConfig<'_>) {
1192         run.builder.ensure(Miri {
1193             compiler: run.builder.compiler_for(
1194                 run.builder.top_stage,
1195                 run.builder.config.build,
1196                 run.target,
1197             ),
1198             target: run.target,
1199         });
1200     }
1201
1202     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1203         // This prevents miri from being built for "dist" or "install"
1204         // on the stable/beta channels. It is a nightly-only tool and should
1205         // not be included.
1206         if !builder.build.unstable_features() {
1207             return None;
1208         }
1209         let compiler = self.compiler;
1210         let target = self.target;
1211
1212         let miri = builder
1213             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1214             .or_else(|| {
1215                 missing_tool("miri", builder.build.config.missing_tools);
1216                 None
1217             })?;
1218         let cargomiri = builder
1219             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1220             .or_else(|| {
1221                 missing_tool("cargo miri", builder.build.config.missing_tools);
1222                 None
1223             })?;
1224
1225         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1226         tarball.set_overlay(OverlayKind::Miri);
1227         tarball.is_preview(true);
1228         tarball.add_file(miri, "bin", 0o755);
1229         tarball.add_file(cargomiri, "bin", 0o755);
1230         tarball.add_legal_and_readme_to("share/doc/miri");
1231         Some(tarball.generate())
1232     }
1233 }
1234
1235 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1236 pub struct Rustfmt {
1237     pub compiler: Compiler,
1238     pub target: TargetSelection,
1239 }
1240
1241 impl Step for Rustfmt {
1242     type Output = Option<GeneratedTarball>;
1243     const DEFAULT: bool = true;
1244     const ONLY_HOSTS: bool = true;
1245
1246     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1247         let default = should_build_extended_tool(&run.builder, "rustfmt");
1248         run.alias("rustfmt").default_condition(default)
1249     }
1250
1251     fn make_run(run: RunConfig<'_>) {
1252         run.builder.ensure(Rustfmt {
1253             compiler: run.builder.compiler_for(
1254                 run.builder.top_stage,
1255                 run.builder.config.build,
1256                 run.target,
1257             ),
1258             target: run.target,
1259         });
1260     }
1261
1262     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1263         let compiler = self.compiler;
1264         let target = self.target;
1265
1266         let rustfmt = builder
1267             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1268             .expect("rustfmt expected to build - essential tool");
1269         let cargofmt = builder
1270             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1271             .expect("cargo fmt expected to build - essential tool");
1272         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1273         tarball.set_overlay(OverlayKind::Rustfmt);
1274         tarball.is_preview(true);
1275         tarball.add_file(rustfmt, "bin", 0o755);
1276         tarball.add_file(cargofmt, "bin", 0o755);
1277         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1278         Some(tarball.generate())
1279     }
1280 }
1281
1282 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1283 pub struct RustDemangler {
1284     pub compiler: Compiler,
1285     pub target: TargetSelection,
1286 }
1287
1288 impl Step for RustDemangler {
1289     type Output = Option<GeneratedTarball>;
1290     const DEFAULT: bool = true;
1291     const ONLY_HOSTS: bool = true;
1292
1293     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1294         // While other tools use `should_build_extended_tool` to decide whether to be run by
1295         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1296         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1297         // we run the step by default when only `extended = true`, and decide whether to actually
1298         // run it or not later.
1299         let default = run.builder.config.extended;
1300         run.alias("rust-demangler").default_condition(default)
1301     }
1302
1303     fn make_run(run: RunConfig<'_>) {
1304         run.builder.ensure(RustDemangler {
1305             compiler: run.builder.compiler_for(
1306                 run.builder.top_stage,
1307                 run.builder.config.build,
1308                 run.target,
1309             ),
1310             target: run.target,
1311         });
1312     }
1313
1314     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1315         let compiler = self.compiler;
1316         let target = self.target;
1317
1318         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1319         let condition = should_build_extended_tool(builder, "rust-demangler")
1320             || builder.config.profiler_enabled(target);
1321         if builder.config.extended && !condition {
1322             return None;
1323         }
1324
1325         let rust_demangler = builder
1326             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1327             .expect("rust-demangler expected to build - in-tree tool");
1328
1329         // Prepare the image directory
1330         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1331         tarball.set_overlay(OverlayKind::RustDemangler);
1332         tarball.is_preview(true);
1333         tarball.add_file(&rust_demangler, "bin", 0o755);
1334         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1335         Some(tarball.generate())
1336     }
1337 }
1338
1339 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1340 pub struct Extended {
1341     stage: u32,
1342     host: TargetSelection,
1343     target: TargetSelection,
1344 }
1345
1346 impl Step for Extended {
1347     type Output = ();
1348     const DEFAULT: bool = true;
1349     const ONLY_HOSTS: bool = true;
1350
1351     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1352         let builder = run.builder;
1353         run.alias("extended").default_condition(builder.config.extended)
1354     }
1355
1356     fn make_run(run: RunConfig<'_>) {
1357         run.builder.ensure(Extended {
1358             stage: run.builder.top_stage,
1359             host: run.builder.config.build,
1360             target: run.target,
1361         });
1362     }
1363
1364     /// Creates a combined installer for the specified target in the provided stage.
1365     fn run(self, builder: &Builder<'_>) {
1366         let target = self.target;
1367         let stage = self.stage;
1368         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1369
1370         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1371
1372         let mut tarballs = Vec::new();
1373         let mut built_tools = HashSet::new();
1374         macro_rules! add_component {
1375             ($name:expr => $step:expr) => {
1376                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1377                     tarballs.push(tarball);
1378                     built_tools.insert($name);
1379                 }
1380             };
1381         }
1382
1383         // When rust-std package split from rustc, we needed to ensure that during
1384         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1385         // the std files during uninstall. To do this ensure that rustc comes
1386         // before rust-std in the list below.
1387         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1388         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1389
1390         if target.ends_with("windows-gnu") {
1391             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1392         }
1393
1394         add_component!("rust-docs" => Docs { host: target });
1395         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1396         add_component!("cargo" => Cargo { compiler, target });
1397         add_component!("rustfmt" => Rustfmt { compiler, target });
1398         add_component!("rls" => Rls { compiler, target });
1399         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1400         add_component!("llvm-components" => LlvmTools { target });
1401         add_component!("clippy" => Clippy { compiler, target });
1402         add_component!("miri" => Miri { compiler, target });
1403         add_component!("analysis" => Analysis { compiler, target });
1404
1405         let etc = builder.src.join("src/etc/installer");
1406
1407         // Avoid producing tarballs during a dry run.
1408         if builder.config.dry_run {
1409             return;
1410         }
1411
1412         let tarball = Tarball::new(builder, "rust", &target.triple);
1413         let generated = tarball.combine(&tarballs);
1414
1415         let tmp = tmpdir(builder).join("combined-tarball");
1416         let work = generated.work_dir();
1417
1418         let mut license = String::new();
1419         license += &builder.read(&builder.src.join("COPYRIGHT"));
1420         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1421         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1422         license.push('\n');
1423         license.push('\n');
1424
1425         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1426         let mut rtf = rtf.to_string();
1427         rtf.push('\n');
1428         for line in license.lines() {
1429             rtf.push_str(line);
1430             rtf.push_str("\\line ");
1431         }
1432         rtf.push('}');
1433
1434         fn filter(contents: &str, marker: &str) -> String {
1435             let start = format!("tool-{}-start", marker);
1436             let end = format!("tool-{}-end", marker);
1437             let mut lines = Vec::new();
1438             let mut omitted = false;
1439             for line in contents.lines() {
1440                 if line.contains(&start) {
1441                     omitted = true;
1442                 } else if line.contains(&end) {
1443                     omitted = false;
1444                 } else if !omitted {
1445                     lines.push(line);
1446                 }
1447             }
1448
1449             lines.join("\n")
1450         }
1451
1452         let xform = |p: &Path| {
1453             let mut contents = t!(fs::read_to_string(p));
1454             for tool in &["rust-demangler", "rust-analyzer", "miri", "rustfmt"] {
1455                 if !built_tools.contains(tool) {
1456                     contents = filter(&contents, tool);
1457                 }
1458             }
1459             let ret = tmp.join(p.file_name().unwrap());
1460             t!(fs::write(&ret, &contents));
1461             ret
1462         };
1463
1464         if target.contains("apple-darwin") {
1465             builder.info("building pkg installer");
1466             let pkg = tmp.join("pkg");
1467             let _ = fs::remove_dir_all(&pkg);
1468
1469             let pkgbuild = |component: &str| {
1470                 let mut cmd = Command::new("pkgbuild");
1471                 cmd.arg("--identifier")
1472                     .arg(format!("org.rust-lang.{}", component))
1473                     .arg("--scripts")
1474                     .arg(pkg.join(component))
1475                     .arg("--nopayload")
1476                     .arg(pkg.join(component).with_extension("pkg"));
1477                 builder.run(&mut cmd);
1478             };
1479
1480             let prepare = |name: &str| {
1481                 builder.create_dir(&pkg.join(name));
1482                 builder.cp_r(
1483                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1484                     &pkg.join(name),
1485                 );
1486                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1487                 pkgbuild(name);
1488             };
1489             prepare("rustc");
1490             prepare("cargo");
1491             prepare("rust-std");
1492             prepare("rust-analysis");
1493             prepare("clippy");
1494             for tool in &["rust-docs", "rust-demangler", "rust-analyzer", "miri"] {
1495                 if built_tools.contains(tool) {
1496                     prepare(tool);
1497                 }
1498             }
1499             // create an 'uninstall' package
1500             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1501             pkgbuild("uninstall");
1502
1503             builder.create_dir(&pkg.join("res"));
1504             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1505             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1506             let mut cmd = Command::new("productbuild");
1507             cmd.arg("--distribution")
1508                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1509                 .arg("--resources")
1510                 .arg(pkg.join("res"))
1511                 .arg(distdir(builder).join(format!(
1512                     "{}-{}.pkg",
1513                     pkgname(builder, "rust"),
1514                     target.triple
1515                 )))
1516                 .arg("--package-path")
1517                 .arg(&pkg);
1518             let _time = timeit(builder);
1519             builder.run(&mut cmd);
1520         }
1521
1522         if target.contains("windows") {
1523             let exe = tmp.join("exe");
1524             let _ = fs::remove_dir_all(&exe);
1525
1526             let prepare = |name: &str| {
1527                 builder.create_dir(&exe.join(name));
1528                 let dir = if name == "rust-std" || name == "rust-analysis" {
1529                     format!("{}-{}", name, target.triple)
1530                 } else if name == "rust-analyzer" {
1531                     "rust-analyzer-preview".to_string()
1532                 } else if name == "clippy" {
1533                     "clippy-preview".to_string()
1534                 } else if name == "rust-demangler" {
1535                     "rust-demangler-preview".to_string()
1536                 } else if name == "miri" {
1537                     "miri-preview".to_string()
1538                 } else {
1539                     name.to_string()
1540                 };
1541                 builder.cp_r(
1542                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1543                     &exe.join(name),
1544                 );
1545                 builder.remove(&exe.join(name).join("manifest.in"));
1546             };
1547             prepare("rustc");
1548             prepare("cargo");
1549             prepare("rust-analysis");
1550             prepare("rust-docs");
1551             prepare("rust-std");
1552             prepare("clippy");
1553             for tool in &["rust-demangler", "rust-analyzer", "miri"] {
1554                 if built_tools.contains(tool) {
1555                     prepare(tool);
1556                 }
1557             }
1558             if target.ends_with("windows-gnu") {
1559                 prepare("rust-mingw");
1560             }
1561
1562             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1563
1564             // Generate msi installer
1565             let wix_path = env::var_os("WIX")
1566                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1567             let wix = PathBuf::from(wix_path);
1568             let heat = wix.join("bin/heat.exe");
1569             let candle = wix.join("bin/candle.exe");
1570             let light = wix.join("bin/light.exe");
1571
1572             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1573             builder.run(
1574                 Command::new(&heat)
1575                     .current_dir(&exe)
1576                     .arg("dir")
1577                     .arg("rustc")
1578                     .args(&heat_flags)
1579                     .arg("-cg")
1580                     .arg("RustcGroup")
1581                     .arg("-dr")
1582                     .arg("Rustc")
1583                     .arg("-var")
1584                     .arg("var.RustcDir")
1585                     .arg("-out")
1586                     .arg(exe.join("RustcGroup.wxs")),
1587             );
1588             builder.run(
1589                 Command::new(&heat)
1590                     .current_dir(&exe)
1591                     .arg("dir")
1592                     .arg("rust-docs")
1593                     .args(&heat_flags)
1594                     .arg("-cg")
1595                     .arg("DocsGroup")
1596                     .arg("-dr")
1597                     .arg("Docs")
1598                     .arg("-var")
1599                     .arg("var.DocsDir")
1600                     .arg("-out")
1601                     .arg(exe.join("DocsGroup.wxs"))
1602                     .arg("-t")
1603                     .arg(etc.join("msi/squash-components.xsl")),
1604             );
1605             builder.run(
1606                 Command::new(&heat)
1607                     .current_dir(&exe)
1608                     .arg("dir")
1609                     .arg("cargo")
1610                     .args(&heat_flags)
1611                     .arg("-cg")
1612                     .arg("CargoGroup")
1613                     .arg("-dr")
1614                     .arg("Cargo")
1615                     .arg("-var")
1616                     .arg("var.CargoDir")
1617                     .arg("-out")
1618                     .arg(exe.join("CargoGroup.wxs"))
1619                     .arg("-t")
1620                     .arg(etc.join("msi/remove-duplicates.xsl")),
1621             );
1622             builder.run(
1623                 Command::new(&heat)
1624                     .current_dir(&exe)
1625                     .arg("dir")
1626                     .arg("rust-std")
1627                     .args(&heat_flags)
1628                     .arg("-cg")
1629                     .arg("StdGroup")
1630                     .arg("-dr")
1631                     .arg("Std")
1632                     .arg("-var")
1633                     .arg("var.StdDir")
1634                     .arg("-out")
1635                     .arg(exe.join("StdGroup.wxs")),
1636             );
1637             if built_tools.contains("rust-analyzer") {
1638                 builder.run(
1639                     Command::new(&heat)
1640                         .current_dir(&exe)
1641                         .arg("dir")
1642                         .arg("rust-analyzer")
1643                         .args(&heat_flags)
1644                         .arg("-cg")
1645                         .arg("RustAnalyzerGroup")
1646                         .arg("-dr")
1647                         .arg("RustAnalyzer")
1648                         .arg("-var")
1649                         .arg("var.RustAnalyzerDir")
1650                         .arg("-out")
1651                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1652                         .arg("-t")
1653                         .arg(etc.join("msi/remove-duplicates.xsl")),
1654                 );
1655             }
1656             builder.run(
1657                 Command::new(&heat)
1658                     .current_dir(&exe)
1659                     .arg("dir")
1660                     .arg("clippy")
1661                     .args(&heat_flags)
1662                     .arg("-cg")
1663                     .arg("ClippyGroup")
1664                     .arg("-dr")
1665                     .arg("Clippy")
1666                     .arg("-var")
1667                     .arg("var.ClippyDir")
1668                     .arg("-out")
1669                     .arg(exe.join("ClippyGroup.wxs"))
1670                     .arg("-t")
1671                     .arg(etc.join("msi/remove-duplicates.xsl")),
1672             );
1673             if built_tools.contains("rust-demangler") {
1674                 builder.run(
1675                     Command::new(&heat)
1676                         .current_dir(&exe)
1677                         .arg("dir")
1678                         .arg("rust-demangler")
1679                         .args(&heat_flags)
1680                         .arg("-cg")
1681                         .arg("RustDemanglerGroup")
1682                         .arg("-dr")
1683                         .arg("RustDemangler")
1684                         .arg("-var")
1685                         .arg("var.RustDemanglerDir")
1686                         .arg("-out")
1687                         .arg(exe.join("RustDemanglerGroup.wxs"))
1688                         .arg("-t")
1689                         .arg(etc.join("msi/remove-duplicates.xsl")),
1690                 );
1691             }
1692             if built_tools.contains("miri") {
1693                 builder.run(
1694                     Command::new(&heat)
1695                         .current_dir(&exe)
1696                         .arg("dir")
1697                         .arg("miri")
1698                         .args(&heat_flags)
1699                         .arg("-cg")
1700                         .arg("MiriGroup")
1701                         .arg("-dr")
1702                         .arg("Miri")
1703                         .arg("-var")
1704                         .arg("var.MiriDir")
1705                         .arg("-out")
1706                         .arg(exe.join("MiriGroup.wxs"))
1707                         .arg("-t")
1708                         .arg(etc.join("msi/remove-duplicates.xsl")),
1709                 );
1710             }
1711             builder.run(
1712                 Command::new(&heat)
1713                     .current_dir(&exe)
1714                     .arg("dir")
1715                     .arg("rust-analysis")
1716                     .args(&heat_flags)
1717                     .arg("-cg")
1718                     .arg("AnalysisGroup")
1719                     .arg("-dr")
1720                     .arg("Analysis")
1721                     .arg("-var")
1722                     .arg("var.AnalysisDir")
1723                     .arg("-out")
1724                     .arg(exe.join("AnalysisGroup.wxs"))
1725                     .arg("-t")
1726                     .arg(etc.join("msi/remove-duplicates.xsl")),
1727             );
1728             if target.ends_with("windows-gnu") {
1729                 builder.run(
1730                     Command::new(&heat)
1731                         .current_dir(&exe)
1732                         .arg("dir")
1733                         .arg("rust-mingw")
1734                         .args(&heat_flags)
1735                         .arg("-cg")
1736                         .arg("GccGroup")
1737                         .arg("-dr")
1738                         .arg("Gcc")
1739                         .arg("-var")
1740                         .arg("var.GccDir")
1741                         .arg("-out")
1742                         .arg(exe.join("GccGroup.wxs")),
1743                 );
1744             }
1745
1746             let candle = |input: &Path| {
1747                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1748                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1749                 let mut cmd = Command::new(&candle);
1750                 cmd.current_dir(&exe)
1751                     .arg("-nologo")
1752                     .arg("-dRustcDir=rustc")
1753                     .arg("-dDocsDir=rust-docs")
1754                     .arg("-dCargoDir=cargo")
1755                     .arg("-dStdDir=rust-std")
1756                     .arg("-dAnalysisDir=rust-analysis")
1757                     .arg("-dClippyDir=clippy")
1758                     .arg("-arch")
1759                     .arg(&arch)
1760                     .arg("-out")
1761                     .arg(&output)
1762                     .arg(&input);
1763                 add_env(builder, &mut cmd, target);
1764
1765                 if built_tools.contains("rust-demangler") {
1766                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1767                 }
1768                 if built_tools.contains("rust-analyzer") {
1769                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1770                 }
1771                 if built_tools.contains("miri") {
1772                     cmd.arg("-dMiriDir=miri");
1773                 }
1774                 if target.ends_with("windows-gnu") {
1775                     cmd.arg("-dGccDir=rust-mingw");
1776                 }
1777                 builder.run(&mut cmd);
1778             };
1779             candle(&xform(&etc.join("msi/rust.wxs")));
1780             candle(&etc.join("msi/ui.wxs"));
1781             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1782             candle("RustcGroup.wxs".as_ref());
1783             candle("DocsGroup.wxs".as_ref());
1784             candle("CargoGroup.wxs".as_ref());
1785             candle("StdGroup.wxs".as_ref());
1786             candle("ClippyGroup.wxs".as_ref());
1787             if built_tools.contains("rust-demangler") {
1788                 candle("RustDemanglerGroup.wxs".as_ref());
1789             }
1790             if built_tools.contains("rust-analyzer") {
1791                 candle("RustAnalyzerGroup.wxs".as_ref());
1792             }
1793             if built_tools.contains("miri") {
1794                 candle("MiriGroup.wxs".as_ref());
1795             }
1796             candle("AnalysisGroup.wxs".as_ref());
1797
1798             if target.ends_with("windows-gnu") {
1799                 candle("GccGroup.wxs".as_ref());
1800             }
1801
1802             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1803             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1804             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1805
1806             builder.info(&format!("building `msi` installer with {:?}", light));
1807             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1808             let mut cmd = Command::new(&light);
1809             cmd.arg("-nologo")
1810                 .arg("-ext")
1811                 .arg("WixUIExtension")
1812                 .arg("-ext")
1813                 .arg("WixUtilExtension")
1814                 .arg("-out")
1815                 .arg(exe.join(&filename))
1816                 .arg("rust.wixobj")
1817                 .arg("ui.wixobj")
1818                 .arg("rustwelcomedlg.wixobj")
1819                 .arg("RustcGroup.wixobj")
1820                 .arg("DocsGroup.wixobj")
1821                 .arg("CargoGroup.wixobj")
1822                 .arg("StdGroup.wixobj")
1823                 .arg("AnalysisGroup.wixobj")
1824                 .arg("ClippyGroup.wixobj")
1825                 .current_dir(&exe);
1826
1827             if built_tools.contains("rust-analyzer") {
1828                 cmd.arg("RustAnalyzerGroup.wixobj");
1829             }
1830             if built_tools.contains("rust-demangler") {
1831                 cmd.arg("RustDemanglerGroup.wixobj");
1832             }
1833             if built_tools.contains("miri") {
1834                 cmd.arg("MiriGroup.wixobj");
1835             }
1836
1837             if target.ends_with("windows-gnu") {
1838                 cmd.arg("GccGroup.wixobj");
1839             }
1840             // ICE57 wrongly complains about the shortcuts
1841             cmd.arg("-sice:ICE57");
1842
1843             let _time = timeit(builder);
1844             builder.run(&mut cmd);
1845
1846             if !builder.config.dry_run {
1847                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1848             }
1849         }
1850     }
1851 }
1852
1853 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1854     let mut parts = builder.version.split('.');
1855     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1856         .env("CFG_RELEASE_NUM", &builder.version)
1857         .env("CFG_RELEASE", builder.rust_release())
1858         .env("CFG_VER_MAJOR", parts.next().unwrap())
1859         .env("CFG_VER_MINOR", parts.next().unwrap())
1860         .env("CFG_VER_PATCH", parts.next().unwrap())
1861         .env("CFG_VER_BUILD", "0") // just needed to build
1862         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1863         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1864         .env("CFG_BUILD", target.triple)
1865         .env("CFG_CHANNEL", &builder.config.channel);
1866
1867     if target.contains("windows-gnullvm") {
1868         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1869     } else if target.contains("windows-gnu") {
1870         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1871     } else {
1872         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1873     }
1874 }
1875
1876 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1877 ///
1878 /// Returns whether the files were actually copied.
1879 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1880     if let Some(config) = builder.config.target_config.get(&target) {
1881         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1882             // If the LLVM was externally provided, then we don't currently copy
1883             // artifacts into the sysroot. This is not necessarily the right
1884             // choice (in particular, it will require the LLVM dylib to be in
1885             // the linker's load path at runtime), but the common use case for
1886             // external LLVMs is distribution provided LLVMs, and in that case
1887             // they're usually in the standard search path (e.g., /usr/lib) and
1888             // copying them here is going to cause problems as we may end up
1889             // with the wrong files and isn't what distributions want.
1890             //
1891             // This behavior may be revisited in the future though.
1892             //
1893             // If the LLVM is coming from ourselves (just from CI) though, we
1894             // still want to install it, as it otherwise won't be available.
1895             return false;
1896         }
1897     }
1898
1899     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1900     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1901     // clear why this is the case, though. llvm-config will emit the versioned
1902     // paths and we don't want those in the sysroot (as we're expecting
1903     // unversioned paths).
1904     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1905         let src_libdir = builder.llvm_out(target).join("lib");
1906         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1907         if llvm_dylib_path.exists() {
1908             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1909         }
1910         !builder.config.dry_run
1911     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1912         let mut cmd = Command::new(llvm_config);
1913         cmd.arg("--libfiles");
1914         builder.verbose(&format!("running {:?}", cmd));
1915         let files = output(&mut cmd);
1916         let build_llvm_out = &builder.llvm_out(builder.config.build);
1917         let target_llvm_out = &builder.llvm_out(target);
1918         for file in files.trim_end().split(' ') {
1919             // If we're not using a custom LLVM, make sure we package for the target.
1920             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1921                 target_llvm_out.join(relative_path)
1922             } else {
1923                 PathBuf::from(file)
1924             };
1925             builder.install(&file, dst_libdir, 0o644);
1926         }
1927         !builder.config.dry_run
1928     } else {
1929         false
1930     }
1931 }
1932
1933 /// Maybe add libLLVM.so to the target lib-dir for linking.
1934 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1935     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1936     // We do not need to copy LLVM files into the sysroot if it is not
1937     // dynamically linked; it is already included into librustc_llvm
1938     // statically.
1939     if builder.llvm_link_shared() {
1940         maybe_install_llvm(builder, target, &dst_libdir);
1941     }
1942 }
1943
1944 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1945 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1946     let dst_libdir =
1947         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1948     // We do not need to copy LLVM files into the sysroot if it is not
1949     // dynamically linked; it is already included into librustc_llvm
1950     // statically.
1951     if builder.llvm_link_shared() {
1952         maybe_install_llvm(builder, target, &dst_libdir);
1953     }
1954 }
1955
1956 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1957 pub struct LlvmTools {
1958     pub target: TargetSelection,
1959 }
1960
1961 impl Step for LlvmTools {
1962     type Output = Option<GeneratedTarball>;
1963     const ONLY_HOSTS: bool = true;
1964     const DEFAULT: bool = true;
1965
1966     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1967         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1968         // FIXME: allow using the names of the tools themselves?
1969         run.alias("llvm-tools").default_condition(default)
1970     }
1971
1972     fn make_run(run: RunConfig<'_>) {
1973         run.builder.ensure(LlvmTools { target: run.target });
1974     }
1975
1976     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1977         let target = self.target;
1978
1979         /* run only if llvm-config isn't used */
1980         if let Some(config) = builder.config.target_config.get(&target) {
1981             if let Some(ref _s) = config.llvm_config {
1982                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1983                 return None;
1984             }
1985         }
1986
1987         builder.ensure(crate::native::Llvm { target });
1988
1989         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1990         tarball.set_overlay(OverlayKind::LLVM);
1991         tarball.is_preview(true);
1992
1993         // Prepare the image directory
1994         let src_bindir = builder.llvm_out(target).join("bin");
1995         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1996         for tool in LLVM_TOOLS {
1997             let exe = src_bindir.join(exe(tool, target));
1998             tarball.add_file(&exe, &dst_bindir, 0o755);
1999         }
2000
2001         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2002         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2003         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2004         // compiler libraries.
2005         maybe_install_llvm_target(builder, target, tarball.image_dir());
2006
2007         Some(tarball.generate())
2008     }
2009 }
2010
2011 // Tarball intended for internal consumption to ease rustc/std development.
2012 //
2013 // Should not be considered stable by end users.
2014 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2015 pub struct RustDev {
2016     pub target: TargetSelection,
2017 }
2018
2019 impl Step for RustDev {
2020     type Output = Option<GeneratedTarball>;
2021     const DEFAULT: bool = true;
2022     const ONLY_HOSTS: bool = true;
2023
2024     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2025         run.alias("rust-dev")
2026     }
2027
2028     fn make_run(run: RunConfig<'_>) {
2029         run.builder.ensure(RustDev { target: run.target });
2030     }
2031
2032     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2033         let target = self.target;
2034
2035         /* run only if llvm-config isn't used */
2036         if let Some(config) = builder.config.target_config.get(&target) {
2037             if let Some(ref _s) = config.llvm_config {
2038                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2039                 return None;
2040             }
2041         }
2042
2043         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2044         tarball.set_overlay(OverlayKind::LLVM);
2045
2046         builder.ensure(crate::native::Llvm { target });
2047
2048         let src_bindir = builder.llvm_out(target).join("bin");
2049         // If updating this list, you likely want to change
2050         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2051         // will not pick up the extra file until LLVM gets bumped.
2052         for bin in &[
2053             "llvm-config",
2054             "llvm-ar",
2055             "llvm-objdump",
2056             "llvm-profdata",
2057             "llvm-bcanalyzer",
2058             "llvm-cov",
2059             "llvm-dwp",
2060             "llvm-nm",
2061             "llvm-dwarfdump",
2062             "llvm-dis",
2063         ] {
2064             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2065         }
2066
2067         // We don't build LLD on some platforms, so only add it if it exists
2068         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2069         if lld_path.exists() {
2070             tarball.add_file(lld_path, "bin", 0o755);
2071         }
2072
2073         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2074
2075         // Copy the include directory as well; needed mostly to build
2076         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2077         // just broadly useful to be able to link against the bundled LLVM.
2078         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2079
2080         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2081         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2082         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2083         // compiler libraries.
2084         let dst_libdir = tarball.image_dir().join("lib");
2085         maybe_install_llvm(builder, target, &dst_libdir);
2086         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2087         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2088
2089         Some(tarball.generate())
2090     }
2091 }
2092
2093 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2094 /// release process to avoid cloning the monorepo and building stuff.
2095 ///
2096 /// Should not be considered stable by end users.
2097 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2098 pub struct BuildManifest {
2099     pub target: TargetSelection,
2100 }
2101
2102 impl Step for BuildManifest {
2103     type Output = GeneratedTarball;
2104     const DEFAULT: bool = false;
2105     const ONLY_HOSTS: bool = true;
2106
2107     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2108         run.alias("build-manifest")
2109     }
2110
2111     fn make_run(run: RunConfig<'_>) {
2112         run.builder.ensure(BuildManifest { target: run.target });
2113     }
2114
2115     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2116         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2117
2118         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2119         tarball.add_file(&build_manifest, "bin", 0o755);
2120         tarball.generate()
2121     }
2122 }
2123
2124 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2125 ///
2126 /// Currently this is the PGO profile data.
2127 ///
2128 /// Should not be considered stable by end users.
2129 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2130 pub struct ReproducibleArtifacts {
2131     pub target: TargetSelection,
2132 }
2133
2134 impl Step for ReproducibleArtifacts {
2135     type Output = Option<GeneratedTarball>;
2136     const DEFAULT: bool = true;
2137     const ONLY_HOSTS: bool = true;
2138
2139     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2140         run.alias("reproducible-artifacts")
2141     }
2142
2143     fn make_run(run: RunConfig<'_>) {
2144         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2145     }
2146
2147     fn run(self, builder: &Builder<'_>) -> Self::Output {
2148         let mut added_anything = false;
2149         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2150         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2151             tarball.add_file(path, ".", 0o644);
2152             added_anything = true;
2153         }
2154         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2155             tarball.add_file(path, ".", 0o644);
2156             added_anything = true;
2157         }
2158         if added_anything { Some(tarball.generate()) } else { None }
2159     }
2160 }