src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("rust-docs").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         let builder = run.builder;
 100         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
 101     }
 102
 103     fn make_run(run: RunConfig<'_>) {
 104         run.builder.ensure(RustcDocs { host: run.target });
 105     }
 106
 107     /// Builds the `rustc-docs` installer component.
 108     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 109         let host = self.host;
 110         builder.default_doc(&[]);
 111
 112         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 113         tarball.set_product_name("Rustc Documentation");
 114         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 115         Some(tarball.generate())
 116     }
 117 }
 118
 119 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 120     let mut found = Vec::with_capacity(files.len());
 121
 122     for file in files {
 123         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 124
 125         if let Some(file_path) = file_path {
 126             found.push(file_path);
 127         } else {
 128             panic!("Could not find '{}' in {:?}", file, path);
 129         }
 130     }
 131
 132     found
 133 }
 134
 135 fn make_win_dist(
 136     rust_root: &Path,
 137     plat_root: &Path,
 138     target: TargetSelection,
 139     builder: &Builder<'_>,
 140 ) {
 141     //Ask gcc where it keeps its stuff
 142     let mut cmd = Command::new(builder.cc(target));
 143     cmd.arg("-print-search-dirs");
 144     let gcc_out = output(&mut cmd);
 145
 146     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 147     let mut lib_path = Vec::new();
 148
 149     for line in gcc_out.lines() {
 150         let idx = line.find(':').unwrap();
 151         let key = &line[..idx];
 152         let trim_chars: &[_] = &[' ', '='];
 153         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 154
 155         if key == "programs" {
 156             bin_path.extend(value);
 157         } else if key == "libraries" {
 158             lib_path.extend(value);
 159         }
 160     }
 161
 162     let compiler = if target == "i686-pc-windows-gnu" {
 163         "i686-w64-mingw32-gcc.exe"
 164     } else if target == "x86_64-pc-windows-gnu" {
 165         "x86_64-w64-mingw32-gcc.exe"
 166     } else {
 167         "gcc.exe"
 168     };
 169     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 170     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 171     if target.starts_with("i686-") {
 172         rustc_dlls.push("libgcc_s_dw2-1.dll");
 173     } else {
 174         rustc_dlls.push("libgcc_s_seh-1.dll");
 175     }
 176
 177     let target_libs = [
 178         //MinGW libs
 179         "libgcc.a",
 180         "libgcc_eh.a",
 181         "libgcc_s.a",
 182         "libm.a",
 183         "libmingw32.a",
 184         "libmingwex.a",
 185         "libstdc++.a",
 186         "libiconv.a",
 187         "libmoldname.a",
 188         "libpthread.a",
 189         //Windows import libs
 190         "libadvapi32.a",
 191         "libbcrypt.a",
 192         "libcomctl32.a",
 193         "libcomdlg32.a",
 194         "libcredui.a",
 195         "libcrypt32.a",
 196         "libdbghelp.a",
 197         "libgdi32.a",
 198         "libimagehlp.a",
 199         "libiphlpapi.a",
 200         "libkernel32.a",
 201         "libmsimg32.a",
 202         "libmsvcrt.a",
 203         "libodbc32.a",
 204         "libole32.a",
 205         "liboleaut32.a",
 206         "libopengl32.a",
 207         "libpsapi.a",
 208         "librpcrt4.a",
 209         "libsecur32.a",
 210         "libsetupapi.a",
 211         "libshell32.a",
 212         "libsynchronization.a",
 213         "libuser32.a",
 214         "libuserenv.a",
 215         "libuuid.a",
 216         "libwinhttp.a",
 217         "libwinmm.a",
 218         "libwinspool.a",
 219         "libws2_32.a",
 220         "libwsock32.a",
 221     ];
 222
 223     //Find mingw artifacts we want to bundle
 224     let target_tools = find_files(&target_tools, &bin_path);
 225     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 226     let target_libs = find_files(&target_libs, &lib_path);
 227
 228     // Copy runtime dlls next to rustc.exe
 229     let dist_bin_dir = rust_root.join("bin/");
 230     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 231     for src in rustc_dlls {
 232         builder.copy_to_folder(&src, &dist_bin_dir);
 233     }
 234
 235     //Copy platform tools to platform-specific bin directory
 236     let target_bin_dir = plat_root
 237         .join("lib")
 238         .join("rustlib")
 239         .join(target.triple)
 240         .join("bin")
 241         .join("self-contained");
 242     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 243     for src in target_tools {
 244         builder.copy_to_folder(&src, &target_bin_dir);
 245     }
 246
 247     // Warn windows-gnu users that the bundled GCC cannot compile C files
 248     builder.create(
 249         &target_bin_dir.join("GCC-WARNING.txt"),
 250         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 251          used as a linker. In order to be able to compile projects containing C code use \
 252          the GCC provided by MinGW or Cygwin.",
 253     );
 254
 255     //Copy platform libs to platform-specific lib directory
 256     let target_lib_dir = plat_root
 257         .join("lib")
 258         .join("rustlib")
 259         .join(target.triple)
 260         .join("lib")
 261         .join("self-contained");
 262     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 263     for src in target_libs {
 264         builder.copy_to_folder(&src, &target_lib_dir);
 265     }
 266 }
 267
 268 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 269 pub struct Mingw {
 270     pub host: TargetSelection,
 271 }
 272
 273 impl Step for Mingw {
 274     type Output = Option<GeneratedTarball>;
 275     const DEFAULT: bool = true;
 276
 277     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 278         run.path("rust-mingw")
 279     }
 280
 281     fn make_run(run: RunConfig<'_>) {
 282         run.builder.ensure(Mingw { host: run.target });
 283     }
 284
 285     /// Builds the `rust-mingw` installer component.
 286     ///
 287     /// This contains all the bits and pieces to run the MinGW Windows targets
 288     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 289     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 290         let host = self.host;
 291         if !host.contains("pc-windows-gnu") {
 292             return None;
 293         }
 294
 295         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 296         tarball.set_product_name("Rust MinGW");
 297
 298         // The first argument is a "temporary directory" which is just
 299         // thrown away (this contains the runtime DLLs included in the rustc package
 300         // above) and the second argument is where to place all the MinGW components
 301         // (which is what we want).
 302         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 303
 304         Some(tarball.generate())
 305     }
 306 }
 307
 308 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 309 pub struct Rustc {
 310     pub compiler: Compiler,
 311 }
 312
 313 impl Step for Rustc {
 314     type Output = GeneratedTarball;
 315     const DEFAULT: bool = true;
 316     const ONLY_HOSTS: bool = true;
 317
 318     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 319         run.path("rustc")
 320     }
 321
 322     fn make_run(run: RunConfig<'_>) {
 323         run.builder
 324             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 325     }
 326
 327     /// Creates the `rustc` installer component.
 328     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 329         let compiler = self.compiler;
 330         let host = self.compiler.host;
 331
 332         let tarball = Tarball::new(builder, "rustc", &host.triple);
 333
 334         // Prepare the rustc "image", what will actually end up getting installed
 335         prepare_image(builder, compiler, tarball.image_dir());
 336
 337         // On MinGW we've got a few runtime DLL dependencies that we need to
 338         // include. The first argument to this script is where to put these DLLs
 339         // (the image we're creating), and the second argument is a junk directory
 340         // to ignore all other MinGW stuff the script creates.
 341         //
 342         // On 32-bit MinGW we're always including a DLL which needs some extra
 343         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 344         // anything requiring us to distribute a license, but it's likely the
 345         // install will *also* include the rust-mingw package, which also needs
 346         // licenses, so to be safe we just include it here in all MinGW packages.
 347         if host.contains("pc-windows-gnu") {
 348             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 349             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 350         }
 351
 352         return tarball.generate();
 353
 354         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 355             let host = compiler.host;
 356             let src = builder.sysroot(compiler);
 357
 358             // Copy rustc/rustdoc binaries
 359             t!(fs::create_dir_all(image.join("bin")));
 360             builder.cp_r(&src.join("bin"), &image.join("bin"));
 361
 362             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 363
 364             let libdir_relative = builder.libdir_relative(compiler);
 365
 366             // Copy runtime DLLs needed by the compiler
 367             if libdir_relative.to_str() != Some("bin") {
 368                 let libdir = builder.rustc_libdir(compiler);
 369                 for entry in builder.read_dir(&libdir) {
 370                     let name = entry.file_name();
 371                     if let Some(s) = name.to_str() {
 372                         if is_dylib(s) {
 373                             // Don't use custom libdir here because ^lib/ will be resolved again
 374                             // with installer
 375                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 376                         }
 377                     }
 378                 }
 379             }
 380
 381             // Copy over the codegen backends
 382             let backends_src = builder.sysroot_codegen_backends(compiler);
 383             let backends_rel = backends_src
 384                 .strip_prefix(&src)
 385                 .unwrap()
 386                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 387                 .unwrap();
 388             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 389             let backends_dst = image.join("lib").join(&backends_rel);
 390
 391             t!(fs::create_dir_all(&backends_dst));
 392             builder.cp_r(&backends_src, &backends_dst);
 393
 394             // Copy libLLVM.so to the lib dir as well, if needed. While not
 395             // technically needed by rustc itself it's needed by lots of other
 396             // components like the llvm tools and LLD. LLD is included below and
 397             // tools/LLDB come later, so let's just throw it in the rustc
 398             // component for now.
 399             maybe_install_llvm_runtime(builder, host, image);
 400
 401             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 402             t!(fs::create_dir_all(&dst_dir));
 403
 404             // Copy over lld if it's there
 405             if builder.config.lld_enabled {
 406                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 407                 let rust_lld = exe("rust-lld", compiler.host);
 408                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 409                 // for `-Z gcc-ld=lld`
 410                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 411                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 412                 t!(fs::create_dir(&gcc_lld_dst_dir));
 413                 for flavor in ["ld", "ld64"] {
 414                     let exe_name = exe(flavor, compiler.host);
 415                     builder
 416                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 417                 }
 418             }
 419
 420             // Man pages
 421             t!(fs::create_dir_all(image.join("share/man/man1")));
 422             let man_src = builder.src.join("src/doc/man");
 423             let man_dst = image.join("share/man/man1");
 424
 425             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 426             let time = env::var("SOURCE_DATE_EPOCH")
 427                 .map(|timestamp| {
 428                     let epoch = timestamp
 429                         .parse()
 430                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 431                         .unwrap();
 432
 433                     time::at(Timespec::new(epoch, 0))
 434                 })
 435                 .unwrap_or_else(|_| time::now());
 436
 437             let month_year = t!(time::strftime("%B %Y", &time));
 438             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 439             // to hardlink, and we don't want to edit the source templates
 440             for file_entry in builder.read_dir(&man_src) {
 441                 let page_src = file_entry.path();
 442                 let page_dst = man_dst.join(file_entry.file_name());
 443                 t!(fs::copy(&page_src, &page_dst));
 444                 // template in month/year and version number
 445                 builder.replace_in_file(
 446                     &page_dst,
 447                     &[
 448                         ("<INSERT DATE HERE>", &month_year),
 449                         ("<INSERT VERSION HERE>", &builder.version),
 450                     ],
 451                 );
 452             }
 453
 454             // Debugger scripts
 455             builder
 456                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 457
 458             // Misc license info
 459             let cp = |file: &str| {
 460                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 461             };
 462             cp("COPYRIGHT");
 463             cp("LICENSE-APACHE");
 464             cp("LICENSE-MIT");
 465             cp("README.md");
 466         }
 467     }
 468 }
 469
 470 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 471 pub struct DebuggerScripts {
 472     pub sysroot: Interned<PathBuf>,
 473     pub host: TargetSelection,
 474 }
 475
 476 impl Step for DebuggerScripts {
 477     type Output = ();
 478
 479     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 480         run.path("src/lldb_batchmode.py")
 481     }
 482
 483     fn make_run(run: RunConfig<'_>) {
 484         run.builder.ensure(DebuggerScripts {
 485             sysroot: run
 486                 .builder
 487                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 488             host: run.target,
 489         });
 490     }
 491
 492     /// Copies debugger scripts for `target` into the `sysroot` specified.
 493     fn run(self, builder: &Builder<'_>) {
 494         let host = self.host;
 495         let sysroot = self.sysroot;
 496         let dst = sysroot.join("lib/rustlib/etc");
 497         t!(fs::create_dir_all(&dst));
 498         let cp_debugger_script = |file: &str| {
 499             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 500         };
 501         if host.contains("windows-msvc") {
 502             // windbg debugger scripts
 503             builder.install(
 504                 &builder.src.join("src/etc/rust-windbg.cmd"),
 505                 &sysroot.join("bin"),
 506                 0o755,
 507             );
 508
 509             cp_debugger_script("natvis/intrinsic.natvis");
 510             cp_debugger_script("natvis/liballoc.natvis");
 511             cp_debugger_script("natvis/libcore.natvis");
 512             cp_debugger_script("natvis/libstd.natvis");
 513         } else {
 514             cp_debugger_script("rust_types.py");
 515
 516             // gdb debugger scripts
 517             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 518             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 519
 520             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 521             cp_debugger_script("gdb_lookup.py");
 522             cp_debugger_script("gdb_providers.py");
 523
 524             // lldb debugger scripts
 525             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 526
 527             cp_debugger_script("lldb_lookup.py");
 528             cp_debugger_script("lldb_providers.py");
 529             cp_debugger_script("lldb_commands")
 530         }
 531     }
 532 }
 533
 534 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 535     // The only true set of target libraries came from the build triple, so
 536     // let's reduce redundant work by only producing archives from that host.
 537     if compiler.host != builder.config.build {
 538         builder.info("\tskipping, not a build host");
 539         true
 540     } else {
 541         false
 542     }
 543 }
 544
 545 /// Copy stamped files into an image's `target/lib` directory.
 546 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 547     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 548     let self_contained_dst = dst.join("self-contained");
 549     t!(fs::create_dir_all(&dst));
 550     t!(fs::create_dir_all(&self_contained_dst));
 551     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 552         if dependency_type == DependencyType::TargetSelfContained {
 553             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 554         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 555             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 556         }
 557     }
 558 }
 559
 560 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 561 pub struct Std {
 562     pub compiler: Compiler,
 563     pub target: TargetSelection,
 564 }
 565
 566 impl Step for Std {
 567     type Output = Option<GeneratedTarball>;
 568     const DEFAULT: bool = true;
 569
 570     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 571         run.path("rust-std")
 572     }
 573
 574     fn make_run(run: RunConfig<'_>) {
 575         run.builder.ensure(Std {
 576             compiler: run.builder.compiler_for(
 577                 run.builder.top_stage,
 578                 run.builder.config.build,
 579                 run.target,
 580             ),
 581             target: run.target,
 582         });
 583     }
 584
 585     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 586         let compiler = self.compiler;
 587         let target = self.target;
 588
 589         if skip_host_target_lib(builder, compiler) {
 590             return None;
 591         }
 592
 593         builder.ensure(compile::Std { compiler, target });
 594
 595         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 596         tarball.include_target_in_component_name(true);
 597
 598         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 599         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 600         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 601
 602         Some(tarball.generate())
 603     }
 604 }
 605
 606 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 607 pub struct RustcDev {
 608     pub compiler: Compiler,
 609     pub target: TargetSelection,
 610 }
 611
 612 impl Step for RustcDev {
 613     type Output = Option<GeneratedTarball>;
 614     const DEFAULT: bool = true;
 615     const ONLY_HOSTS: bool = true;
 616
 617     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 618         run.path("rustc-dev")
 619     }
 620
 621     fn make_run(run: RunConfig<'_>) {
 622         run.builder.ensure(RustcDev {
 623             compiler: run.builder.compiler_for(
 624                 run.builder.top_stage,
 625                 run.builder.config.build,
 626                 run.target,
 627             ),
 628             target: run.target,
 629         });
 630     }
 631
 632     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 633         let compiler = self.compiler;
 634         let target = self.target;
 635         if skip_host_target_lib(builder, compiler) {
 636             return None;
 637         }
 638
 639         builder.ensure(compile::Rustc { compiler, target });
 640
 641         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 642
 643         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 644         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 645         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 646
 647         let src_files = &["Cargo.lock"];
 648         // This is the reduced set of paths which will become the rustc-dev component
 649         // (essentially the compiler crates and all of their path dependencies).
 650         copy_src_dirs(
 651             builder,
 652             &builder.src,
 653             &["compiler"],
 654             &[],
 655             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 656         );
 657         // This particular crate is used as a build dependency of the above.
 658         copy_src_dirs(
 659             builder,
 660             &builder.src,
 661             &["src/build_helper"],
 662             &[],
 663             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 664         );
 665         for file in src_files {
 666             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 667         }
 668
 669         Some(tarball.generate())
 670     }
 671 }
 672
 673 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 674 pub struct Analysis {
 675     pub compiler: Compiler,
 676     pub target: TargetSelection,
 677 }
 678
 679 impl Step for Analysis {
 680     type Output = Option<GeneratedTarball>;
 681     const DEFAULT: bool = true;
 682
 683     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 684         let default = should_build_extended_tool(&run.builder, "analysis");
 685         run.path("rust-analysis").default_condition(default)
 686     }
 687
 688     fn make_run(run: RunConfig<'_>) {
 689         run.builder.ensure(Analysis {
 690             // Find the actual compiler (handling the full bootstrap option) which
 691             // produced the save-analysis data because that data isn't copied
 692             // through the sysroot uplifting.
 693             compiler: run.builder.compiler_for(
 694                 run.builder.top_stage,
 695                 run.builder.config.build,
 696                 run.target,
 697             ),
 698             target: run.target,
 699         });
 700     }
 701
 702     /// Creates a tarball of save-analysis metadata, if available.
 703     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 704         let compiler = self.compiler;
 705         let target = self.target;
 706         if compiler.host != builder.config.build {
 707             return None;
 708         }
 709
 710         builder.ensure(compile::Std { compiler, target });
 711         let src = builder
 712             .stage_out(compiler, Mode::Std)
 713             .join(target.triple)
 714             .join(builder.cargo_dir())
 715             .join("deps")
 716             .join("save-analysis");
 717
 718         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 719         tarball.include_target_in_component_name(true);
 720         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 721         Some(tarball.generate())
 722     }
 723 }
 724
 725 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 726 /// `dst_dir`.
 727 fn copy_src_dirs(
 728     builder: &Builder<'_>,
 729     base: &Path,
 730     src_dirs: &[&str],
 731     exclude_dirs: &[&str],
 732     dst_dir: &Path,
 733 ) {
 734     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 735         let spath = match path.to_str() {
 736             Some(path) => path,
 737             None => return false,
 738         };
 739         if spath.ends_with('~') || spath.ends_with(".pyc") {
 740             return false;
 741         }
 742
 743         const LLVM_PROJECTS: &[&str] = &[
 744             "llvm-project/clang",
 745             "llvm-project\\clang",
 746             "llvm-project/libunwind",
 747             "llvm-project\\libunwind",
 748             "llvm-project/lld",
 749             "llvm-project\\lld",
 750             "llvm-project/lldb",
 751             "llvm-project\\lldb",
 752             "llvm-project/llvm",
 753             "llvm-project\\llvm",
 754             "llvm-project/compiler-rt",
 755             "llvm-project\\compiler-rt",
 756         ];
 757         if spath.contains("llvm-project")
 758             && !spath.ends_with("llvm-project")
 759             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 760         {
 761             return false;
 762         }
 763
 764         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 765         if LLVM_TEST.iter().any(|path| spath.contains(path))
 766             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 767         {
 768             return false;
 769         }
 770
 771         let full_path = Path::new(dir).join(path);
 772         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 773             return false;
 774         }
 775
 776         let excludes = [
 777             "CVS",
 778             "RCS",
 779             "SCCS",
 780             ".git",
 781             ".gitignore",
 782             ".gitmodules",
 783             ".gitattributes",
 784             ".cvsignore",
 785             ".svn",
 786             ".arch-ids",
 787             "{arch}",
 788             "=RELEASE-ID",
 789             "=meta-update",
 790             "=update",
 791             ".bzr",
 792             ".bzrignore",
 793             ".bzrtags",
 794             ".hg",
 795             ".hgignore",
 796             ".hgrags",
 797             "_darcs",
 798         ];
 799         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 800     }
 801
 802     // Copy the directories using our filter
 803     for item in src_dirs {
 804         let dst = &dst_dir.join(item);
 805         t!(fs::create_dir_all(dst));
 806         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 807     }
 808 }
 809
 810 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 811 pub struct Src;
 812
 813 impl Step for Src {
 814     /// The output path of the src installer tarball
 815     type Output = GeneratedTarball;
 816     const DEFAULT: bool = true;
 817     const ONLY_HOSTS: bool = true;
 818
 819     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 820         run.path("rust-src")
 821     }
 822
 823     fn make_run(run: RunConfig<'_>) {
 824         run.builder.ensure(Src);
 825     }
 826
 827     /// Creates the `rust-src` installer component
 828     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 829         let tarball = Tarball::new_targetless(builder, "rust-src");
 830
 831         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 832         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 833         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 834         // and fix them...
 835         //
 836         // NOTE: if you update the paths here, you also should update the "virtual" path
 837         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 838         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 839
 840         let src_files = ["Cargo.lock"];
 841         // This is the reduced set of paths which will become the rust-src component
 842         // (essentially libstd and all of its path dependencies).
 843         copy_src_dirs(
 844             builder,
 845             &builder.src,
 846             &["library", "src/llvm-project/libunwind"],
 847             &[
 848                 // not needed and contains symlinks which rustup currently
 849                 // chokes on when unpacking.
 850                 "library/backtrace/crates",
 851             ],
 852             &dst_src,
 853         );
 854         for file in src_files.iter() {
 855             builder.copy(&builder.src.join(file), &dst_src.join(file));
 856         }
 857
 858         tarball.generate()
 859     }
 860 }
 861
 862 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 863 pub struct PlainSourceTarball;
 864
 865 impl Step for PlainSourceTarball {
 866     /// Produces the location of the tarball generated
 867     type Output = GeneratedTarball;
 868     const DEFAULT: bool = true;
 869     const ONLY_HOSTS: bool = true;
 870
 871     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 872         let builder = run.builder;
 873         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 874     }
 875
 876     fn make_run(run: RunConfig<'_>) {
 877         run.builder.ensure(PlainSourceTarball);
 878     }
 879
 880     /// Creates the plain source tarball
 881     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 882         let tarball = Tarball::new(builder, "rustc", "src");
 883         let plain_dst_src = tarball.image_dir();
 884
 885         // This is the set of root paths which will become part of the source package
 886         let src_files = [
 887             "COPYRIGHT",
 888             "LICENSE-APACHE",
 889             "LICENSE-MIT",
 890             "CONTRIBUTING.md",
 891             "README.md",
 892             "RELEASES.md",
 893             "configure",
 894             "x.py",
 895             "config.toml.example",
 896             "Cargo.toml",
 897             "Cargo.lock",
 898         ];
 899         let src_dirs = ["src", "compiler", "library"];
 900
 901         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 902
 903         // Copy the files normally
 904         for item in &src_files {
 905             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 906         }
 907
 908         // Create the version file
 909         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 910         if let Some(sha) = builder.rust_sha() {
 911             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 912         }
 913
 914         // If we're building from git sources, we need to vendor a complete distribution.
 915         if builder.rust_info.is_git() {
 916             // Vendor all Cargo dependencies
 917             let mut cmd = Command::new(&builder.initial_cargo);
 918             cmd.arg("vendor")
 919                 .arg("--sync")
 920                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 921                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 922                 .current_dir(&plain_dst_src);
 923             builder.run(&mut cmd);
 924         }
 925
 926         tarball.bare()
 927     }
 928 }
 929
 930 // We have to run a few shell scripts, which choke quite a bit on both `\`
 931 // characters and on `C:\` paths, so normalize both of them away.
 932 pub fn sanitize_sh(path: &Path) -> String {
 933     let path = path.to_str().unwrap().replace("\\", "/");
 934     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 935
 936     fn unc_to_lfs(s: &str) -> &str {
 937         s.strip_prefix("//?/").unwrap_or(s)
 938     }
 939
 940     fn change_drive(s: &str) -> Option<String> {
 941         let mut ch = s.chars();
 942         let drive = ch.next().unwrap_or('C');
 943         if ch.next() != Some(':') {
 944             return None;
 945         }
 946         if ch.next() != Some('/') {
 947             return None;
 948         }
 949         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 950     }
 951 }
 952
 953 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 954 pub struct Cargo {
 955     pub compiler: Compiler,
 956     pub target: TargetSelection,
 957 }
 958
 959 impl Step for Cargo {
 960     type Output = Option<GeneratedTarball>;
 961     const DEFAULT: bool = true;
 962     const ONLY_HOSTS: bool = true;
 963
 964     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 965         let default = should_build_extended_tool(&run.builder, "cargo");
 966         run.path("cargo").default_condition(default)
 967     }
 968
 969     fn make_run(run: RunConfig<'_>) {
 970         run.builder.ensure(Cargo {
 971             compiler: run.builder.compiler_for(
 972                 run.builder.top_stage,
 973                 run.builder.config.build,
 974                 run.target,
 975             ),
 976             target: run.target,
 977         });
 978     }
 979
 980     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 981         let compiler = self.compiler;
 982         let target = self.target;
 983
 984         let cargo = builder.ensure(tool::Cargo { compiler, target });
 985         let src = builder.src.join("src/tools/cargo");
 986         let etc = src.join("src/etc");
 987
 988         // Prepare the image directory
 989         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 990         tarball.set_overlay(OverlayKind::Cargo);
 991
 992         tarball.add_file(&cargo, "bin", 0o755);
 993         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 994         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 995         tarball.add_dir(etc.join("man"), "share/man/man1");
 996         tarball.add_legal_and_readme_to("share/doc/cargo");
 997
 998         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 999             let dirent = dirent.expect("read dir entry");
1000             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1001                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1002             }
1003         }
1004
1005         Some(tarball.generate())
1006     }
1007 }
1008
1009 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1010 pub struct Rls {
1011     pub compiler: Compiler,
1012     pub target: TargetSelection,
1013 }
1014
1015 impl Step for Rls {
1016     type Output = Option<GeneratedTarball>;
1017     const ONLY_HOSTS: bool = true;
1018     const DEFAULT: bool = true;
1019
1020     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1021         let default = should_build_extended_tool(&run.builder, "rls");
1022         run.path("rls").default_condition(default)
1023     }
1024
1025     fn make_run(run: RunConfig<'_>) {
1026         run.builder.ensure(Rls {
1027             compiler: run.builder.compiler_for(
1028                 run.builder.top_stage,
1029                 run.builder.config.build,
1030                 run.target,
1031             ),
1032             target: run.target,
1033         });
1034     }
1035
1036     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1037         let compiler = self.compiler;
1038         let target = self.target;
1039
1040         let rls = builder
1041             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1042             .or_else(|| {
1043                 missing_tool("RLS", builder.build.config.missing_tools);
1044                 None
1045             })?;
1046
1047         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1048         tarball.set_overlay(OverlayKind::RLS);
1049         tarball.is_preview(true);
1050         tarball.add_file(rls, "bin", 0o755);
1051         tarball.add_legal_and_readme_to("share/doc/rls");
1052         Some(tarball.generate())
1053     }
1054 }
1055
1056 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1057 pub struct RustAnalyzer {
1058     pub compiler: Compiler,
1059     pub target: TargetSelection,
1060 }
1061
1062 impl Step for RustAnalyzer {
1063     type Output = Option<GeneratedTarball>;
1064     const DEFAULT: bool = true;
1065     const ONLY_HOSTS: bool = true;
1066
1067     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1068         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1069         run.path("rust-analyzer").default_condition(default)
1070     }
1071
1072     fn make_run(run: RunConfig<'_>) {
1073         run.builder.ensure(RustAnalyzer {
1074             compiler: run.builder.compiler_for(
1075                 run.builder.top_stage,
1076                 run.builder.config.build,
1077                 run.target,
1078             ),
1079             target: run.target,
1080         });
1081     }
1082
1083     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1084         // This prevents rust-analyzer from being built for "dist" or "install"
1085         // on the stable/beta channels. It is a nightly-only tool and should
1086         // not be included.
1087         if !builder.build.unstable_features() {
1088             return None;
1089         }
1090         let compiler = self.compiler;
1091         let target = self.target;
1092
1093         if target.contains("riscv64") {
1094             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1095             // to build. See #74813 for details.
1096             return None;
1097         }
1098
1099         let rust_analyzer = builder
1100             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1101             .expect("rust-analyzer always builds");
1102
1103         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1104         tarball.set_overlay(OverlayKind::RustAnalyzer);
1105         tarball.is_preview(true);
1106         tarball.add_file(rust_analyzer, "bin", 0o755);
1107         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1108         Some(tarball.generate())
1109     }
1110 }
1111
1112 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1113 pub struct Clippy {
1114     pub compiler: Compiler,
1115     pub target: TargetSelection,
1116 }
1117
1118 impl Step for Clippy {
1119     type Output = Option<GeneratedTarball>;
1120     const DEFAULT: bool = true;
1121     const ONLY_HOSTS: bool = true;
1122
1123     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1124         let default = should_build_extended_tool(&run.builder, "clippy");
1125         run.path("clippy").default_condition(default)
1126     }
1127
1128     fn make_run(run: RunConfig<'_>) {
1129         run.builder.ensure(Clippy {
1130             compiler: run.builder.compiler_for(
1131                 run.builder.top_stage,
1132                 run.builder.config.build,
1133                 run.target,
1134             ),
1135             target: run.target,
1136         });
1137     }
1138
1139     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1140         let compiler = self.compiler;
1141         let target = self.target;
1142
1143         // Prepare the image directory
1144         // We expect clippy to build, because we've exited this step above if tool
1145         // state for clippy isn't testing.
1146         let clippy = builder
1147             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1148             .expect("clippy expected to build - essential tool");
1149         let cargoclippy = builder
1150             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1151             .expect("clippy expected to build - essential tool");
1152
1153         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1154         tarball.set_overlay(OverlayKind::Clippy);
1155         tarball.is_preview(true);
1156         tarball.add_file(clippy, "bin", 0o755);
1157         tarball.add_file(cargoclippy, "bin", 0o755);
1158         tarball.add_legal_and_readme_to("share/doc/clippy");
1159         Some(tarball.generate())
1160     }
1161 }
1162
1163 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1164 pub struct Miri {
1165     pub compiler: Compiler,
1166     pub target: TargetSelection,
1167 }
1168
1169 impl Step for Miri {
1170     type Output = Option<GeneratedTarball>;
1171     const DEFAULT: bool = true;
1172     const ONLY_HOSTS: bool = true;
1173
1174     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1175         let default = should_build_extended_tool(&run.builder, "miri");
1176         run.path("miri").default_condition(default)
1177     }
1178
1179     fn make_run(run: RunConfig<'_>) {
1180         run.builder.ensure(Miri {
1181             compiler: run.builder.compiler_for(
1182                 run.builder.top_stage,
1183                 run.builder.config.build,
1184                 run.target,
1185             ),
1186             target: run.target,
1187         });
1188     }
1189
1190     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1191         // This prevents miri from being built for "dist" or "install"
1192         // on the stable/beta channels. It is a nightly-only tool and should
1193         // not be included.
1194         if !builder.build.unstable_features() {
1195             return None;
1196         }
1197         let compiler = self.compiler;
1198         let target = self.target;
1199
1200         let miri = builder
1201             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1202             .or_else(|| {
1203                 missing_tool("miri", builder.build.config.missing_tools);
1204                 None
1205             })?;
1206         let cargomiri = builder
1207             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1208             .or_else(|| {
1209                 missing_tool("cargo miri", builder.build.config.missing_tools);
1210                 None
1211             })?;
1212
1213         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1214         tarball.set_overlay(OverlayKind::Miri);
1215         tarball.is_preview(true);
1216         tarball.add_file(miri, "bin", 0o755);
1217         tarball.add_file(cargomiri, "bin", 0o755);
1218         tarball.add_legal_and_readme_to("share/doc/miri");
1219         Some(tarball.generate())
1220     }
1221 }
1222
1223 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1224 pub struct Rustfmt {
1225     pub compiler: Compiler,
1226     pub target: TargetSelection,
1227 }
1228
1229 impl Step for Rustfmt {
1230     type Output = Option<GeneratedTarball>;
1231     const DEFAULT: bool = true;
1232     const ONLY_HOSTS: bool = true;
1233
1234     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1235         let default = should_build_extended_tool(&run.builder, "rustfmt");
1236         run.path("rustfmt").default_condition(default)
1237     }
1238
1239     fn make_run(run: RunConfig<'_>) {
1240         run.builder.ensure(Rustfmt {
1241             compiler: run.builder.compiler_for(
1242                 run.builder.top_stage,
1243                 run.builder.config.build,
1244                 run.target,
1245             ),
1246             target: run.target,
1247         });
1248     }
1249
1250     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1251         let compiler = self.compiler;
1252         let target = self.target;
1253
1254         let rustfmt = builder
1255             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1256             .or_else(|| {
1257                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1258                 None
1259             })?;
1260         let cargofmt = builder
1261             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1262             .or_else(|| {
1263                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1264                 None
1265             })?;
1266
1267         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1268         tarball.set_overlay(OverlayKind::Rustfmt);
1269         tarball.is_preview(true);
1270         tarball.add_file(rustfmt, "bin", 0o755);
1271         tarball.add_file(cargofmt, "bin", 0o755);
1272         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1273         Some(tarball.generate())
1274     }
1275 }
1276
1277 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1278 pub struct RustDemangler {
1279     pub compiler: Compiler,
1280     pub target: TargetSelection,
1281 }
1282
1283 impl Step for RustDemangler {
1284     type Output = Option<GeneratedTarball>;
1285     const DEFAULT: bool = true;
1286     const ONLY_HOSTS: bool = true;
1287
1288     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1289         // While other tools use `should_build_extended_tool` to decide whether to be run by
1290         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1291         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1292         // we run the step by default when only `extended = true`, and decide whether to actually
1293         // run it or not later.
1294         let default = run.builder.config.extended;
1295         run.path("rust-demangler").default_condition(default)
1296     }
1297
1298     fn make_run(run: RunConfig<'_>) {
1299         run.builder.ensure(RustDemangler {
1300             compiler: run.builder.compiler_for(
1301                 run.builder.top_stage,
1302                 run.builder.config.build,
1303                 run.target,
1304             ),
1305             target: run.target,
1306         });
1307     }
1308
1309     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1310         let compiler = self.compiler;
1311         let target = self.target;
1312
1313         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1314         let condition = should_build_extended_tool(builder, "rust-demangler")
1315             || builder.config.profiler_enabled(target);
1316         if builder.config.extended && !condition {
1317             return None;
1318         }
1319
1320         let rust_demangler = builder
1321             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1322             .expect("rust-demangler expected to build - in-tree tool");
1323
1324         // Prepare the image directory
1325         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1326         tarball.set_overlay(OverlayKind::RustDemangler);
1327         tarball.is_preview(true);
1328         tarball.add_file(&rust_demangler, "bin", 0o755);
1329         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1330         Some(tarball.generate())
1331     }
1332 }
1333
1334 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1335 pub struct Extended {
1336     stage: u32,
1337     host: TargetSelection,
1338     target: TargetSelection,
1339 }
1340
1341 impl Step for Extended {
1342     type Output = ();
1343     const DEFAULT: bool = true;
1344     const ONLY_HOSTS: bool = true;
1345
1346     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1347         let builder = run.builder;
1348         run.path("extended").default_condition(builder.config.extended)
1349     }
1350
1351     fn make_run(run: RunConfig<'_>) {
1352         run.builder.ensure(Extended {
1353             stage: run.builder.top_stage,
1354             host: run.builder.config.build,
1355             target: run.target,
1356         });
1357     }
1358
1359     /// Creates a combined installer for the specified target in the provided stage.
1360     fn run(self, builder: &Builder<'_>) {
1361         let target = self.target;
1362         let stage = self.stage;
1363         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1364
1365         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1366
1367         let mut tarballs = Vec::new();
1368         let mut built_tools = HashSet::new();
1369         macro_rules! add_component {
1370             ($name:expr => $step:expr) => {
1371                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1372                     tarballs.push(tarball);
1373                     built_tools.insert($name);
1374                 }
1375             };
1376         }
1377
1378         // When rust-std package split from rustc, we needed to ensure that during
1379         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1380         // the std files during uninstall. To do this ensure that rustc comes
1381         // before rust-std in the list below.
1382         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1383         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1384
1385         if target.contains("windows-gnu") {
1386             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1387         }
1388
1389         add_component!("rust-docs" => Docs { host: target });
1390         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1391         add_component!("cargo" => Cargo { compiler, target });
1392         add_component!("rustfmt" => Rustfmt { compiler, target });
1393         add_component!("rls" => Rls { compiler, target });
1394         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1395         add_component!("llvm-components" => LlvmTools { target });
1396         add_component!("clippy" => Clippy { compiler, target });
1397         add_component!("miri" => Miri { compiler, target });
1398         add_component!("analysis" => Analysis { compiler, target });
1399
1400         let etc = builder.src.join("src/etc/installer");
1401
1402         // Avoid producing tarballs during a dry run.
1403         if builder.config.dry_run {
1404             return;
1405         }
1406
1407         let tarball = Tarball::new(builder, "rust", &target.triple);
1408         let generated = tarball.combine(&tarballs);
1409
1410         let tmp = tmpdir(builder).join("combined-tarball");
1411         let work = generated.work_dir();
1412
1413         let mut license = String::new();
1414         license += &builder.read(&builder.src.join("COPYRIGHT"));
1415         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1416         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1417         license.push('\n');
1418         license.push('\n');
1419
1420         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1421         let mut rtf = rtf.to_string();
1422         rtf.push('\n');
1423         for line in license.lines() {
1424             rtf.push_str(line);
1425             rtf.push_str("\\line ");
1426         }
1427         rtf.push('}');
1428
1429         fn filter(contents: &str, marker: &str) -> String {
1430             let start = format!("tool-{}-start", marker);
1431             let end = format!("tool-{}-end", marker);
1432             let mut lines = Vec::new();
1433             let mut omitted = false;
1434             for line in contents.lines() {
1435                 if line.contains(&start) {
1436                     omitted = true;
1437                 } else if line.contains(&end) {
1438                     omitted = false;
1439                 } else if !omitted {
1440                     lines.push(line);
1441                 }
1442             }
1443
1444             lines.join("\n")
1445         }
1446
1447         let xform = |p: &Path| {
1448             let mut contents = t!(fs::read_to_string(p));
1449             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1450                 if !built_tools.contains(tool) {
1451                     contents = filter(&contents, tool);
1452                 }
1453             }
1454             let ret = tmp.join(p.file_name().unwrap());
1455             t!(fs::write(&ret, &contents));
1456             ret
1457         };
1458
1459         if target.contains("apple-darwin") {
1460             builder.info("building pkg installer");
1461             let pkg = tmp.join("pkg");
1462             let _ = fs::remove_dir_all(&pkg);
1463
1464             let pkgbuild = |component: &str| {
1465                 let mut cmd = Command::new("pkgbuild");
1466                 cmd.arg("--identifier")
1467                     .arg(format!("org.rust-lang.{}", component))
1468                     .arg("--scripts")
1469                     .arg(pkg.join(component))
1470                     .arg("--nopayload")
1471                     .arg(pkg.join(component).with_extension("pkg"));
1472                 builder.run(&mut cmd);
1473             };
1474
1475             let prepare = |name: &str| {
1476                 builder.create_dir(&pkg.join(name));
1477                 builder.cp_r(
1478                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1479                     &pkg.join(name),
1480                 );
1481                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1482                 pkgbuild(name);
1483             };
1484             prepare("rustc");
1485             prepare("cargo");
1486             prepare("rust-std");
1487             prepare("rust-analysis");
1488             prepare("clippy");
1489             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1490                 if built_tools.contains(tool) {
1491                     prepare(tool);
1492                 }
1493             }
1494             // create an 'uninstall' package
1495             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1496             pkgbuild("uninstall");
1497
1498             builder.create_dir(&pkg.join("res"));
1499             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1500             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1501             let mut cmd = Command::new("productbuild");
1502             cmd.arg("--distribution")
1503                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1504                 .arg("--resources")
1505                 .arg(pkg.join("res"))
1506                 .arg(distdir(builder).join(format!(
1507                     "{}-{}.pkg",
1508                     pkgname(builder, "rust"),
1509                     target.triple
1510                 )))
1511                 .arg("--package-path")
1512                 .arg(&pkg);
1513             let _time = timeit(builder);
1514             builder.run(&mut cmd);
1515         }
1516
1517         if target.contains("windows") {
1518             let exe = tmp.join("exe");
1519             let _ = fs::remove_dir_all(&exe);
1520
1521             let prepare = |name: &str| {
1522                 builder.create_dir(&exe.join(name));
1523                 let dir = if name == "rust-std" || name == "rust-analysis" {
1524                     format!("{}-{}", name, target.triple)
1525                 } else if name == "rls" {
1526                     "rls-preview".to_string()
1527                 } else if name == "rust-analyzer" {
1528                     "rust-analyzer-preview".to_string()
1529                 } else if name == "clippy" {
1530                     "clippy-preview".to_string()
1531                 } else if name == "rust-demangler" {
1532                     "rust-demangler-preview".to_string()
1533                 } else if name == "miri" {
1534                     "miri-preview".to_string()
1535                 } else {
1536                     name.to_string()
1537                 };
1538                 builder.cp_r(
1539                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1540                     &exe.join(name),
1541                 );
1542                 builder.remove(&exe.join(name).join("manifest.in"));
1543             };
1544             prepare("rustc");
1545             prepare("cargo");
1546             prepare("rust-analysis");
1547             prepare("rust-docs");
1548             prepare("rust-std");
1549             prepare("clippy");
1550             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1551                 if built_tools.contains(tool) {
1552                     prepare(tool);
1553                 }
1554             }
1555             if target.contains("windows-gnu") {
1556                 prepare("rust-mingw");
1557             }
1558
1559             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1560
1561             // Generate msi installer
1562             let wix_path = env::var_os("WIX")
1563                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1564             let wix = PathBuf::from(wix_path);
1565             let heat = wix.join("bin/heat.exe");
1566             let candle = wix.join("bin/candle.exe");
1567             let light = wix.join("bin/light.exe");
1568
1569             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1570             builder.run(
1571                 Command::new(&heat)
1572                     .current_dir(&exe)
1573                     .arg("dir")
1574                     .arg("rustc")
1575                     .args(&heat_flags)
1576                     .arg("-cg")
1577                     .arg("RustcGroup")
1578                     .arg("-dr")
1579                     .arg("Rustc")
1580                     .arg("-var")
1581                     .arg("var.RustcDir")
1582                     .arg("-out")
1583                     .arg(exe.join("RustcGroup.wxs")),
1584             );
1585             builder.run(
1586                 Command::new(&heat)
1587                     .current_dir(&exe)
1588                     .arg("dir")
1589                     .arg("rust-docs")
1590                     .args(&heat_flags)
1591                     .arg("-cg")
1592                     .arg("DocsGroup")
1593                     .arg("-dr")
1594                     .arg("Docs")
1595                     .arg("-var")
1596                     .arg("var.DocsDir")
1597                     .arg("-out")
1598                     .arg(exe.join("DocsGroup.wxs"))
1599                     .arg("-t")
1600                     .arg(etc.join("msi/squash-components.xsl")),
1601             );
1602             builder.run(
1603                 Command::new(&heat)
1604                     .current_dir(&exe)
1605                     .arg("dir")
1606                     .arg("cargo")
1607                     .args(&heat_flags)
1608                     .arg("-cg")
1609                     .arg("CargoGroup")
1610                     .arg("-dr")
1611                     .arg("Cargo")
1612                     .arg("-var")
1613                     .arg("var.CargoDir")
1614                     .arg("-out")
1615                     .arg(exe.join("CargoGroup.wxs"))
1616                     .arg("-t")
1617                     .arg(etc.join("msi/remove-duplicates.xsl")),
1618             );
1619             builder.run(
1620                 Command::new(&heat)
1621                     .current_dir(&exe)
1622                     .arg("dir")
1623                     .arg("rust-std")
1624                     .args(&heat_flags)
1625                     .arg("-cg")
1626                     .arg("StdGroup")
1627                     .arg("-dr")
1628                     .arg("Std")
1629                     .arg("-var")
1630                     .arg("var.StdDir")
1631                     .arg("-out")
1632                     .arg(exe.join("StdGroup.wxs")),
1633             );
1634             if built_tools.contains("rls") {
1635                 builder.run(
1636                     Command::new(&heat)
1637                         .current_dir(&exe)
1638                         .arg("dir")
1639                         .arg("rls")
1640                         .args(&heat_flags)
1641                         .arg("-cg")
1642                         .arg("RlsGroup")
1643                         .arg("-dr")
1644                         .arg("Rls")
1645                         .arg("-var")
1646                         .arg("var.RlsDir")
1647                         .arg("-out")
1648                         .arg(exe.join("RlsGroup.wxs"))
1649                         .arg("-t")
1650                         .arg(etc.join("msi/remove-duplicates.xsl")),
1651                 );
1652             }
1653             if built_tools.contains("rust-analyzer") {
1654                 builder.run(
1655                     Command::new(&heat)
1656                         .current_dir(&exe)
1657                         .arg("dir")
1658                         .arg("rust-analyzer")
1659                         .args(&heat_flags)
1660                         .arg("-cg")
1661                         .arg("RustAnalyzerGroup")
1662                         .arg("-dr")
1663                         .arg("RustAnalyzer")
1664                         .arg("-var")
1665                         .arg("var.RustAnalyzerDir")
1666                         .arg("-out")
1667                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1668                         .arg("-t")
1669                         .arg(etc.join("msi/remove-duplicates.xsl")),
1670                 );
1671             }
1672             builder.run(
1673                 Command::new(&heat)
1674                     .current_dir(&exe)
1675                     .arg("dir")
1676                     .arg("clippy")
1677                     .args(&heat_flags)
1678                     .arg("-cg")
1679                     .arg("ClippyGroup")
1680                     .arg("-dr")
1681                     .arg("Clippy")
1682                     .arg("-var")
1683                     .arg("var.ClippyDir")
1684                     .arg("-out")
1685                     .arg(exe.join("ClippyGroup.wxs"))
1686                     .arg("-t")
1687                     .arg(etc.join("msi/remove-duplicates.xsl")),
1688             );
1689             if built_tools.contains("rust-demangler") {
1690                 builder.run(
1691                     Command::new(&heat)
1692                         .current_dir(&exe)
1693                         .arg("dir")
1694                         .arg("rust-demangler")
1695                         .args(&heat_flags)
1696                         .arg("-cg")
1697                         .arg("RustDemanglerGroup")
1698                         .arg("-dr")
1699                         .arg("RustDemangler")
1700                         .arg("-var")
1701                         .arg("var.RustDemanglerDir")
1702                         .arg("-out")
1703                         .arg(exe.join("RustDemanglerGroup.wxs"))
1704                         .arg("-t")
1705                         .arg(etc.join("msi/remove-duplicates.xsl")),
1706                 );
1707             }
1708             if built_tools.contains("miri") {
1709                 builder.run(
1710                     Command::new(&heat)
1711                         .current_dir(&exe)
1712                         .arg("dir")
1713                         .arg("miri")
1714                         .args(&heat_flags)
1715                         .arg("-cg")
1716                         .arg("MiriGroup")
1717                         .arg("-dr")
1718                         .arg("Miri")
1719                         .arg("-var")
1720                         .arg("var.MiriDir")
1721                         .arg("-out")
1722                         .arg(exe.join("MiriGroup.wxs"))
1723                         .arg("-t")
1724                         .arg(etc.join("msi/remove-duplicates.xsl")),
1725                 );
1726             }
1727             builder.run(
1728                 Command::new(&heat)
1729                     .current_dir(&exe)
1730                     .arg("dir")
1731                     .arg("rust-analysis")
1732                     .args(&heat_flags)
1733                     .arg("-cg")
1734                     .arg("AnalysisGroup")
1735                     .arg("-dr")
1736                     .arg("Analysis")
1737                     .arg("-var")
1738                     .arg("var.AnalysisDir")
1739                     .arg("-out")
1740                     .arg(exe.join("AnalysisGroup.wxs"))
1741                     .arg("-t")
1742                     .arg(etc.join("msi/remove-duplicates.xsl")),
1743             );
1744             if target.contains("windows-gnu") {
1745                 builder.run(
1746                     Command::new(&heat)
1747                         .current_dir(&exe)
1748                         .arg("dir")
1749                         .arg("rust-mingw")
1750                         .args(&heat_flags)
1751                         .arg("-cg")
1752                         .arg("GccGroup")
1753                         .arg("-dr")
1754                         .arg("Gcc")
1755                         .arg("-var")
1756                         .arg("var.GccDir")
1757                         .arg("-out")
1758                         .arg(exe.join("GccGroup.wxs")),
1759                 );
1760             }
1761
1762             let candle = |input: &Path| {
1763                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1764                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1765                 let mut cmd = Command::new(&candle);
1766                 cmd.current_dir(&exe)
1767                     .arg("-nologo")
1768                     .arg("-dRustcDir=rustc")
1769                     .arg("-dDocsDir=rust-docs")
1770                     .arg("-dCargoDir=cargo")
1771                     .arg("-dStdDir=rust-std")
1772                     .arg("-dAnalysisDir=rust-analysis")
1773                     .arg("-dClippyDir=clippy")
1774                     .arg("-arch")
1775                     .arg(&arch)
1776                     .arg("-out")
1777                     .arg(&output)
1778                     .arg(&input);
1779                 add_env(builder, &mut cmd, target);
1780
1781                 if built_tools.contains("rust-demangler") {
1782                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1783                 }
1784                 if built_tools.contains("rls") {
1785                     cmd.arg("-dRlsDir=rls");
1786                 }
1787                 if built_tools.contains("rust-analyzer") {
1788                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1789                 }
1790                 if built_tools.contains("miri") {
1791                     cmd.arg("-dMiriDir=miri");
1792                 }
1793                 if target.contains("windows-gnu") {
1794                     cmd.arg("-dGccDir=rust-mingw");
1795                 }
1796                 builder.run(&mut cmd);
1797             };
1798             candle(&xform(&etc.join("msi/rust.wxs")));
1799             candle(&etc.join("msi/ui.wxs"));
1800             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1801             candle("RustcGroup.wxs".as_ref());
1802             candle("DocsGroup.wxs".as_ref());
1803             candle("CargoGroup.wxs".as_ref());
1804             candle("StdGroup.wxs".as_ref());
1805             candle("ClippyGroup.wxs".as_ref());
1806             if built_tools.contains("rust-demangler") {
1807                 candle("RustDemanglerGroup.wxs".as_ref());
1808             }
1809             if built_tools.contains("rls") {
1810                 candle("RlsGroup.wxs".as_ref());
1811             }
1812             if built_tools.contains("rust-analyzer") {
1813                 candle("RustAnalyzerGroup.wxs".as_ref());
1814             }
1815             if built_tools.contains("miri") {
1816                 candle("MiriGroup.wxs".as_ref());
1817             }
1818             candle("AnalysisGroup.wxs".as_ref());
1819
1820             if target.contains("windows-gnu") {
1821                 candle("GccGroup.wxs".as_ref());
1822             }
1823
1824             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1825             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1826             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1827
1828             builder.info(&format!("building `msi` installer with {:?}", light));
1829             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1830             let mut cmd = Command::new(&light);
1831             cmd.arg("-nologo")
1832                 .arg("-ext")
1833                 .arg("WixUIExtension")
1834                 .arg("-ext")
1835                 .arg("WixUtilExtension")
1836                 .arg("-out")
1837                 .arg(exe.join(&filename))
1838                 .arg("rust.wixobj")
1839                 .arg("ui.wixobj")
1840                 .arg("rustwelcomedlg.wixobj")
1841                 .arg("RustcGroup.wixobj")
1842                 .arg("DocsGroup.wixobj")
1843                 .arg("CargoGroup.wixobj")
1844                 .arg("StdGroup.wixobj")
1845                 .arg("AnalysisGroup.wixobj")
1846                 .arg("ClippyGroup.wixobj")
1847                 .current_dir(&exe);
1848
1849             if built_tools.contains("rls") {
1850                 cmd.arg("RlsGroup.wixobj");
1851             }
1852             if built_tools.contains("rust-analyzer") {
1853                 cmd.arg("RustAnalyzerGroup.wixobj");
1854             }
1855             if built_tools.contains("rust-demangler") {
1856                 cmd.arg("RustDemanglerGroup.wixobj");
1857             }
1858             if built_tools.contains("miri") {
1859                 cmd.arg("MiriGroup.wixobj");
1860             }
1861
1862             if target.contains("windows-gnu") {
1863                 cmd.arg("GccGroup.wixobj");
1864             }
1865             // ICE57 wrongly complains about the shortcuts
1866             cmd.arg("-sice:ICE57");
1867
1868             let _time = timeit(builder);
1869             builder.run(&mut cmd);
1870
1871             if !builder.config.dry_run {
1872                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1873             }
1874         }
1875     }
1876 }
1877
1878 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1879     let mut parts = builder.version.split('.');
1880     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1881         .env("CFG_RELEASE_NUM", &builder.version)
1882         .env("CFG_RELEASE", builder.rust_release())
1883         .env("CFG_VER_MAJOR", parts.next().unwrap())
1884         .env("CFG_VER_MINOR", parts.next().unwrap())
1885         .env("CFG_VER_PATCH", parts.next().unwrap())
1886         .env("CFG_VER_BUILD", "0") // just needed to build
1887         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1888         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1889         .env("CFG_BUILD", target.triple)
1890         .env("CFG_CHANNEL", &builder.config.channel);
1891
1892     if target.contains("windows-gnu") {
1893         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1894     } else {
1895         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1896     }
1897
1898     if target.contains("x86_64") {
1899         cmd.env("CFG_PLATFORM", "x64");
1900     } else {
1901         cmd.env("CFG_PLATFORM", "x86");
1902     }
1903 }
1904
1905 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1906 ///
1907
1908 /// Returns whether the files were actually copied.
1909 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1910     if let Some(config) = builder.config.target_config.get(&target) {
1911         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1912             // If the LLVM was externally provided, then we don't currently copy
1913             // artifacts into the sysroot. This is not necessarily the right
1914             // choice (in particular, it will require the LLVM dylib to be in
1915             // the linker's load path at runtime), but the common use case for
1916             // external LLVMs is distribution provided LLVMs, and in that case
1917             // they're usually in the standard search path (e.g., /usr/lib) and
1918             // copying them here is going to cause problems as we may end up
1919             // with the wrong files and isn't what distributions want.
1920             //
1921             // This behavior may be revisited in the future though.
1922             //
1923             // If the LLVM is coming from ourselves (just from CI) though, we
1924             // still want to install it, as it otherwise won't be available.
1925             return false;
1926         }
1927     }
1928
1929     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1930     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1931     // clear why this is the case, though. llvm-config will emit the versioned
1932     // paths and we don't want those in the sysroot (as we're expecting
1933     // unversioned paths).
1934     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1935         let src_libdir = builder.llvm_out(target).join("lib");
1936         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1937         if llvm_dylib_path.exists() {
1938             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1939         }
1940         !builder.config.dry_run
1941     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1942         let mut cmd = Command::new(llvm_config);
1943         cmd.arg("--libfiles");
1944         builder.verbose(&format!("running {:?}", cmd));
1945         let files = output(&mut cmd);
1946         let build_llvm_out = &builder.llvm_out(builder.config.build);
1947         let target_llvm_out = &builder.llvm_out(target);
1948         for file in files.trim_end().split(' ') {
1949             // If we're not using a custom LLVM, make sure we package for the target.
1950             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1951                 target_llvm_out.join(relative_path)
1952             } else {
1953                 PathBuf::from(file)
1954             };
1955             builder.install(&file, dst_libdir, 0o644);
1956         }
1957         !builder.config.dry_run
1958     } else {
1959         false
1960     }
1961 }
1962
1963 /// Maybe add libLLVM.so to the target lib-dir for linking.
1964 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1965     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1966     // We do not need to copy LLVM files into the sysroot if it is not
1967     // dynamically linked; it is already included into librustc_llvm
1968     // statically.
1969     if builder.config.llvm_link_shared {
1970         maybe_install_llvm(builder, target, &dst_libdir);
1971     }
1972 }
1973
1974 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1975 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1976     let dst_libdir =
1977         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1978     // We do not need to copy LLVM files into the sysroot if it is not
1979     // dynamically linked; it is already included into librustc_llvm
1980     // statically.
1981     if builder.config.llvm_link_shared {
1982         maybe_install_llvm(builder, target, &dst_libdir);
1983     }
1984 }
1985
1986 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1987 pub struct LlvmTools {
1988     pub target: TargetSelection,
1989 }
1990
1991 impl Step for LlvmTools {
1992     type Output = Option<GeneratedTarball>;
1993     const ONLY_HOSTS: bool = true;
1994     const DEFAULT: bool = true;
1995
1996     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1997         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1998         run.path("llvm-tools").default_condition(default)
1999     }
2000
2001     fn make_run(run: RunConfig<'_>) {
2002         run.builder.ensure(LlvmTools { target: run.target });
2003     }
2004
2005     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2006         let target = self.target;
2007
2008         /* run only if llvm-config isn't used */
2009         if let Some(config) = builder.config.target_config.get(&target) {
2010             if let Some(ref _s) = config.llvm_config {
2011                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2012                 return None;
2013             }
2014         }
2015
2016         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2017         tarball.set_overlay(OverlayKind::LLVM);
2018         tarball.is_preview(true);
2019
2020         // Prepare the image directory
2021         let src_bindir = builder.llvm_out(target).join("bin");
2022         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2023         for tool in LLVM_TOOLS {
2024             let exe = src_bindir.join(exe(tool, target));
2025             tarball.add_file(&exe, &dst_bindir, 0o755);
2026         }
2027
2028         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2029         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2030         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2031         // compiler libraries.
2032         maybe_install_llvm_target(builder, target, tarball.image_dir());
2033
2034         Some(tarball.generate())
2035     }
2036 }
2037
2038 // Tarball intended for internal consumption to ease rustc/std development.
2039 //
2040 // Should not be considered stable by end users.
2041 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2042 pub struct RustDev {
2043     pub target: TargetSelection,
2044 }
2045
2046 impl Step for RustDev {
2047     type Output = Option<GeneratedTarball>;
2048     const DEFAULT: bool = true;
2049     const ONLY_HOSTS: bool = true;
2050
2051     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2052         run.path("rust-dev")
2053     }
2054
2055     fn make_run(run: RunConfig<'_>) {
2056         run.builder.ensure(RustDev { target: run.target });
2057     }
2058
2059     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2060         let target = self.target;
2061
2062         /* run only if llvm-config isn't used */
2063         if let Some(config) = builder.config.target_config.get(&target) {
2064             if let Some(ref _s) = config.llvm_config {
2065                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2066                 return None;
2067             }
2068         }
2069
2070         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2071         tarball.set_overlay(OverlayKind::LLVM);
2072
2073         let src_bindir = builder.llvm_out(target).join("bin");
2074         for bin in &[
2075             "llvm-config",
2076             "llvm-ar",
2077             "llvm-objdump",
2078             "llvm-profdata",
2079             "llvm-bcanalyzer",
2080             "llvm-cov",
2081             "llvm-dwp",
2082         ] {
2083             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2084         }
2085
2086         // We don't build LLD on some platforms, so only add it if it exists
2087         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2088         if lld_path.exists() {
2089             tarball.add_file(lld_path, "bin", 0o755);
2090         }
2091
2092         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2093
2094         // Copy the include directory as well; needed mostly to build
2095         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2096         // just broadly useful to be able to link against the bundled LLVM.
2097         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2098
2099         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2100         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2101         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2102         // compiler libraries.
2103         let dst_libdir = tarball.image_dir().join("lib");
2104         maybe_install_llvm(builder, target, &dst_libdir);
2105         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2106         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2107
2108         Some(tarball.generate())
2109     }
2110 }
2111
2112 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2113 /// release process to avoid cloning the monorepo and building stuff.
2114 ///
2115 /// Should not be considered stable by end users.
2116 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2117 pub struct BuildManifest {
2118     pub target: TargetSelection,
2119 }
2120
2121 impl Step for BuildManifest {
2122     type Output = GeneratedTarball;
2123     const DEFAULT: bool = false;
2124     const ONLY_HOSTS: bool = true;
2125
2126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2127         run.path("build-manifest")
2128     }
2129
2130     fn make_run(run: RunConfig<'_>) {
2131         run.builder.ensure(BuildManifest { target: run.target });
2132     }
2133
2134     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2135         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2136
2137         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2138         tarball.add_file(&build_manifest, "bin", 0o755);
2139         tarball.generate()
2140     }
2141 }
2142
2143 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2144 ///
2145 /// Currently this is the PGO profile data.
2146 ///
2147 /// Should not be considered stable by end users.
2148 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2149 pub struct ReproducibleArtifacts {
2150     pub target: TargetSelection,
2151 }
2152
2153 impl Step for ReproducibleArtifacts {
2154     type Output = Option<GeneratedTarball>;
2155     const DEFAULT: bool = true;
2156     const ONLY_HOSTS: bool = true;
2157
2158     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2159         run.path("reproducible-artifacts")
2160     }
2161
2162     fn make_run(run: RunConfig<'_>) {
2163         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2164     }
2165
2166     fn run(self, builder: &Builder<'_>) -> Self::Output {
2167         let mut added_anything = false;
2168         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2169         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2170             tarball.add_file(path, ".", 0o644);
2171             added_anything = true;
2172         }
2173         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2174             tarball.add_file(path, ".", 0o644);
2175             added_anything = true;
2176         }
2177         if added_anything { Some(tarball.generate()) } else { None }
2178     }
2179 }