src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("src/doc").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         run.path("src/librustc")
 100     }
 101
 102     fn make_run(run: RunConfig<'_>) {
 103         run.builder.ensure(RustcDocs { host: run.target });
 104     }
 105
 106     /// Builds the `rustc-docs` installer component.
 107     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 108         let host = self.host;
 109         if !builder.config.compiler_docs {
 110             return None;
 111         }
 112         builder.default_doc(&[]);
 113
 114         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 115         tarball.set_product_name("Rustc Documentation");
 116         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 117         Some(tarball.generate())
 118     }
 119 }
 120
 121 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 122     let mut found = Vec::with_capacity(files.len());
 123
 124     for file in files {
 125         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 126
 127         if let Some(file_path) = file_path {
 128             found.push(file_path);
 129         } else {
 130             panic!("Could not find '{}' in {:?}", file, path);
 131         }
 132     }
 133
 134     found
 135 }
 136
 137 fn make_win_dist(
 138     rust_root: &Path,
 139     plat_root: &Path,
 140     target: TargetSelection,
 141     builder: &Builder<'_>,
 142 ) {
 143     //Ask gcc where it keeps its stuff
 144     let mut cmd = Command::new(builder.cc(target));
 145     cmd.arg("-print-search-dirs");
 146     let gcc_out = output(&mut cmd);
 147
 148     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 149     let mut lib_path = Vec::new();
 150
 151     for line in gcc_out.lines() {
 152         let idx = line.find(':').unwrap();
 153         let key = &line[..idx];
 154         let trim_chars: &[_] = &[' ', '='];
 155         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 156
 157         if key == "programs" {
 158             bin_path.extend(value);
 159         } else if key == "libraries" {
 160             lib_path.extend(value);
 161         }
 162     }
 163
 164     let compiler = if target == "i686-pc-windows-gnu" {
 165         "i686-w64-mingw32-gcc.exe"
 166     } else if target == "x86_64-pc-windows-gnu" {
 167         "x86_64-w64-mingw32-gcc.exe"
 168     } else {
 169         "gcc.exe"
 170     };
 171     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 172     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 173     if target.starts_with("i686-") {
 174         rustc_dlls.push("libgcc_s_dw2-1.dll");
 175     } else {
 176         rustc_dlls.push("libgcc_s_seh-1.dll");
 177     }
 178
 179     let target_libs = [
 180         //MinGW libs
 181         "libgcc.a",
 182         "libgcc_eh.a",
 183         "libgcc_s.a",
 184         "libm.a",
 185         "libmingw32.a",
 186         "libmingwex.a",
 187         "libstdc++.a",
 188         "libiconv.a",
 189         "libmoldname.a",
 190         "libpthread.a",
 191         //Windows import libs
 192         "libadvapi32.a",
 193         "libbcrypt.a",
 194         "libcomctl32.a",
 195         "libcomdlg32.a",
 196         "libcredui.a",
 197         "libcrypt32.a",
 198         "libdbghelp.a",
 199         "libgdi32.a",
 200         "libimagehlp.a",
 201         "libiphlpapi.a",
 202         "libkernel32.a",
 203         "libmsimg32.a",
 204         "libmsvcrt.a",
 205         "libodbc32.a",
 206         "libole32.a",
 207         "liboleaut32.a",
 208         "libopengl32.a",
 209         "libpsapi.a",
 210         "librpcrt4.a",
 211         "libsecur32.a",
 212         "libsetupapi.a",
 213         "libshell32.a",
 214         "libsynchronization.a",
 215         "libuser32.a",
 216         "libuserenv.a",
 217         "libuuid.a",
 218         "libwinhttp.a",
 219         "libwinmm.a",
 220         "libwinspool.a",
 221         "libws2_32.a",
 222         "libwsock32.a",
 223     ];
 224
 225     //Find mingw artifacts we want to bundle
 226     let target_tools = find_files(&target_tools, &bin_path);
 227     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 228     let target_libs = find_files(&target_libs, &lib_path);
 229
 230     // Copy runtime dlls next to rustc.exe
 231     let dist_bin_dir = rust_root.join("bin/");
 232     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 233     for src in rustc_dlls {
 234         builder.copy_to_folder(&src, &dist_bin_dir);
 235     }
 236
 237     //Copy platform tools to platform-specific bin directory
 238     let target_bin_dir = plat_root
 239         .join("lib")
 240         .join("rustlib")
 241         .join(target.triple)
 242         .join("bin")
 243         .join("self-contained");
 244     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 245     for src in target_tools {
 246         builder.copy_to_folder(&src, &target_bin_dir);
 247     }
 248
 249     // Warn windows-gnu users that the bundled GCC cannot compile C files
 250     builder.create(
 251         &target_bin_dir.join("GCC-WARNING.txt"),
 252         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 253          used as a linker. In order to be able to compile projects containing C code use \
 254          the GCC provided by MinGW or Cygwin.",
 255     );
 256
 257     //Copy platform libs to platform-specific lib directory
 258     let target_lib_dir = plat_root
 259         .join("lib")
 260         .join("rustlib")
 261         .join(target.triple)
 262         .join("lib")
 263         .join("self-contained");
 264     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 265     for src in target_libs {
 266         builder.copy_to_folder(&src, &target_lib_dir);
 267     }
 268 }
 269
 270 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 271 pub struct Mingw {
 272     pub host: TargetSelection,
 273 }
 274
 275 impl Step for Mingw {
 276     type Output = Option<GeneratedTarball>;
 277     const DEFAULT: bool = true;
 278
 279     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 280         run.never()
 281     }
 282
 283     fn make_run(run: RunConfig<'_>) {
 284         run.builder.ensure(Mingw { host: run.target });
 285     }
 286
 287     /// Builds the `rust-mingw` installer component.
 288     ///
 289     /// This contains all the bits and pieces to run the MinGW Windows targets
 290     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 291     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 292         let host = self.host;
 293         if !host.contains("pc-windows-gnu") {
 294             return None;
 295         }
 296
 297         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 298         tarball.set_product_name("Rust MinGW");
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 305
 306         Some(tarball.generate())
 307     }
 308 }
 309
 310 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 311 pub struct Rustc {
 312     pub compiler: Compiler,
 313 }
 314
 315 impl Step for Rustc {
 316     type Output = GeneratedTarball;
 317     const DEFAULT: bool = true;
 318     const ONLY_HOSTS: bool = true;
 319
 320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 321         run.path("src/librustc")
 322     }
 323
 324     fn make_run(run: RunConfig<'_>) {
 325         run.builder
 326             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 327     }
 328
 329     /// Creates the `rustc` installer component.
 330     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 331         let compiler = self.compiler;
 332         let host = self.compiler.host;
 333
 334         let tarball = Tarball::new(builder, "rustc", &host.triple);
 335
 336         // Prepare the rustc "image", what will actually end up getting installed
 337         prepare_image(builder, compiler, tarball.image_dir());
 338
 339         // On MinGW we've got a few runtime DLL dependencies that we need to
 340         // include. The first argument to this script is where to put these DLLs
 341         // (the image we're creating), and the second argument is a junk directory
 342         // to ignore all other MinGW stuff the script creates.
 343         //
 344         // On 32-bit MinGW we're always including a DLL which needs some extra
 345         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 346         // anything requiring us to distribute a license, but it's likely the
 347         // install will *also* include the rust-mingw package, which also needs
 348         // licenses, so to be safe we just include it here in all MinGW packages.
 349         if host.contains("pc-windows-gnu") {
 350             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 351             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 352         }
 353
 354         return tarball.generate();
 355
 356         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 357             let host = compiler.host;
 358             let src = builder.sysroot(compiler);
 359
 360             // Copy rustc/rustdoc binaries
 361             t!(fs::create_dir_all(image.join("bin")));
 362             builder.cp_r(&src.join("bin"), &image.join("bin"));
 363
 364             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 365
 366             let libdir_relative = builder.libdir_relative(compiler);
 367
 368             // Copy runtime DLLs needed by the compiler
 369             if libdir_relative.to_str() != Some("bin") {
 370                 let libdir = builder.rustc_libdir(compiler);
 371                 for entry in builder.read_dir(&libdir) {
 372                     let name = entry.file_name();
 373                     if let Some(s) = name.to_str() {
 374                         if is_dylib(s) {
 375                             // Don't use custom libdir here because ^lib/ will be resolved again
 376                             // with installer
 377                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 378                         }
 379                     }
 380                 }
 381             }
 382
 383             // Copy over the codegen backends
 384             let backends_src = builder.sysroot_codegen_backends(compiler);
 385             let backends_rel = backends_src
 386                 .strip_prefix(&src)
 387                 .unwrap()
 388                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 389                 .unwrap();
 390             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 391             let backends_dst = image.join("lib").join(&backends_rel);
 392
 393             t!(fs::create_dir_all(&backends_dst));
 394             builder.cp_r(&backends_src, &backends_dst);
 395
 396             // Copy libLLVM.so to the lib dir as well, if needed. While not
 397             // technically needed by rustc itself it's needed by lots of other
 398             // components like the llvm tools and LLD. LLD is included below and
 399             // tools/LLDB come later, so let's just throw it in the rustc
 400             // component for now.
 401             maybe_install_llvm_runtime(builder, host, image);
 402
 403             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 405             t!(fs::create_dir_all(&dst_dir));
 406
 407             // Copy over lld if it's there
 408             if builder.config.lld_enabled {
 409                 let rust_lld = exe("rust-lld", compiler.host);
 410                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 411                 // for `-Z gcc-ld=lld`
 412                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 413                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 414                 t!(fs::create_dir(&gcc_lld_dst_dir));
 415                 for flavor in ["ld", "ld64"] {
 416                     let exe_name = exe(flavor, compiler.host);
 417                     builder
 418                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 419                 }
 420             }
 421
 422             // Copy over llvm-dwp if it's there
 423             let exe = exe("rust-llvm-dwp", compiler.host);
 424             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 425
 426             // Man pages
 427             t!(fs::create_dir_all(image.join("share/man/man1")));
 428             let man_src = builder.src.join("src/doc/man");
 429             let man_dst = image.join("share/man/man1");
 430
 431             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 432             let time = env::var("SOURCE_DATE_EPOCH")
 433                 .map(|timestamp| {
 434                     let epoch = timestamp
 435                         .parse()
 436                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 437                         .unwrap();
 438
 439                     time::at(Timespec::new(epoch, 0))
 440                 })
 441                 .unwrap_or_else(|_| time::now());
 442
 443             let month_year = t!(time::strftime("%B %Y", &time));
 444             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 445             // to hardlink, and we don't want to edit the source templates
 446             for file_entry in builder.read_dir(&man_src) {
 447                 let page_src = file_entry.path();
 448                 let page_dst = man_dst.join(file_entry.file_name());
 449                 t!(fs::copy(&page_src, &page_dst));
 450                 // template in month/year and version number
 451                 builder.replace_in_file(
 452                     &page_dst,
 453                     &[
 454                         ("<INSERT DATE HERE>", &month_year),
 455                         ("<INSERT VERSION HERE>", &builder.version),
 456                     ],
 457                 );
 458             }
 459
 460             // Debugger scripts
 461             builder
 462                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 463
 464             // Misc license info
 465             let cp = |file: &str| {
 466                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 467             };
 468             cp("COPYRIGHT");
 469             cp("LICENSE-APACHE");
 470             cp("LICENSE-MIT");
 471             cp("README.md");
 472         }
 473     }
 474 }
 475
 476 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 477 pub struct DebuggerScripts {
 478     pub sysroot: Interned<PathBuf>,
 479     pub host: TargetSelection,
 480 }
 481
 482 impl Step for DebuggerScripts {
 483     type Output = ();
 484
 485     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 486         run.path("src/lldb_batchmode.py")
 487     }
 488
 489     fn make_run(run: RunConfig<'_>) {
 490         run.builder.ensure(DebuggerScripts {
 491             sysroot: run
 492                 .builder
 493                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 494             host: run.target,
 495         });
 496     }
 497
 498     /// Copies debugger scripts for `target` into the `sysroot` specified.
 499     fn run(self, builder: &Builder<'_>) {
 500         let host = self.host;
 501         let sysroot = self.sysroot;
 502         let dst = sysroot.join("lib/rustlib/etc");
 503         t!(fs::create_dir_all(&dst));
 504         let cp_debugger_script = |file: &str| {
 505             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 506         };
 507         if host.contains("windows-msvc") {
 508             // windbg debugger scripts
 509             builder.install(
 510                 &builder.src.join("src/etc/rust-windbg.cmd"),
 511                 &sysroot.join("bin"),
 512                 0o755,
 513             );
 514
 515             cp_debugger_script("natvis/intrinsic.natvis");
 516             cp_debugger_script("natvis/liballoc.natvis");
 517             cp_debugger_script("natvis/libcore.natvis");
 518             cp_debugger_script("natvis/libstd.natvis");
 519         } else {
 520             cp_debugger_script("rust_types.py");
 521
 522             // gdb debugger scripts
 523             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 524             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 525
 526             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 527             cp_debugger_script("gdb_lookup.py");
 528             cp_debugger_script("gdb_providers.py");
 529
 530             // lldb debugger scripts
 531             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 532
 533             cp_debugger_script("lldb_lookup.py");
 534             cp_debugger_script("lldb_providers.py");
 535             cp_debugger_script("lldb_commands")
 536         }
 537     }
 538 }
 539
 540 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 541     // The only true set of target libraries came from the build triple, so
 542     // let's reduce redundant work by only producing archives from that host.
 543     if compiler.host != builder.config.build {
 544         builder.info("\tskipping, not a build host");
 545         true
 546     } else {
 547         false
 548     }
 549 }
 550
 551 /// Copy stamped files into an image's `target/lib` directory.
 552 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 553     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 554     let self_contained_dst = dst.join("self-contained");
 555     t!(fs::create_dir_all(&dst));
 556     t!(fs::create_dir_all(&self_contained_dst));
 557     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 558         if dependency_type == DependencyType::TargetSelfContained {
 559             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 560         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 561             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 562         }
 563     }
 564 }
 565
 566 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 567 pub struct Std {
 568     pub compiler: Compiler,
 569     pub target: TargetSelection,
 570 }
 571
 572 impl Step for Std {
 573     type Output = Option<GeneratedTarball>;
 574     const DEFAULT: bool = true;
 575
 576     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 577         run.path("library/std")
 578     }
 579
 580     fn make_run(run: RunConfig<'_>) {
 581         run.builder.ensure(Std {
 582             compiler: run.builder.compiler_for(
 583                 run.builder.top_stage,
 584                 run.builder.config.build,
 585                 run.target,
 586             ),
 587             target: run.target,
 588         });
 589     }
 590
 591     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 592         let compiler = self.compiler;
 593         let target = self.target;
 594
 595         if skip_host_target_lib(builder, compiler) {
 596             return None;
 597         }
 598
 599         builder.ensure(compile::Std { compiler, target });
 600
 601         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 602         tarball.include_target_in_component_name(true);
 603
 604         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 605         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 606         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 607
 608         Some(tarball.generate())
 609     }
 610 }
 611
 612 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 613 pub struct RustcDev {
 614     pub compiler: Compiler,
 615     pub target: TargetSelection,
 616 }
 617
 618 impl Step for RustcDev {
 619     type Output = Option<GeneratedTarball>;
 620     const DEFAULT: bool = true;
 621     const ONLY_HOSTS: bool = true;
 622
 623     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 624         run.path("rustc-dev")
 625     }
 626
 627     fn make_run(run: RunConfig<'_>) {
 628         run.builder.ensure(RustcDev {
 629             compiler: run.builder.compiler_for(
 630                 run.builder.top_stage,
 631                 run.builder.config.build,
 632                 run.target,
 633             ),
 634             target: run.target,
 635         });
 636     }
 637
 638     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 639         let compiler = self.compiler;
 640         let target = self.target;
 641         if skip_host_target_lib(builder, compiler) {
 642             return None;
 643         }
 644
 645         builder.ensure(compile::Rustc { compiler, target });
 646
 647         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 648
 649         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 650         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 651         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 652
 653         let src_files = &["Cargo.lock"];
 654         // This is the reduced set of paths which will become the rustc-dev component
 655         // (essentially the compiler crates and all of their path dependencies).
 656         copy_src_dirs(
 657             builder,
 658             &builder.src,
 659             &["compiler"],
 660             &[],
 661             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 662         );
 663         // This particular crate is used as a build dependency of the above.
 664         copy_src_dirs(
 665             builder,
 666             &builder.src,
 667             &["src/build_helper"],
 668             &[],
 669             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 670         );
 671         for file in src_files {
 672             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 673         }
 674
 675         Some(tarball.generate())
 676     }
 677 }
 678
 679 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 680 pub struct Analysis {
 681     pub compiler: Compiler,
 682     pub target: TargetSelection,
 683 }
 684
 685 impl Step for Analysis {
 686     type Output = Option<GeneratedTarball>;
 687     const DEFAULT: bool = true;
 688
 689     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 690         let default = should_build_extended_tool(&run.builder, "analysis");
 691         run.path("analysis").default_condition(default)
 692     }
 693
 694     fn make_run(run: RunConfig<'_>) {
 695         run.builder.ensure(Analysis {
 696             // Find the actual compiler (handling the full bootstrap option) which
 697             // produced the save-analysis data because that data isn't copied
 698             // through the sysroot uplifting.
 699             compiler: run.builder.compiler_for(
 700                 run.builder.top_stage,
 701                 run.builder.config.build,
 702                 run.target,
 703             ),
 704             target: run.target,
 705         });
 706     }
 707
 708     /// Creates a tarball of save-analysis metadata, if available.
 709     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 710         let compiler = self.compiler;
 711         let target = self.target;
 712         if compiler.host != builder.config.build {
 713             return None;
 714         }
 715
 716         builder.ensure(compile::Std { compiler, target });
 717         let src = builder
 718             .stage_out(compiler, Mode::Std)
 719             .join(target.triple)
 720             .join(builder.cargo_dir())
 721             .join("deps")
 722             .join("save-analysis");
 723
 724         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 725         tarball.include_target_in_component_name(true);
 726         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 727         Some(tarball.generate())
 728     }
 729 }
 730
 731 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 732 /// `dst_dir`.
 733 fn copy_src_dirs(
 734     builder: &Builder<'_>,
 735     base: &Path,
 736     src_dirs: &[&str],
 737     exclude_dirs: &[&str],
 738     dst_dir: &Path,
 739 ) {
 740     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 741         let spath = match path.to_str() {
 742             Some(path) => path,
 743             None => return false,
 744         };
 745         if spath.ends_with('~') || spath.ends_with(".pyc") {
 746             return false;
 747         }
 748
 749         const LLVM_PROJECTS: &[&str] = &[
 750             "llvm-project/clang",
 751             "llvm-project\\clang",
 752             "llvm-project/libunwind",
 753             "llvm-project\\libunwind",
 754             "llvm-project/lld",
 755             "llvm-project\\lld",
 756             "llvm-project/lldb",
 757             "llvm-project\\lldb",
 758             "llvm-project/llvm",
 759             "llvm-project\\llvm",
 760             "llvm-project/compiler-rt",
 761             "llvm-project\\compiler-rt",
 762         ];
 763         if spath.contains("llvm-project")
 764             && !spath.ends_with("llvm-project")
 765             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 766         {
 767             return false;
 768         }
 769
 770         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 771         if LLVM_TEST.iter().any(|path| spath.contains(path))
 772             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 773         {
 774             return false;
 775         }
 776
 777         let full_path = Path::new(dir).join(path);
 778         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 779             return false;
 780         }
 781
 782         let excludes = [
 783             "CVS",
 784             "RCS",
 785             "SCCS",
 786             ".git",
 787             ".gitignore",
 788             ".gitmodules",
 789             ".gitattributes",
 790             ".cvsignore",
 791             ".svn",
 792             ".arch-ids",
 793             "{arch}",
 794             "=RELEASE-ID",
 795             "=meta-update",
 796             "=update",
 797             ".bzr",
 798             ".bzrignore",
 799             ".bzrtags",
 800             ".hg",
 801             ".hgignore",
 802             ".hgrags",
 803             "_darcs",
 804         ];
 805         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 806     }
 807
 808     // Copy the directories using our filter
 809     for item in src_dirs {
 810         let dst = &dst_dir.join(item);
 811         t!(fs::create_dir_all(dst));
 812         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 813     }
 814 }
 815
 816 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 817 pub struct Src;
 818
 819 impl Step for Src {
 820     /// The output path of the src installer tarball
 821     type Output = GeneratedTarball;
 822     const DEFAULT: bool = true;
 823     const ONLY_HOSTS: bool = true;
 824
 825     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 826         run.path("src")
 827     }
 828
 829     fn make_run(run: RunConfig<'_>) {
 830         run.builder.ensure(Src);
 831     }
 832
 833     /// Creates the `rust-src` installer component
 834     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 835         let tarball = Tarball::new_targetless(builder, "rust-src");
 836
 837         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 838         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 839         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 840         // and fix them...
 841         //
 842         // NOTE: if you update the paths here, you also should update the "virtual" path
 843         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 844         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 845
 846         let src_files = ["Cargo.lock"];
 847         // This is the reduced set of paths which will become the rust-src component
 848         // (essentially libstd and all of its path dependencies).
 849         copy_src_dirs(
 850             builder,
 851             &builder.src,
 852             &["library", "src/llvm-project/libunwind"],
 853             &[
 854                 // not needed and contains symlinks which rustup currently
 855                 // chokes on when unpacking.
 856                 "library/backtrace/crates",
 857             ],
 858             &dst_src,
 859         );
 860         for file in src_files.iter() {
 861             builder.copy(&builder.src.join(file), &dst_src.join(file));
 862         }
 863
 864         tarball.generate()
 865     }
 866 }
 867
 868 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 869 pub struct PlainSourceTarball;
 870
 871 impl Step for PlainSourceTarball {
 872     /// Produces the location of the tarball generated
 873     type Output = GeneratedTarball;
 874     const DEFAULT: bool = true;
 875     const ONLY_HOSTS: bool = true;
 876
 877     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 878         let builder = run.builder;
 879         run.path("src").default_condition(builder.config.rust_dist_src)
 880     }
 881
 882     fn make_run(run: RunConfig<'_>) {
 883         run.builder.ensure(PlainSourceTarball);
 884     }
 885
 886     /// Creates the plain source tarball
 887     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 888         let tarball = Tarball::new(builder, "rustc", "src");
 889         let plain_dst_src = tarball.image_dir();
 890
 891         // This is the set of root paths which will become part of the source package
 892         let src_files = [
 893             "COPYRIGHT",
 894             "LICENSE-APACHE",
 895             "LICENSE-MIT",
 896             "CONTRIBUTING.md",
 897             "README.md",
 898             "RELEASES.md",
 899             "configure",
 900             "x.py",
 901             "config.toml.example",
 902             "Cargo.toml",
 903             "Cargo.lock",
 904         ];
 905         let src_dirs = ["src", "compiler", "library"];
 906
 907         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 908
 909         // Copy the files normally
 910         for item in &src_files {
 911             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 912         }
 913
 914         // Create the version file
 915         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 916         if let Some(sha) = builder.rust_sha() {
 917             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 918         }
 919
 920         // If we're building from git sources, we need to vendor a complete distribution.
 921         if builder.rust_info.is_git() {
 922             // Vendor all Cargo dependencies
 923             let mut cmd = Command::new(&builder.initial_cargo);
 924             cmd.arg("vendor")
 925                 .arg("--sync")
 926                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 927                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 928                 .current_dir(&plain_dst_src);
 929             builder.run(&mut cmd);
 930         }
 931
 932         tarball.bare()
 933     }
 934 }
 935
 936 // We have to run a few shell scripts, which choke quite a bit on both `\`
 937 // characters and on `C:\` paths, so normalize both of them away.
 938 pub fn sanitize_sh(path: &Path) -> String {
 939     let path = path.to_str().unwrap().replace("\\", "/");
 940     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 941
 942     fn unc_to_lfs(s: &str) -> &str {
 943         s.strip_prefix("//?/").unwrap_or(s)
 944     }
 945
 946     fn change_drive(s: &str) -> Option<String> {
 947         let mut ch = s.chars();
 948         let drive = ch.next().unwrap_or('C');
 949         if ch.next() != Some(':') {
 950             return None;
 951         }
 952         if ch.next() != Some('/') {
 953             return None;
 954         }
 955         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 956     }
 957 }
 958
 959 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 960 pub struct Cargo {
 961     pub compiler: Compiler,
 962     pub target: TargetSelection,
 963 }
 964
 965 impl Step for Cargo {
 966     type Output = Option<GeneratedTarball>;
 967     const DEFAULT: bool = true;
 968     const ONLY_HOSTS: bool = true;
 969
 970     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 971         let default = should_build_extended_tool(&run.builder, "cargo");
 972         run.path("cargo").default_condition(default)
 973     }
 974
 975     fn make_run(run: RunConfig<'_>) {
 976         run.builder.ensure(Cargo {
 977             compiler: run.builder.compiler_for(
 978                 run.builder.top_stage,
 979                 run.builder.config.build,
 980                 run.target,
 981             ),
 982             target: run.target,
 983         });
 984     }
 985
 986     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 987         let compiler = self.compiler;
 988         let target = self.target;
 989
 990         let cargo = builder.ensure(tool::Cargo { compiler, target });
 991         let src = builder.src.join("src/tools/cargo");
 992         let etc = src.join("src/etc");
 993
 994         // Prepare the image directory
 995         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 996         tarball.set_overlay(OverlayKind::Cargo);
 997
 998         tarball.add_file(&cargo, "bin", 0o755);
 999         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
1000         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1001         tarball.add_dir(etc.join("man"), "share/man/man1");
1002         tarball.add_legal_and_readme_to("share/doc/cargo");
1003
1004         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1005             let dirent = dirent.expect("read dir entry");
1006             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1007                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1008             }
1009         }
1010
1011         Some(tarball.generate())
1012     }
1013 }
1014
1015 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1016 pub struct Rls {
1017     pub compiler: Compiler,
1018     pub target: TargetSelection,
1019 }
1020
1021 impl Step for Rls {
1022     type Output = Option<GeneratedTarball>;
1023     const ONLY_HOSTS: bool = true;
1024     const DEFAULT: bool = true;
1025
1026     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1027         let default = should_build_extended_tool(&run.builder, "rls");
1028         run.path("rls").default_condition(default)
1029     }
1030
1031     fn make_run(run: RunConfig<'_>) {
1032         run.builder.ensure(Rls {
1033             compiler: run.builder.compiler_for(
1034                 run.builder.top_stage,
1035                 run.builder.config.build,
1036                 run.target,
1037             ),
1038             target: run.target,
1039         });
1040     }
1041
1042     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1043         let compiler = self.compiler;
1044         let target = self.target;
1045
1046         let rls = builder
1047             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1048             .or_else(|| {
1049                 missing_tool("RLS", builder.build.config.missing_tools);
1050                 None
1051             })?;
1052
1053         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1054         tarball.set_overlay(OverlayKind::RLS);
1055         tarball.is_preview(true);
1056         tarball.add_file(rls, "bin", 0o755);
1057         tarball.add_legal_and_readme_to("share/doc/rls");
1058         Some(tarball.generate())
1059     }
1060 }
1061
1062 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1063 pub struct RustAnalyzer {
1064     pub compiler: Compiler,
1065     pub target: TargetSelection,
1066 }
1067
1068 impl Step for RustAnalyzer {
1069     type Output = Option<GeneratedTarball>;
1070     const DEFAULT: bool = true;
1071     const ONLY_HOSTS: bool = true;
1072
1073     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1074         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1075         run.path("rust-analyzer").default_condition(default)
1076     }
1077
1078     fn make_run(run: RunConfig<'_>) {
1079         run.builder.ensure(RustAnalyzer {
1080             compiler: run.builder.compiler_for(
1081                 run.builder.top_stage,
1082                 run.builder.config.build,
1083                 run.target,
1084             ),
1085             target: run.target,
1086         });
1087     }
1088
1089     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1090         // This prevents rust-analyzer from being built for "dist" or "install"
1091         // on the stable/beta channels. It is a nightly-only tool and should
1092         // not be included.
1093         if !builder.build.unstable_features() {
1094             return None;
1095         }
1096         let compiler = self.compiler;
1097         let target = self.target;
1098
1099         if target.contains("riscv64") {
1100             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1101             // to build. See #74813 for details.
1102             return None;
1103         }
1104
1105         let rust_analyzer = builder
1106             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1107             .expect("rust-analyzer always builds");
1108
1109         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1110         tarball.set_overlay(OverlayKind::RustAnalyzer);
1111         tarball.is_preview(true);
1112         tarball.add_file(rust_analyzer, "bin", 0o755);
1113         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1114         Some(tarball.generate())
1115     }
1116 }
1117
1118 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1119 pub struct Clippy {
1120     pub compiler: Compiler,
1121     pub target: TargetSelection,
1122 }
1123
1124 impl Step for Clippy {
1125     type Output = Option<GeneratedTarball>;
1126     const DEFAULT: bool = true;
1127     const ONLY_HOSTS: bool = true;
1128
1129     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1130         let default = should_build_extended_tool(&run.builder, "clippy");
1131         run.path("clippy").default_condition(default)
1132     }
1133
1134     fn make_run(run: RunConfig<'_>) {
1135         run.builder.ensure(Clippy {
1136             compiler: run.builder.compiler_for(
1137                 run.builder.top_stage,
1138                 run.builder.config.build,
1139                 run.target,
1140             ),
1141             target: run.target,
1142         });
1143     }
1144
1145     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1146         let compiler = self.compiler;
1147         let target = self.target;
1148
1149         // Prepare the image directory
1150         // We expect clippy to build, because we've exited this step above if tool
1151         // state for clippy isn't testing.
1152         let clippy = builder
1153             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1154             .expect("clippy expected to build - essential tool");
1155         let cargoclippy = builder
1156             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1157             .expect("clippy expected to build - essential tool");
1158
1159         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1160         tarball.set_overlay(OverlayKind::Clippy);
1161         tarball.is_preview(true);
1162         tarball.add_file(clippy, "bin", 0o755);
1163         tarball.add_file(cargoclippy, "bin", 0o755);
1164         tarball.add_legal_and_readme_to("share/doc/clippy");
1165         Some(tarball.generate())
1166     }
1167 }
1168
1169 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1170 pub struct Miri {
1171     pub compiler: Compiler,
1172     pub target: TargetSelection,
1173 }
1174
1175 impl Step for Miri {
1176     type Output = Option<GeneratedTarball>;
1177     const DEFAULT: bool = true;
1178     const ONLY_HOSTS: bool = true;
1179
1180     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1181         let default = should_build_extended_tool(&run.builder, "miri");
1182         run.path("miri").default_condition(default)
1183     }
1184
1185     fn make_run(run: RunConfig<'_>) {
1186         run.builder.ensure(Miri {
1187             compiler: run.builder.compiler_for(
1188                 run.builder.top_stage,
1189                 run.builder.config.build,
1190                 run.target,
1191             ),
1192             target: run.target,
1193         });
1194     }
1195
1196     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1197         // This prevents miri from being built for "dist" or "install"
1198         // on the stable/beta channels. It is a nightly-only tool and should
1199         // not be included.
1200         if !builder.build.unstable_features() {
1201             return None;
1202         }
1203         let compiler = self.compiler;
1204         let target = self.target;
1205
1206         let miri = builder
1207             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1208             .or_else(|| {
1209                 missing_tool("miri", builder.build.config.missing_tools);
1210                 None
1211             })?;
1212         let cargomiri = builder
1213             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1214             .or_else(|| {
1215                 missing_tool("cargo miri", builder.build.config.missing_tools);
1216                 None
1217             })?;
1218
1219         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1220         tarball.set_overlay(OverlayKind::Miri);
1221         tarball.is_preview(true);
1222         tarball.add_file(miri, "bin", 0o755);
1223         tarball.add_file(cargomiri, "bin", 0o755);
1224         tarball.add_legal_and_readme_to("share/doc/miri");
1225         Some(tarball.generate())
1226     }
1227 }
1228
1229 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1230 pub struct Rustfmt {
1231     pub compiler: Compiler,
1232     pub target: TargetSelection,
1233 }
1234
1235 impl Step for Rustfmt {
1236     type Output = Option<GeneratedTarball>;
1237     const DEFAULT: bool = true;
1238     const ONLY_HOSTS: bool = true;
1239
1240     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1241         let default = should_build_extended_tool(&run.builder, "rustfmt");
1242         run.path("rustfmt").default_condition(default)
1243     }
1244
1245     fn make_run(run: RunConfig<'_>) {
1246         run.builder.ensure(Rustfmt {
1247             compiler: run.builder.compiler_for(
1248                 run.builder.top_stage,
1249                 run.builder.config.build,
1250                 run.target,
1251             ),
1252             target: run.target,
1253         });
1254     }
1255
1256     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1257         let compiler = self.compiler;
1258         let target = self.target;
1259
1260         let rustfmt = builder
1261             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1262             .or_else(|| {
1263                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1264                 None
1265             })?;
1266         let cargofmt = builder
1267             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1268             .or_else(|| {
1269                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1270                 None
1271             })?;
1272
1273         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1274         tarball.set_overlay(OverlayKind::Rustfmt);
1275         tarball.is_preview(true);
1276         tarball.add_file(rustfmt, "bin", 0o755);
1277         tarball.add_file(cargofmt, "bin", 0o755);
1278         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1279         Some(tarball.generate())
1280     }
1281 }
1282
1283 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1284 pub struct RustDemangler {
1285     pub compiler: Compiler,
1286     pub target: TargetSelection,
1287 }
1288
1289 impl Step for RustDemangler {
1290     type Output = Option<GeneratedTarball>;
1291     const DEFAULT: bool = true;
1292     const ONLY_HOSTS: bool = true;
1293
1294     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1295         // While other tools use `should_build_extended_tool` to decide whether to be run by
1296         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1297         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1298         // we run the step by default when only `extended = true`, and decide whether to actually
1299         // run it or not later.
1300         let default = run.builder.config.extended;
1301         run.path("rust-demangler").default_condition(default)
1302     }
1303
1304     fn make_run(run: RunConfig<'_>) {
1305         run.builder.ensure(RustDemangler {
1306             compiler: run.builder.compiler_for(
1307                 run.builder.top_stage,
1308                 run.builder.config.build,
1309                 run.target,
1310             ),
1311             target: run.target,
1312         });
1313     }
1314
1315     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1316         let compiler = self.compiler;
1317         let target = self.target;
1318
1319         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1320         let condition = should_build_extended_tool(builder, "rust-demangler")
1321             || builder.config.profiler_enabled(target);
1322         if builder.config.extended && !condition {
1323             return None;
1324         }
1325
1326         let rust_demangler = builder
1327             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1328             .expect("rust-demangler expected to build - in-tree tool");
1329
1330         // Prepare the image directory
1331         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1332         tarball.set_overlay(OverlayKind::RustDemangler);
1333         tarball.is_preview(true);
1334         tarball.add_file(&rust_demangler, "bin", 0o755);
1335         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1336         Some(tarball.generate())
1337     }
1338 }
1339
1340 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1341 pub struct Extended {
1342     stage: u32,
1343     host: TargetSelection,
1344     target: TargetSelection,
1345 }
1346
1347 impl Step for Extended {
1348     type Output = ();
1349     const DEFAULT: bool = true;
1350     const ONLY_HOSTS: bool = true;
1351
1352     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1353         let builder = run.builder;
1354         run.path("extended").default_condition(builder.config.extended)
1355     }
1356
1357     fn make_run(run: RunConfig<'_>) {
1358         run.builder.ensure(Extended {
1359             stage: run.builder.top_stage,
1360             host: run.builder.config.build,
1361             target: run.target,
1362         });
1363     }
1364
1365     /// Creates a combined installer for the specified target in the provided stage.
1366     fn run(self, builder: &Builder<'_>) {
1367         let target = self.target;
1368         let stage = self.stage;
1369         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1370
1371         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1372
1373         let mut tarballs = Vec::new();
1374         let mut built_tools = HashSet::new();
1375         macro_rules! add_component {
1376             ($name:expr => $step:expr) => {
1377                 if let Some(tarball) = builder.ensure_if_default($step) {
1378                     tarballs.push(tarball);
1379                     built_tools.insert($name);
1380                 }
1381             };
1382         }
1383
1384         // When rust-std package split from rustc, we needed to ensure that during
1385         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1386         // the std files during uninstall. To do this ensure that rustc comes
1387         // before rust-std in the list below.
1388         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1389         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1390
1391         if target.contains("windows-gnu") {
1392             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1393         }
1394
1395         add_component!("rust-docs" => Docs { host: target });
1396         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1397         add_component!("cargo" => Cargo { compiler, target });
1398         add_component!("rustfmt" => Rustfmt { compiler, target });
1399         add_component!("rls" => Rls { compiler, target });
1400         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1401         add_component!("llvm-components" => LlvmTools { target });
1402         add_component!("clippy" => Clippy { compiler, target });
1403         add_component!("miri" => Miri { compiler, target });
1404         add_component!("analysis" => Analysis { compiler, target });
1405
1406         let etc = builder.src.join("src/etc/installer");
1407
1408         // Avoid producing tarballs during a dry run.
1409         if builder.config.dry_run {
1410             return;
1411         }
1412
1413         let tarball = Tarball::new(builder, "rust", &target.triple);
1414         let generated = tarball.combine(&tarballs);
1415
1416         let tmp = tmpdir(builder).join("combined-tarball");
1417         let work = generated.work_dir();
1418
1419         let mut license = String::new();
1420         license += &builder.read(&builder.src.join("COPYRIGHT"));
1421         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1422         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1423         license.push('\n');
1424         license.push('\n');
1425
1426         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1427         let mut rtf = rtf.to_string();
1428         rtf.push('\n');
1429         for line in license.lines() {
1430             rtf.push_str(line);
1431             rtf.push_str("\\line ");
1432         }
1433         rtf.push('}');
1434
1435         fn filter(contents: &str, marker: &str) -> String {
1436             let start = format!("tool-{}-start", marker);
1437             let end = format!("tool-{}-end", marker);
1438             let mut lines = Vec::new();
1439             let mut omitted = false;
1440             for line in contents.lines() {
1441                 if line.contains(&start) {
1442                     omitted = true;
1443                 } else if line.contains(&end) {
1444                     omitted = false;
1445                 } else if !omitted {
1446                     lines.push(line);
1447                 }
1448             }
1449
1450             lines.join("\n")
1451         }
1452
1453         let xform = |p: &Path| {
1454             let mut contents = t!(fs::read_to_string(p));
1455             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1456                 if !built_tools.contains(tool) {
1457                     contents = filter(&contents, tool);
1458                 }
1459             }
1460             let ret = tmp.join(p.file_name().unwrap());
1461             t!(fs::write(&ret, &contents));
1462             ret
1463         };
1464
1465         if target.contains("apple-darwin") {
1466             builder.info("building pkg installer");
1467             let pkg = tmp.join("pkg");
1468             let _ = fs::remove_dir_all(&pkg);
1469
1470             let pkgbuild = |component: &str| {
1471                 let mut cmd = Command::new("pkgbuild");
1472                 cmd.arg("--identifier")
1473                     .arg(format!("org.rust-lang.{}", component))
1474                     .arg("--scripts")
1475                     .arg(pkg.join(component))
1476                     .arg("--nopayload")
1477                     .arg(pkg.join(component).with_extension("pkg"));
1478                 builder.run(&mut cmd);
1479             };
1480
1481             let prepare = |name: &str| {
1482                 builder.create_dir(&pkg.join(name));
1483                 builder.cp_r(
1484                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1485                     &pkg.join(name),
1486                 );
1487                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1488                 pkgbuild(name);
1489             };
1490             prepare("rustc");
1491             prepare("cargo");
1492             prepare("rust-docs");
1493             prepare("rust-std");
1494             prepare("rust-analysis");
1495             prepare("clippy");
1496             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1497                 if built_tools.contains(tool) {
1498                     prepare(tool);
1499                 }
1500             }
1501             // create an 'uninstall' package
1502             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1503             pkgbuild("uninstall");
1504
1505             builder.create_dir(&pkg.join("res"));
1506             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1507             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1508             let mut cmd = Command::new("productbuild");
1509             cmd.arg("--distribution")
1510                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1511                 .arg("--resources")
1512                 .arg(pkg.join("res"))
1513                 .arg(distdir(builder).join(format!(
1514                     "{}-{}.pkg",
1515                     pkgname(builder, "rust"),
1516                     target.triple
1517                 )))
1518                 .arg("--package-path")
1519                 .arg(&pkg);
1520             let _time = timeit(builder);
1521             builder.run(&mut cmd);
1522         }
1523
1524         if target.contains("windows") {
1525             let exe = tmp.join("exe");
1526             let _ = fs::remove_dir_all(&exe);
1527
1528             let prepare = |name: &str| {
1529                 builder.create_dir(&exe.join(name));
1530                 let dir = if name == "rust-std" || name == "rust-analysis" {
1531                     format!("{}-{}", name, target.triple)
1532                 } else if name == "rls" {
1533                     "rls-preview".to_string()
1534                 } else if name == "rust-analyzer" {
1535                     "rust-analyzer-preview".to_string()
1536                 } else if name == "clippy" {
1537                     "clippy-preview".to_string()
1538                 } else if name == "rust-demangler" {
1539                     "rust-demangler-preview".to_string()
1540                 } else if name == "miri" {
1541                     "miri-preview".to_string()
1542                 } else {
1543                     name.to_string()
1544                 };
1545                 builder.cp_r(
1546                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1547                     &exe.join(name),
1548                 );
1549                 builder.remove(&exe.join(name).join("manifest.in"));
1550             };
1551             prepare("rustc");
1552             prepare("cargo");
1553             prepare("rust-analysis");
1554             prepare("rust-docs");
1555             prepare("rust-std");
1556             prepare("clippy");
1557             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1558                 if built_tools.contains(tool) {
1559                     prepare(tool);
1560                 }
1561             }
1562             if target.contains("windows-gnu") {
1563                 prepare("rust-mingw");
1564             }
1565
1566             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1567
1568             // Generate msi installer
1569             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1570             let heat = wix.join("bin/heat.exe");
1571             let candle = wix.join("bin/candle.exe");
1572             let light = wix.join("bin/light.exe");
1573
1574             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1575             builder.run(
1576                 Command::new(&heat)
1577                     .current_dir(&exe)
1578                     .arg("dir")
1579                     .arg("rustc")
1580                     .args(&heat_flags)
1581                     .arg("-cg")
1582                     .arg("RustcGroup")
1583                     .arg("-dr")
1584                     .arg("Rustc")
1585                     .arg("-var")
1586                     .arg("var.RustcDir")
1587                     .arg("-out")
1588                     .arg(exe.join("RustcGroup.wxs")),
1589             );
1590             builder.run(
1591                 Command::new(&heat)
1592                     .current_dir(&exe)
1593                     .arg("dir")
1594                     .arg("rust-docs")
1595                     .args(&heat_flags)
1596                     .arg("-cg")
1597                     .arg("DocsGroup")
1598                     .arg("-dr")
1599                     .arg("Docs")
1600                     .arg("-var")
1601                     .arg("var.DocsDir")
1602                     .arg("-out")
1603                     .arg(exe.join("DocsGroup.wxs"))
1604                     .arg("-t")
1605                     .arg(etc.join("msi/squash-components.xsl")),
1606             );
1607             builder.run(
1608                 Command::new(&heat)
1609                     .current_dir(&exe)
1610                     .arg("dir")
1611                     .arg("cargo")
1612                     .args(&heat_flags)
1613                     .arg("-cg")
1614                     .arg("CargoGroup")
1615                     .arg("-dr")
1616                     .arg("Cargo")
1617                     .arg("-var")
1618                     .arg("var.CargoDir")
1619                     .arg("-out")
1620                     .arg(exe.join("CargoGroup.wxs"))
1621                     .arg("-t")
1622                     .arg(etc.join("msi/remove-duplicates.xsl")),
1623             );
1624             builder.run(
1625                 Command::new(&heat)
1626                     .current_dir(&exe)
1627                     .arg("dir")
1628                     .arg("rust-std")
1629                     .args(&heat_flags)
1630                     .arg("-cg")
1631                     .arg("StdGroup")
1632                     .arg("-dr")
1633                     .arg("Std")
1634                     .arg("-var")
1635                     .arg("var.StdDir")
1636                     .arg("-out")
1637                     .arg(exe.join("StdGroup.wxs")),
1638             );
1639             if built_tools.contains("rls") {
1640                 builder.run(
1641                     Command::new(&heat)
1642                         .current_dir(&exe)
1643                         .arg("dir")
1644                         .arg("rls")
1645                         .args(&heat_flags)
1646                         .arg("-cg")
1647                         .arg("RlsGroup")
1648                         .arg("-dr")
1649                         .arg("Rls")
1650                         .arg("-var")
1651                         .arg("var.RlsDir")
1652                         .arg("-out")
1653                         .arg(exe.join("RlsGroup.wxs"))
1654                         .arg("-t")
1655                         .arg(etc.join("msi/remove-duplicates.xsl")),
1656                 );
1657             }
1658             if built_tools.contains("rust-analyzer") {
1659                 builder.run(
1660                     Command::new(&heat)
1661                         .current_dir(&exe)
1662                         .arg("dir")
1663                         .arg("rust-analyzer")
1664                         .args(&heat_flags)
1665                         .arg("-cg")
1666                         .arg("RustAnalyzerGroup")
1667                         .arg("-dr")
1668                         .arg("RustAnalyzer")
1669                         .arg("-var")
1670                         .arg("var.RustAnalyzerDir")
1671                         .arg("-out")
1672                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1673                         .arg("-t")
1674                         .arg(etc.join("msi/remove-duplicates.xsl")),
1675                 );
1676             }
1677             builder.run(
1678                 Command::new(&heat)
1679                     .current_dir(&exe)
1680                     .arg("dir")
1681                     .arg("clippy")
1682                     .args(&heat_flags)
1683                     .arg("-cg")
1684                     .arg("ClippyGroup")
1685                     .arg("-dr")
1686                     .arg("Clippy")
1687                     .arg("-var")
1688                     .arg("var.ClippyDir")
1689                     .arg("-out")
1690                     .arg(exe.join("ClippyGroup.wxs"))
1691                     .arg("-t")
1692                     .arg(etc.join("msi/remove-duplicates.xsl")),
1693             );
1694             if built_tools.contains("rust-demangler") {
1695                 builder.run(
1696                     Command::new(&heat)
1697                         .current_dir(&exe)
1698                         .arg("dir")
1699                         .arg("rust-demangler")
1700                         .args(&heat_flags)
1701                         .arg("-cg")
1702                         .arg("RustDemanglerGroup")
1703                         .arg("-dr")
1704                         .arg("RustDemangler")
1705                         .arg("-var")
1706                         .arg("var.RustDemanglerDir")
1707                         .arg("-out")
1708                         .arg(exe.join("RustDemanglerGroup.wxs"))
1709                         .arg("-t")
1710                         .arg(etc.join("msi/remove-duplicates.xsl")),
1711                 );
1712             }
1713             if built_tools.contains("miri") {
1714                 builder.run(
1715                     Command::new(&heat)
1716                         .current_dir(&exe)
1717                         .arg("dir")
1718                         .arg("miri")
1719                         .args(&heat_flags)
1720                         .arg("-cg")
1721                         .arg("MiriGroup")
1722                         .arg("-dr")
1723                         .arg("Miri")
1724                         .arg("-var")
1725                         .arg("var.MiriDir")
1726                         .arg("-out")
1727                         .arg(exe.join("MiriGroup.wxs"))
1728                         .arg("-t")
1729                         .arg(etc.join("msi/remove-duplicates.xsl")),
1730                 );
1731             }
1732             builder.run(
1733                 Command::new(&heat)
1734                     .current_dir(&exe)
1735                     .arg("dir")
1736                     .arg("rust-analysis")
1737                     .args(&heat_flags)
1738                     .arg("-cg")
1739                     .arg("AnalysisGroup")
1740                     .arg("-dr")
1741                     .arg("Analysis")
1742                     .arg("-var")
1743                     .arg("var.AnalysisDir")
1744                     .arg("-out")
1745                     .arg(exe.join("AnalysisGroup.wxs"))
1746                     .arg("-t")
1747                     .arg(etc.join("msi/remove-duplicates.xsl")),
1748             );
1749             if target.contains("windows-gnu") {
1750                 builder.run(
1751                     Command::new(&heat)
1752                         .current_dir(&exe)
1753                         .arg("dir")
1754                         .arg("rust-mingw")
1755                         .args(&heat_flags)
1756                         .arg("-cg")
1757                         .arg("GccGroup")
1758                         .arg("-dr")
1759                         .arg("Gcc")
1760                         .arg("-var")
1761                         .arg("var.GccDir")
1762                         .arg("-out")
1763                         .arg(exe.join("GccGroup.wxs")),
1764                 );
1765             }
1766
1767             let candle = |input: &Path| {
1768                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1769                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1770                 let mut cmd = Command::new(&candle);
1771                 cmd.current_dir(&exe)
1772                     .arg("-nologo")
1773                     .arg("-dRustcDir=rustc")
1774                     .arg("-dDocsDir=rust-docs")
1775                     .arg("-dCargoDir=cargo")
1776                     .arg("-dStdDir=rust-std")
1777                     .arg("-dAnalysisDir=rust-analysis")
1778                     .arg("-dClippyDir=clippy")
1779                     .arg("-arch")
1780                     .arg(&arch)
1781                     .arg("-out")
1782                     .arg(&output)
1783                     .arg(&input);
1784                 add_env(builder, &mut cmd, target);
1785
1786                 if built_tools.contains("rust-demangler") {
1787                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1788                 }
1789                 if built_tools.contains("rls") {
1790                     cmd.arg("-dRlsDir=rls");
1791                 }
1792                 if built_tools.contains("rust-analyzer") {
1793                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1794                 }
1795                 if built_tools.contains("miri") {
1796                     cmd.arg("-dMiriDir=miri");
1797                 }
1798                 if target.contains("windows-gnu") {
1799                     cmd.arg("-dGccDir=rust-mingw");
1800                 }
1801                 builder.run(&mut cmd);
1802             };
1803             candle(&xform(&etc.join("msi/rust.wxs")));
1804             candle(&etc.join("msi/ui.wxs"));
1805             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1806             candle("RustcGroup.wxs".as_ref());
1807             candle("DocsGroup.wxs".as_ref());
1808             candle("CargoGroup.wxs".as_ref());
1809             candle("StdGroup.wxs".as_ref());
1810             candle("ClippyGroup.wxs".as_ref());
1811             if built_tools.contains("rust-demangler") {
1812                 candle("RustDemanglerGroup.wxs".as_ref());
1813             }
1814             if built_tools.contains("rls") {
1815                 candle("RlsGroup.wxs".as_ref());
1816             }
1817             if built_tools.contains("rust-analyzer") {
1818                 candle("RustAnalyzerGroup.wxs".as_ref());
1819             }
1820             if built_tools.contains("miri") {
1821                 candle("MiriGroup.wxs".as_ref());
1822             }
1823             candle("AnalysisGroup.wxs".as_ref());
1824
1825             if target.contains("windows-gnu") {
1826                 candle("GccGroup.wxs".as_ref());
1827             }
1828
1829             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1830             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1831             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1832
1833             builder.info(&format!("building `msi` installer with {:?}", light));
1834             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1835             let mut cmd = Command::new(&light);
1836             cmd.arg("-nologo")
1837                 .arg("-ext")
1838                 .arg("WixUIExtension")
1839                 .arg("-ext")
1840                 .arg("WixUtilExtension")
1841                 .arg("-out")
1842                 .arg(exe.join(&filename))
1843                 .arg("rust.wixobj")
1844                 .arg("ui.wixobj")
1845                 .arg("rustwelcomedlg.wixobj")
1846                 .arg("RustcGroup.wixobj")
1847                 .arg("DocsGroup.wixobj")
1848                 .arg("CargoGroup.wixobj")
1849                 .arg("StdGroup.wixobj")
1850                 .arg("AnalysisGroup.wixobj")
1851                 .arg("ClippyGroup.wixobj")
1852                 .current_dir(&exe);
1853
1854             if built_tools.contains("rls") {
1855                 cmd.arg("RlsGroup.wixobj");
1856             }
1857             if built_tools.contains("rust-analyzer") {
1858                 cmd.arg("RustAnalyzerGroup.wixobj");
1859             }
1860             if built_tools.contains("rust-demangler") {
1861                 cmd.arg("RustDemanglerGroup.wixobj");
1862             }
1863             if built_tools.contains("miri") {
1864                 cmd.arg("MiriGroup.wixobj");
1865             }
1866
1867             if target.contains("windows-gnu") {
1868                 cmd.arg("GccGroup.wixobj");
1869             }
1870             // ICE57 wrongly complains about the shortcuts
1871             cmd.arg("-sice:ICE57");
1872
1873             let _time = timeit(builder);
1874             builder.run(&mut cmd);
1875
1876             if !builder.config.dry_run {
1877                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1878             }
1879         }
1880     }
1881 }
1882
1883 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1884     let mut parts = builder.version.split('.');
1885     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1886         .env("CFG_RELEASE_NUM", &builder.version)
1887         .env("CFG_RELEASE", builder.rust_release())
1888         .env("CFG_VER_MAJOR", parts.next().unwrap())
1889         .env("CFG_VER_MINOR", parts.next().unwrap())
1890         .env("CFG_VER_PATCH", parts.next().unwrap())
1891         .env("CFG_VER_BUILD", "0") // just needed to build
1892         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1893         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1894         .env("CFG_BUILD", target.triple)
1895         .env("CFG_CHANNEL", &builder.config.channel);
1896
1897     if target.contains("windows-gnu") {
1898         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1899     } else {
1900         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1901     }
1902
1903     if target.contains("x86_64") {
1904         cmd.env("CFG_PLATFORM", "x64");
1905     } else {
1906         cmd.env("CFG_PLATFORM", "x86");
1907     }
1908 }
1909
1910 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1911 ///
1912
1913 /// Returns whether the files were actually copied.
1914 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1915     if let Some(config) = builder.config.target_config.get(&target) {
1916         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1917             // If the LLVM was externally provided, then we don't currently copy
1918             // artifacts into the sysroot. This is not necessarily the right
1919             // choice (in particular, it will require the LLVM dylib to be in
1920             // the linker's load path at runtime), but the common use case for
1921             // external LLVMs is distribution provided LLVMs, and in that case
1922             // they're usually in the standard search path (e.g., /usr/lib) and
1923             // copying them here is going to cause problems as we may end up
1924             // with the wrong files and isn't what distributions want.
1925             //
1926             // This behavior may be revisited in the future though.
1927             //
1928             // If the LLVM is coming from ourselves (just from CI) though, we
1929             // still want to install it, as it otherwise won't be available.
1930             return false;
1931         }
1932     }
1933
1934     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1935     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1936     // clear why this is the case, though. llvm-config will emit the versioned
1937     // paths and we don't want those in the sysroot (as we're expecting
1938     // unversioned paths).
1939     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1940         let src_libdir = builder.llvm_out(target).join("lib");
1941         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1942         if llvm_dylib_path.exists() {
1943             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1944         }
1945         !builder.config.dry_run
1946     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1947         let mut cmd = Command::new(llvm_config);
1948         cmd.arg("--libfiles");
1949         builder.verbose(&format!("running {:?}", cmd));
1950         let files = output(&mut cmd);
1951         let build_llvm_out = &builder.llvm_out(builder.config.build);
1952         let target_llvm_out = &builder.llvm_out(target);
1953         for file in files.trim_end().split(' ') {
1954             // If we're not using a custom LLVM, make sure we package for the target.
1955             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1956                 target_llvm_out.join(relative_path)
1957             } else {
1958                 PathBuf::from(file)
1959             };
1960             builder.install(&file, dst_libdir, 0o644);
1961         }
1962         !builder.config.dry_run
1963     } else {
1964         false
1965     }
1966 }
1967
1968 /// Maybe add libLLVM.so to the target lib-dir for linking.
1969 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1970     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1971     // We do not need to copy LLVM files into the sysroot if it is not
1972     // dynamically linked; it is already included into librustc_llvm
1973     // statically.
1974     if builder.config.llvm_link_shared {
1975         maybe_install_llvm(builder, target, &dst_libdir);
1976     }
1977 }
1978
1979 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1980 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1981     let dst_libdir =
1982         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1983     // We do not need to copy LLVM files into the sysroot if it is not
1984     // dynamically linked; it is already included into librustc_llvm
1985     // statically.
1986     if builder.config.llvm_link_shared {
1987         maybe_install_llvm(builder, target, &dst_libdir);
1988     }
1989 }
1990
1991 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1992 pub struct LlvmTools {
1993     pub target: TargetSelection,
1994 }
1995
1996 impl Step for LlvmTools {
1997     type Output = Option<GeneratedTarball>;
1998     const ONLY_HOSTS: bool = true;
1999     const DEFAULT: bool = true;
2000
2001     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2002         let default = should_build_extended_tool(&run.builder, "llvm-tools");
2003         run.path("llvm-tools").default_condition(default)
2004     }
2005
2006     fn make_run(run: RunConfig<'_>) {
2007         run.builder.ensure(LlvmTools { target: run.target });
2008     }
2009
2010     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2011         let target = self.target;
2012
2013         /* run only if llvm-config isn't used */
2014         if let Some(config) = builder.config.target_config.get(&target) {
2015             if let Some(ref _s) = config.llvm_config {
2016                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2017                 return None;
2018             }
2019         }
2020
2021         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2022         tarball.set_overlay(OverlayKind::LLVM);
2023         tarball.is_preview(true);
2024
2025         // Prepare the image directory
2026         let src_bindir = builder.llvm_out(target).join("bin");
2027         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2028         for tool in LLVM_TOOLS {
2029             let exe = src_bindir.join(exe(tool, target));
2030             tarball.add_file(&exe, &dst_bindir, 0o755);
2031         }
2032
2033         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2034         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2035         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2036         // compiler libraries.
2037         maybe_install_llvm_target(builder, target, tarball.image_dir());
2038
2039         Some(tarball.generate())
2040     }
2041 }
2042
2043 // Tarball intended for internal consumption to ease rustc/std development.
2044 //
2045 // Should not be considered stable by end users.
2046 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2047 pub struct RustDev {
2048     pub target: TargetSelection,
2049 }
2050
2051 impl Step for RustDev {
2052     type Output = Option<GeneratedTarball>;
2053     const DEFAULT: bool = true;
2054     const ONLY_HOSTS: bool = true;
2055
2056     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2057         run.path("rust-dev")
2058     }
2059
2060     fn make_run(run: RunConfig<'_>) {
2061         run.builder.ensure(RustDev { target: run.target });
2062     }
2063
2064     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2065         let target = self.target;
2066
2067         /* run only if llvm-config isn't used */
2068         if let Some(config) = builder.config.target_config.get(&target) {
2069             if let Some(ref _s) = config.llvm_config {
2070                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2071                 return None;
2072             }
2073         }
2074
2075         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2076         tarball.set_overlay(OverlayKind::LLVM);
2077
2078         let src_bindir = builder.llvm_out(target).join("bin");
2079         for bin in &[
2080             "llvm-config",
2081             "llvm-ar",
2082             "llvm-objdump",
2083             "llvm-profdata",
2084             "llvm-bcanalyzer",
2085             "llvm-cov",
2086             "llvm-dwp",
2087         ] {
2088             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2089         }
2090         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2091
2092         // Copy the include directory as well; needed mostly to build
2093         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2094         // just broadly useful to be able to link against the bundled LLVM.
2095         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2096
2097         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2098         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2099         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2100         // compiler libraries.
2101         let dst_libdir = tarball.image_dir().join("lib");
2102         maybe_install_llvm(builder, target, &dst_libdir);
2103         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2104         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2105
2106         Some(tarball.generate())
2107     }
2108 }
2109
2110 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2111 /// release process to avoid cloning the monorepo and building stuff.
2112 ///
2113 /// Should not be considered stable by end users.
2114 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2115 pub struct BuildManifest {
2116     pub target: TargetSelection,
2117 }
2118
2119 impl Step for BuildManifest {
2120     type Output = GeneratedTarball;
2121     const DEFAULT: bool = false;
2122     const ONLY_HOSTS: bool = true;
2123
2124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2125         run.path("src/tools/build-manifest")
2126     }
2127
2128     fn make_run(run: RunConfig<'_>) {
2129         run.builder.ensure(BuildManifest { target: run.target });
2130     }
2131
2132     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2133         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2134
2135         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2136         tarball.add_file(&build_manifest, "bin", 0o755);
2137         tarball.generate()
2138     }
2139 }
2140
2141 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2142 ///
2143 /// Currently this is the PGO profile data.
2144 ///
2145 /// Should not be considered stable by end users.
2146 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2147 pub struct ReproducibleArtifacts {
2148     pub target: TargetSelection,
2149 }
2150
2151 impl Step for ReproducibleArtifacts {
2152     type Output = Option<GeneratedTarball>;
2153     const DEFAULT: bool = true;
2154     const ONLY_HOSTS: bool = true;
2155
2156     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2157         run.path("reproducible")
2158     }
2159
2160     fn make_run(run: RunConfig<'_>) {
2161         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2162     }
2163
2164     fn run(self, builder: &Builder<'_>) -> Self::Output {
2165         let mut added_anything = false;
2166         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2167         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2168             tarball.add_file(path, ".", 0o644);
2169             added_anything = true;
2170         }
2171         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2172             tarball.add_file(path, ".", 0o644);
2173             added_anything = true;
2174         }
2175         if added_anything { Some(tarball.generate()) } else { None }
2176     }
2177 }