src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::ffi::OsStr;
  14 use std::fs;
  15 use std::path::{Path, PathBuf};
  16 use std::process::Command;
  17
  18 use object::read::archive::ArchiveFile;
  19 use object::BinaryFormat;
  20
  21 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  22 use crate::cache::{Interned, INTERNER};
  23 use crate::channel;
  24 use crate::compile;
  25 use crate::config::TargetSelection;
  26 use crate::doc::DocumentationFormat;
  27 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  28 use crate::tool::{self, Tool};
  29 use crate::util::{exe, is_dylib, output, t, timeit};
  30 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  31
  32 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  33     format!("{}-{}", component, builder.rust_package_vers())
  34 }
  35
  36 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("dist")
  38 }
  39
  40 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  41     builder.out.join("tmp/dist")
  42 }
  43
  44 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  45     if !builder.config.extended {
  46         return false;
  47     }
  48     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  49 }
  50
  51 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  52 pub struct Docs {
  53     pub host: TargetSelection,
  54 }
  55
  56 impl Step for Docs {
  57     type Output = Option<GeneratedTarball>;
  58     const DEFAULT: bool = true;
  59
  60     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  61         let default = run.builder.config.docs;
  62         run.alias("rust-docs").default_condition(default)
  63     }
  64
  65     fn make_run(run: RunConfig<'_>) {
  66         run.builder.ensure(Docs { host: run.target });
  67     }
  68
  69     /// Builds the `rust-docs` installer component.
  70     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  71         let host = self.host;
  72         builder.default_doc(&[]);
  73
  74         let dest = "share/doc/rust/html";
  75
  76         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  77         tarball.set_product_name("Rust Documentation");
  78         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  79         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  80         Some(tarball.generate())
  81     }
  82 }
  83
  84 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  85 pub struct JsonDocs {
  86     pub host: TargetSelection,
  87 }
  88
  89 impl Step for JsonDocs {
  90     type Output = Option<GeneratedTarball>;
  91     const DEFAULT: bool = true;
  92
  93     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  94         let default = run.builder.config.docs;
  95         run.alias("rust-docs-json").default_condition(default)
  96     }
  97
  98     fn make_run(run: RunConfig<'_>) {
  99         run.builder.ensure(JsonDocs { host: run.target });
 100     }
 101
 102     /// Builds the `rust-docs-json` installer component.
 103     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 104         let host = self.host;
 105         builder.ensure(crate::doc::Std {
 106             stage: builder.top_stage,
 107             target: host,
 108             format: DocumentationFormat::JSON,
 109         });
 110
 111         let dest = "share/doc/rust/json";
 112
 113         let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple);
 114         tarball.set_product_name("Rust Documentation In JSON Format");
 115         tarball.is_preview(true);
 116         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 117         Some(tarball.generate())
 118     }
 119 }
 120
 121 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 122 pub struct RustcDocs {
 123     pub host: TargetSelection,
 124 }
 125
 126 impl Step for RustcDocs {
 127     type Output = Option<GeneratedTarball>;
 128     const DEFAULT: bool = true;
 129
 130     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 131         let builder = run.builder;
 132         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 133     }
 134
 135     fn make_run(run: RunConfig<'_>) {
 136         run.builder.ensure(RustcDocs { host: run.target });
 137     }
 138
 139     /// Builds the `rustc-docs` installer component.
 140     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 141         let host = self.host;
 142         builder.default_doc(&[]);
 143
 144         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 145         tarball.set_product_name("Rustc Documentation");
 146         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 147         Some(tarball.generate())
 148     }
 149 }
 150
 151 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 152     let mut found = Vec::with_capacity(files.len());
 153
 154     for file in files {
 155         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 156
 157         if let Some(file_path) = file_path {
 158             found.push(file_path);
 159         } else {
 160             panic!("Could not find '{}' in {:?}", file, path);
 161         }
 162     }
 163
 164     found
 165 }
 166
 167 fn make_win_dist(
 168     rust_root: &Path,
 169     plat_root: &Path,
 170     target: TargetSelection,
 171     builder: &Builder<'_>,
 172 ) {
 173     //Ask gcc where it keeps its stuff
 174     let mut cmd = Command::new(builder.cc(target));
 175     cmd.arg("-print-search-dirs");
 176     let gcc_out = output(&mut cmd);
 177
 178     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 179     let mut lib_path = Vec::new();
 180
 181     for line in gcc_out.lines() {
 182         let idx = line.find(':').unwrap();
 183         let key = &line[..idx];
 184         let trim_chars: &[_] = &[' ', '='];
 185         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 186
 187         if key == "programs" {
 188             bin_path.extend(value);
 189         } else if key == "libraries" {
 190             lib_path.extend(value);
 191         }
 192     }
 193
 194     let compiler = if target == "i686-pc-windows-gnu" {
 195         "i686-w64-mingw32-gcc.exe"
 196     } else if target == "x86_64-pc-windows-gnu" {
 197         "x86_64-w64-mingw32-gcc.exe"
 198     } else {
 199         "gcc.exe"
 200     };
 201     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 202     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 203     if target.starts_with("i686-") {
 204         rustc_dlls.push("libgcc_s_dw2-1.dll");
 205     } else {
 206         rustc_dlls.push("libgcc_s_seh-1.dll");
 207     }
 208
 209     let target_libs = [
 210         //MinGW libs
 211         "libgcc.a",
 212         "libgcc_eh.a",
 213         "libgcc_s.a",
 214         "libm.a",
 215         "libmingw32.a",
 216         "libmingwex.a",
 217         "libstdc++.a",
 218         "libiconv.a",
 219         "libmoldname.a",
 220         "libpthread.a",
 221         //Windows import libs
 222         "libadvapi32.a",
 223         "libbcrypt.a",
 224         "libcomctl32.a",
 225         "libcomdlg32.a",
 226         "libcredui.a",
 227         "libcrypt32.a",
 228         "libdbghelp.a",
 229         "libgdi32.a",
 230         "libimagehlp.a",
 231         "libiphlpapi.a",
 232         "libkernel32.a",
 233         "libmsimg32.a",
 234         "libmsvcrt.a",
 235         "libodbc32.a",
 236         "libole32.a",
 237         "liboleaut32.a",
 238         "libopengl32.a",
 239         "libpsapi.a",
 240         "librpcrt4.a",
 241         "libsecur32.a",
 242         "libsetupapi.a",
 243         "libshell32.a",
 244         "libsynchronization.a",
 245         "libuser32.a",
 246         "libuserenv.a",
 247         "libuuid.a",
 248         "libwinhttp.a",
 249         "libwinmm.a",
 250         "libwinspool.a",
 251         "libws2_32.a",
 252         "libwsock32.a",
 253     ];
 254
 255     //Find mingw artifacts we want to bundle
 256     let target_tools = find_files(&target_tools, &bin_path);
 257     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 258     let target_libs = find_files(&target_libs, &lib_path);
 259
 260     // Copy runtime dlls next to rustc.exe
 261     let dist_bin_dir = rust_root.join("bin/");
 262     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 263     for src in rustc_dlls {
 264         builder.copy_to_folder(&src, &dist_bin_dir);
 265     }
 266
 267     //Copy platform tools to platform-specific bin directory
 268     let target_bin_dir = plat_root
 269         .join("lib")
 270         .join("rustlib")
 271         .join(target.triple)
 272         .join("bin")
 273         .join("self-contained");
 274     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 275     for src in target_tools {
 276         builder.copy_to_folder(&src, &target_bin_dir);
 277     }
 278
 279     // Warn windows-gnu users that the bundled GCC cannot compile C files
 280     builder.create(
 281         &target_bin_dir.join("GCC-WARNING.txt"),
 282         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 283          used as a linker. In order to be able to compile projects containing C code use \
 284          the GCC provided by MinGW or Cygwin.",
 285     );
 286
 287     //Copy platform libs to platform-specific lib directory
 288     let target_lib_dir = plat_root
 289         .join("lib")
 290         .join("rustlib")
 291         .join(target.triple)
 292         .join("lib")
 293         .join("self-contained");
 294     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 295     for src in target_libs {
 296         builder.copy_to_folder(&src, &target_lib_dir);
 297     }
 298 }
 299
 300 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 301 pub struct Mingw {
 302     pub host: TargetSelection,
 303 }
 304
 305 impl Step for Mingw {
 306     type Output = Option<GeneratedTarball>;
 307     const DEFAULT: bool = true;
 308
 309     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 310         run.alias("rust-mingw")
 311     }
 312
 313     fn make_run(run: RunConfig<'_>) {
 314         run.builder.ensure(Mingw { host: run.target });
 315     }
 316
 317     /// Builds the `rust-mingw` installer component.
 318     ///
 319     /// This contains all the bits and pieces to run the MinGW Windows targets
 320     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 321     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 322         let host = self.host;
 323         if !host.ends_with("pc-windows-gnu") {
 324             return None;
 325         }
 326
 327         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 328         tarball.set_product_name("Rust MinGW");
 329
 330         // The first argument is a "temporary directory" which is just
 331         // thrown away (this contains the runtime DLLs included in the rustc package
 332         // above) and the second argument is where to place all the MinGW components
 333         // (which is what we want).
 334         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 335
 336         Some(tarball.generate())
 337     }
 338 }
 339
 340 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 341 pub struct Rustc {
 342     pub compiler: Compiler,
 343 }
 344
 345 impl Step for Rustc {
 346     type Output = GeneratedTarball;
 347     const DEFAULT: bool = true;
 348     const ONLY_HOSTS: bool = true;
 349
 350     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 351         run.alias("rustc")
 352     }
 353
 354     fn make_run(run: RunConfig<'_>) {
 355         run.builder
 356             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 357     }
 358
 359     /// Creates the `rustc` installer component.
 360     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 361         let compiler = self.compiler;
 362         let host = self.compiler.host;
 363
 364         let tarball = Tarball::new(builder, "rustc", &host.triple);
 365
 366         // Prepare the rustc "image", what will actually end up getting installed
 367         prepare_image(builder, compiler, tarball.image_dir());
 368
 369         // On MinGW we've got a few runtime DLL dependencies that we need to
 370         // include. The first argument to this script is where to put these DLLs
 371         // (the image we're creating), and the second argument is a junk directory
 372         // to ignore all other MinGW stuff the script creates.
 373         //
 374         // On 32-bit MinGW we're always including a DLL which needs some extra
 375         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 376         // anything requiring us to distribute a license, but it's likely the
 377         // install will *also* include the rust-mingw package, which also needs
 378         // licenses, so to be safe we just include it here in all MinGW packages.
 379         if host.ends_with("pc-windows-gnu") {
 380             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 381             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 382         }
 383
 384         return tarball.generate();
 385
 386         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 387             let host = compiler.host;
 388             let src = builder.sysroot(compiler);
 389
 390             // Copy rustc/rustdoc binaries
 391             t!(fs::create_dir_all(image.join("bin")));
 392             builder.cp_r(&src.join("bin"), &image.join("bin"));
 393
 394             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 395
 396             let ra_proc_macro_srv = builder
 397                 .ensure(tool::RustAnalyzerProcMacroSrv {
 398                     compiler: builder.compiler_for(
 399                         compiler.stage,
 400                         builder.config.build,
 401                         compiler.host,
 402                     ),
 403                     target: compiler.host,
 404                 })
 405                 .expect("rust-analyzer-proc-macro-server always builds");
 406             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 407
 408             let libdir_relative = builder.libdir_relative(compiler);
 409
 410             // Copy runtime DLLs needed by the compiler
 411             if libdir_relative.to_str() != Some("bin") {
 412                 let libdir = builder.rustc_libdir(compiler);
 413                 for entry in builder.read_dir(&libdir) {
 414                     let name = entry.file_name();
 415                     if let Some(s) = name.to_str() {
 416                         if is_dylib(s) {
 417                             // Don't use custom libdir here because ^lib/ will be resolved again
 418                             // with installer
 419                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 420                         }
 421                     }
 422                 }
 423             }
 424
 425             // Copy over the codegen backends
 426             let backends_src = builder.sysroot_codegen_backends(compiler);
 427             let backends_rel = backends_src
 428                 .strip_prefix(&src)
 429                 .unwrap()
 430                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 431                 .unwrap();
 432             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 433             let backends_dst = image.join("lib").join(&backends_rel);
 434
 435             t!(fs::create_dir_all(&backends_dst));
 436             builder.cp_r(&backends_src, &backends_dst);
 437
 438             // Copy libLLVM.so to the lib dir as well, if needed. While not
 439             // technically needed by rustc itself it's needed by lots of other
 440             // components like the llvm tools and LLD. LLD is included below and
 441             // tools/LLDB come later, so let's just throw it in the rustc
 442             // component for now.
 443             maybe_install_llvm_runtime(builder, host, image);
 444
 445             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 446             t!(fs::create_dir_all(&dst_dir));
 447
 448             // Copy over lld if it's there
 449             if builder.config.lld_enabled {
 450                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 451                 let rust_lld = exe("rust-lld", compiler.host);
 452                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 453                 // for `-Z gcc-ld=lld`
 454                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 455                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 456                 t!(fs::create_dir(&gcc_lld_dst_dir));
 457                 for name in crate::LLD_FILE_NAMES {
 458                     let exe_name = exe(name, compiler.host);
 459                     builder
 460                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 461                 }
 462             }
 463
 464             // Man pages
 465             t!(fs::create_dir_all(image.join("share/man/man1")));
 466             let man_src = builder.src.join("src/doc/man");
 467             let man_dst = image.join("share/man/man1");
 468
 469             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 470             // to hardlink, and we don't want to edit the source templates
 471             for file_entry in builder.read_dir(&man_src) {
 472                 let page_src = file_entry.path();
 473                 let page_dst = man_dst.join(file_entry.file_name());
 474                 let src_text = t!(std::fs::read_to_string(&page_src));
 475                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 476                 t!(std::fs::write(&page_dst, &new_text));
 477                 t!(fs::copy(&page_src, &page_dst));
 478             }
 479
 480             // Debugger scripts
 481             builder
 482                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 483
 484             // Misc license info
 485             let cp = |file: &str| {
 486                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 487             };
 488             cp("COPYRIGHT");
 489             cp("LICENSE-APACHE");
 490             cp("LICENSE-MIT");
 491             cp("README.md");
 492         }
 493     }
 494 }
 495
 496 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 497 pub struct DebuggerScripts {
 498     pub sysroot: Interned<PathBuf>,
 499     pub host: TargetSelection,
 500 }
 501
 502 impl Step for DebuggerScripts {
 503     type Output = ();
 504
 505     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 506         run.never()
 507     }
 508
 509     /// Copies debugger scripts for `target` into the `sysroot` specified.
 510     fn run(self, builder: &Builder<'_>) {
 511         let host = self.host;
 512         let sysroot = self.sysroot;
 513         let dst = sysroot.join("lib/rustlib/etc");
 514         t!(fs::create_dir_all(&dst));
 515         let cp_debugger_script = |file: &str| {
 516             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 517         };
 518         if host.contains("windows-msvc") {
 519             // windbg debugger scripts
 520             builder.install(
 521                 &builder.src.join("src/etc/rust-windbg.cmd"),
 522                 &sysroot.join("bin"),
 523                 0o755,
 524             );
 525
 526             cp_debugger_script("natvis/intrinsic.natvis");
 527             cp_debugger_script("natvis/liballoc.natvis");
 528             cp_debugger_script("natvis/libcore.natvis");
 529             cp_debugger_script("natvis/libstd.natvis");
 530         } else {
 531             cp_debugger_script("rust_types.py");
 532
 533             // gdb debugger scripts
 534             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 535             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 536
 537             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 538             cp_debugger_script("gdb_lookup.py");
 539             cp_debugger_script("gdb_providers.py");
 540
 541             // lldb debugger scripts
 542             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 543
 544             cp_debugger_script("lldb_lookup.py");
 545             cp_debugger_script("lldb_providers.py");
 546             cp_debugger_script("lldb_commands")
 547         }
 548     }
 549 }
 550
 551 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 552     // The only true set of target libraries came from the build triple, so
 553     // let's reduce redundant work by only producing archives from that host.
 554     if compiler.host != builder.config.build {
 555         builder.info("\tskipping, not a build host");
 556         true
 557     } else {
 558         false
 559     }
 560 }
 561
 562 /// Check that all objects in rlibs for UEFI targets are COFF. This
 563 /// ensures that the C compiler isn't producing ELF objects, which would
 564 /// not link correctly with the COFF objects.
 565 fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) {
 566     if !target.ends_with("-uefi") {
 567         return;
 568     }
 569
 570     for (path, _) in builder.read_stamp_file(stamp) {
 571         if path.extension() != Some(OsStr::new("rlib")) {
 572             continue;
 573         }
 574
 575         let data = t!(fs::read(&path));
 576         let data = data.as_slice();
 577         let archive = t!(ArchiveFile::parse(data));
 578         for member in archive.members() {
 579             let member = t!(member);
 580             let member_data = t!(member.data(data));
 581
 582             let is_coff = match object::File::parse(member_data) {
 583                 Ok(member_file) => member_file.format() == BinaryFormat::Coff,
 584                 Err(_) => false,
 585             };
 586
 587             if !is_coff {
 588                 let member_name = String::from_utf8_lossy(member.name());
 589                 panic!("member {} in {} is not COFF", member_name, path.display());
 590             }
 591         }
 592     }
 593 }
 594
 595 /// Copy stamped files into an image's `target/lib` directory.
 596 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 597     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 598     let self_contained_dst = dst.join("self-contained");
 599     t!(fs::create_dir_all(&dst));
 600     t!(fs::create_dir_all(&self_contained_dst));
 601     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 602         if dependency_type == DependencyType::TargetSelfContained {
 603             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 604         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 605             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 606         }
 607     }
 608 }
 609
 610 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 611 pub struct Std {
 612     pub compiler: Compiler,
 613     pub target: TargetSelection,
 614 }
 615
 616 impl Step for Std {
 617     type Output = Option<GeneratedTarball>;
 618     const DEFAULT: bool = true;
 619
 620     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 621         run.alias("rust-std")
 622     }
 623
 624     fn make_run(run: RunConfig<'_>) {
 625         run.builder.ensure(Std {
 626             compiler: run.builder.compiler_for(
 627                 run.builder.top_stage,
 628                 run.builder.config.build,
 629                 run.target,
 630             ),
 631             target: run.target,
 632         });
 633     }
 634
 635     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 636         let compiler = self.compiler;
 637         let target = self.target;
 638
 639         if skip_host_target_lib(builder, compiler) {
 640             return None;
 641         }
 642
 643         builder.ensure(compile::Std::new(compiler, target));
 644
 645         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 646         tarball.include_target_in_component_name(true);
 647
 648         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 649         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 650         verify_uefi_rlib_format(builder, target, &stamp);
 651         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 652
 653         Some(tarball.generate())
 654     }
 655 }
 656
 657 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 658 pub struct RustcDev {
 659     pub compiler: Compiler,
 660     pub target: TargetSelection,
 661 }
 662
 663 impl Step for RustcDev {
 664     type Output = Option<GeneratedTarball>;
 665     const DEFAULT: bool = true;
 666     const ONLY_HOSTS: bool = true;
 667
 668     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 669         run.alias("rustc-dev")
 670     }
 671
 672     fn make_run(run: RunConfig<'_>) {
 673         run.builder.ensure(RustcDev {
 674             compiler: run.builder.compiler_for(
 675                 run.builder.top_stage,
 676                 run.builder.config.build,
 677                 run.target,
 678             ),
 679             target: run.target,
 680         });
 681     }
 682
 683     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 684         let compiler = self.compiler;
 685         let target = self.target;
 686         if skip_host_target_lib(builder, compiler) {
 687             return None;
 688         }
 689
 690         builder.ensure(compile::Rustc::new(compiler, target));
 691
 692         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 693
 694         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 695         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 696         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 697
 698         let src_files = &["Cargo.lock"];
 699         // This is the reduced set of paths which will become the rustc-dev component
 700         // (essentially the compiler crates and all of their path dependencies).
 701         copy_src_dirs(
 702             builder,
 703             &builder.src,
 704             &["compiler"],
 705             &[],
 706             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 707         );
 708         for file in src_files {
 709             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 710         }
 711
 712         Some(tarball.generate())
 713     }
 714 }
 715
 716 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 717 pub struct Analysis {
 718     pub compiler: Compiler,
 719     pub target: TargetSelection,
 720 }
 721
 722 impl Step for Analysis {
 723     type Output = Option<GeneratedTarball>;
 724     const DEFAULT: bool = true;
 725
 726     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 727         let default = should_build_extended_tool(&run.builder, "analysis");
 728         run.alias("rust-analysis").default_condition(default)
 729     }
 730
 731     fn make_run(run: RunConfig<'_>) {
 732         run.builder.ensure(Analysis {
 733             // Find the actual compiler (handling the full bootstrap option) which
 734             // produced the save-analysis data because that data isn't copied
 735             // through the sysroot uplifting.
 736             compiler: run.builder.compiler_for(
 737                 run.builder.top_stage,
 738                 run.builder.config.build,
 739                 run.target,
 740             ),
 741             target: run.target,
 742         });
 743     }
 744
 745     /// Creates a tarball of save-analysis metadata, if available.
 746     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 747         let compiler = self.compiler;
 748         let target = self.target;
 749         if compiler.host != builder.config.build {
 750             return None;
 751         }
 752
 753         builder.ensure(compile::Std::new(compiler, target));
 754         let src = builder
 755             .stage_out(compiler, Mode::Std)
 756             .join(target.triple)
 757             .join(builder.cargo_dir())
 758             .join("deps")
 759             .join("save-analysis");
 760
 761         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 762         tarball.include_target_in_component_name(true);
 763         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 764         Some(tarball.generate())
 765     }
 766 }
 767
 768 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 769 /// `dst_dir`.
 770 fn copy_src_dirs(
 771     builder: &Builder<'_>,
 772     base: &Path,
 773     src_dirs: &[&str],
 774     exclude_dirs: &[&str],
 775     dst_dir: &Path,
 776 ) {
 777     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 778         let spath = match path.to_str() {
 779             Some(path) => path,
 780             None => return false,
 781         };
 782         if spath.ends_with('~') || spath.ends_with(".pyc") {
 783             return false;
 784         }
 785
 786         const LLVM_PROJECTS: &[&str] = &[
 787             "llvm-project/clang",
 788             "llvm-project\\clang",
 789             "llvm-project/libunwind",
 790             "llvm-project\\libunwind",
 791             "llvm-project/lld",
 792             "llvm-project\\lld",
 793             "llvm-project/lldb",
 794             "llvm-project\\lldb",
 795             "llvm-project/llvm",
 796             "llvm-project\\llvm",
 797             "llvm-project/compiler-rt",
 798             "llvm-project\\compiler-rt",
 799             "llvm-project/cmake",
 800             "llvm-project\\cmake",
 801         ];
 802         if spath.contains("llvm-project")
 803             && !spath.ends_with("llvm-project")
 804             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 805         {
 806             return false;
 807         }
 808
 809         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 810         if LLVM_TEST.iter().any(|path| spath.contains(path))
 811             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 812         {
 813             return false;
 814         }
 815
 816         let full_path = Path::new(dir).join(path);
 817         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 818             return false;
 819         }
 820
 821         let excludes = [
 822             "CVS",
 823             "RCS",
 824             "SCCS",
 825             ".git",
 826             ".gitignore",
 827             ".gitmodules",
 828             ".gitattributes",
 829             ".cvsignore",
 830             ".svn",
 831             ".arch-ids",
 832             "{arch}",
 833             "=RELEASE-ID",
 834             "=meta-update",
 835             "=update",
 836             ".bzr",
 837             ".bzrignore",
 838             ".bzrtags",
 839             ".hg",
 840             ".hgignore",
 841             ".hgrags",
 842             "_darcs",
 843         ];
 844         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 845     }
 846
 847     // Copy the directories using our filter
 848     for item in src_dirs {
 849         let dst = &dst_dir.join(item);
 850         t!(fs::create_dir_all(dst));
 851         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 852     }
 853 }
 854
 855 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 856 pub struct Src;
 857
 858 impl Step for Src {
 859     /// The output path of the src installer tarball
 860     type Output = GeneratedTarball;
 861     const DEFAULT: bool = true;
 862     const ONLY_HOSTS: bool = true;
 863
 864     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 865         run.alias("rust-src")
 866     }
 867
 868     fn make_run(run: RunConfig<'_>) {
 869         run.builder.ensure(Src);
 870     }
 871
 872     /// Creates the `rust-src` installer component
 873     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 874         let tarball = Tarball::new_targetless(builder, "rust-src");
 875
 876         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 877         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 878         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 879         // and fix them...
 880         //
 881         // NOTE: if you update the paths here, you also should update the "virtual" path
 882         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 883         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 884
 885         let src_files = ["Cargo.lock"];
 886         // This is the reduced set of paths which will become the rust-src component
 887         // (essentially libstd and all of its path dependencies).
 888         copy_src_dirs(
 889             builder,
 890             &builder.src,
 891             &["library", "src/llvm-project/libunwind"],
 892             &[
 893                 // not needed and contains symlinks which rustup currently
 894                 // chokes on when unpacking.
 895                 "library/backtrace/crates",
 896                 // these are 30MB combined and aren't necessary for building
 897                 // the standard library.
 898                 "library/stdarch/Cargo.toml",
 899                 "library/stdarch/crates/stdarch-verify",
 900                 "library/stdarch/crates/intrinsic-test",
 901             ],
 902             &dst_src,
 903         );
 904         for file in src_files.iter() {
 905             builder.copy(&builder.src.join(file), &dst_src.join(file));
 906         }
 907
 908         tarball.generate()
 909     }
 910 }
 911
 912 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 913 pub struct PlainSourceTarball;
 914
 915 impl Step for PlainSourceTarball {
 916     /// Produces the location of the tarball generated
 917     type Output = GeneratedTarball;
 918     const DEFAULT: bool = true;
 919     const ONLY_HOSTS: bool = true;
 920
 921     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 922         let builder = run.builder;
 923         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 924     }
 925
 926     fn make_run(run: RunConfig<'_>) {
 927         run.builder.ensure(PlainSourceTarball);
 928     }
 929
 930     /// Creates the plain source tarball
 931     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 932         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 933         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 934         // It also contains symbolic links, unlike other any other dist tarball.
 935         // It's used for distros building rustc from source in a pre-vendored environment.
 936         let mut tarball = Tarball::new(builder, "rustc", "src");
 937         tarball.permit_symlinks(true);
 938         let plain_dst_src = tarball.image_dir();
 939
 940         // This is the set of root paths which will become part of the source package
 941         let src_files = [
 942             "COPYRIGHT",
 943             "LICENSE-APACHE",
 944             "LICENSE-MIT",
 945             "CONTRIBUTING.md",
 946             "README.md",
 947             "RELEASES.md",
 948             "configure",
 949             "x.py",
 950             "config.toml.example",
 951             "Cargo.toml",
 952             "Cargo.lock",
 953         ];
 954         let src_dirs = ["src", "compiler", "library"];
 955
 956         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 957
 958         // Copy the files normally
 959         for item in &src_files {
 960             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 961         }
 962
 963         // Create the version file
 964         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 965         if let Some(info) = builder.rust_info().info() {
 966             channel::write_commit_hash_file(&plain_dst_src, &info.sha);
 967             channel::write_commit_info_file(&plain_dst_src, info);
 968         }
 969
 970         // If we're building from git sources, we need to vendor a complete distribution.
 971         if builder.rust_info().is_managed_git_subrepository() {
 972             // Ensure we have the submodules checked out.
 973             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 974
 975             // Vendor all Cargo dependencies
 976             let mut cmd = Command::new(&builder.initial_cargo);
 977             cmd.arg("vendor")
 978                 .arg("--sync")
 979                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 980                 .arg("--sync")
 981                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 982                 .arg("--sync")
 983                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 984                 .current_dir(&plain_dst_src);
 985
 986             let config = if !builder.config.dry_run() {
 987                 t!(String::from_utf8(t!(cmd.output()).stdout))
 988             } else {
 989                 String::new()
 990             };
 991
 992             let cargo_config_dir = plain_dst_src.join(".cargo");
 993             builder.create_dir(&cargo_config_dir);
 994             builder.create(&cargo_config_dir.join("config.toml"), &config);
 995         }
 996
 997         tarball.bare()
 998     }
 999 }
1000
1001 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1002 pub struct Cargo {
1003     pub compiler: Compiler,
1004     pub target: TargetSelection,
1005 }
1006
1007 impl Step for Cargo {
1008     type Output = Option<GeneratedTarball>;
1009     const DEFAULT: bool = true;
1010     const ONLY_HOSTS: bool = true;
1011
1012     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1013         let default = should_build_extended_tool(&run.builder, "cargo");
1014         run.alias("cargo").default_condition(default)
1015     }
1016
1017     fn make_run(run: RunConfig<'_>) {
1018         run.builder.ensure(Cargo {
1019             compiler: run.builder.compiler_for(
1020                 run.builder.top_stage,
1021                 run.builder.config.build,
1022                 run.target,
1023             ),
1024             target: run.target,
1025         });
1026     }
1027
1028     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1029         let compiler = self.compiler;
1030         let target = self.target;
1031
1032         let cargo = builder.ensure(tool::Cargo { compiler, target });
1033         let src = builder.src.join("src/tools/cargo");
1034         let etc = src.join("src/etc");
1035
1036         // Prepare the image directory
1037         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
1038         tarball.set_overlay(OverlayKind::Cargo);
1039
1040         tarball.add_file(&cargo, "bin", 0o755);
1041         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
1042         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1043         tarball.add_dir(etc.join("man"), "share/man/man1");
1044         tarball.add_legal_and_readme_to("share/doc/cargo");
1045
1046         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1047             let dirent = dirent.expect("read dir entry");
1048             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1049                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1050             }
1051         }
1052
1053         Some(tarball.generate())
1054     }
1055 }
1056
1057 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1058 pub struct Rls {
1059     pub compiler: Compiler,
1060     pub target: TargetSelection,
1061 }
1062
1063 impl Step for Rls {
1064     type Output = Option<GeneratedTarball>;
1065     const ONLY_HOSTS: bool = true;
1066     const DEFAULT: bool = true;
1067
1068     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1069         let default = should_build_extended_tool(&run.builder, "rls");
1070         run.alias("rls").default_condition(default)
1071     }
1072
1073     fn make_run(run: RunConfig<'_>) {
1074         run.builder.ensure(Rls {
1075             compiler: run.builder.compiler_for(
1076                 run.builder.top_stage,
1077                 run.builder.config.build,
1078                 run.target,
1079             ),
1080             target: run.target,
1081         });
1082     }
1083
1084     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1085         let compiler = self.compiler;
1086         let target = self.target;
1087
1088         let rls = builder
1089             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1090             .expect("rls expected to build");
1091
1092         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1093         tarball.set_overlay(OverlayKind::RLS);
1094         tarball.is_preview(true);
1095         tarball.add_file(rls, "bin", 0o755);
1096         tarball.add_legal_and_readme_to("share/doc/rls");
1097         Some(tarball.generate())
1098     }
1099 }
1100
1101 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1102 pub struct RustAnalyzer {
1103     pub compiler: Compiler,
1104     pub target: TargetSelection,
1105 }
1106
1107 impl Step for RustAnalyzer {
1108     type Output = Option<GeneratedTarball>;
1109     const DEFAULT: bool = true;
1110     const ONLY_HOSTS: bool = true;
1111
1112     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1113         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1114         run.alias("rust-analyzer").default_condition(default)
1115     }
1116
1117     fn make_run(run: RunConfig<'_>) {
1118         run.builder.ensure(RustAnalyzer {
1119             compiler: run.builder.compiler_for(
1120                 run.builder.top_stage,
1121                 run.builder.config.build,
1122                 run.target,
1123             ),
1124             target: run.target,
1125         });
1126     }
1127
1128     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1129         let compiler = self.compiler;
1130         let target = self.target;
1131
1132         if target.contains("riscv64") {
1133             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1134             // to build. See #74813 for details.
1135             return None;
1136         }
1137
1138         let rust_analyzer = builder
1139             .ensure(tool::RustAnalyzer { compiler, target })
1140             .expect("rust-analyzer always builds");
1141
1142         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1143         tarball.set_overlay(OverlayKind::RustAnalyzer);
1144         tarball.is_preview(true);
1145         tarball.add_file(rust_analyzer, "bin", 0o755);
1146         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1147         Some(tarball.generate())
1148     }
1149 }
1150
1151 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1152 pub struct Clippy {
1153     pub compiler: Compiler,
1154     pub target: TargetSelection,
1155 }
1156
1157 impl Step for Clippy {
1158     type Output = Option<GeneratedTarball>;
1159     const DEFAULT: bool = true;
1160     const ONLY_HOSTS: bool = true;
1161
1162     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1163         let default = should_build_extended_tool(&run.builder, "clippy");
1164         run.alias("clippy").default_condition(default)
1165     }
1166
1167     fn make_run(run: RunConfig<'_>) {
1168         run.builder.ensure(Clippy {
1169             compiler: run.builder.compiler_for(
1170                 run.builder.top_stage,
1171                 run.builder.config.build,
1172                 run.target,
1173             ),
1174             target: run.target,
1175         });
1176     }
1177
1178     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1179         let compiler = self.compiler;
1180         let target = self.target;
1181
1182         // Prepare the image directory
1183         // We expect clippy to build, because we've exited this step above if tool
1184         // state for clippy isn't testing.
1185         let clippy = builder
1186             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1187             .expect("clippy expected to build - essential tool");
1188         let cargoclippy = builder
1189             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1190             .expect("clippy expected to build - essential tool");
1191
1192         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1193         tarball.set_overlay(OverlayKind::Clippy);
1194         tarball.add_file(clippy, "bin", 0o755);
1195         tarball.add_file(cargoclippy, "bin", 0o755);
1196         tarball.add_legal_and_readme_to("share/doc/clippy");
1197         Some(tarball.generate())
1198     }
1199 }
1200
1201 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1202 pub struct Miri {
1203     pub compiler: Compiler,
1204     pub target: TargetSelection,
1205 }
1206
1207 impl Step for Miri {
1208     type Output = Option<GeneratedTarball>;
1209     const DEFAULT: bool = true;
1210     const ONLY_HOSTS: bool = true;
1211
1212     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1213         let default = should_build_extended_tool(&run.builder, "miri");
1214         run.alias("miri").default_condition(default)
1215     }
1216
1217     fn make_run(run: RunConfig<'_>) {
1218         run.builder.ensure(Miri {
1219             compiler: run.builder.compiler_for(
1220                 run.builder.top_stage,
1221                 run.builder.config.build,
1222                 run.target,
1223             ),
1224             target: run.target,
1225         });
1226     }
1227
1228     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1229         // This prevents miri from being built for "dist" or "install"
1230         // on the stable/beta channels. It is a nightly-only tool and should
1231         // not be included.
1232         if !builder.build.unstable_features() {
1233             return None;
1234         }
1235         let compiler = self.compiler;
1236         let target = self.target;
1237
1238         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1239         let cargomiri =
1240             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1241
1242         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1243         tarball.set_overlay(OverlayKind::Miri);
1244         tarball.is_preview(true);
1245         tarball.add_file(miri, "bin", 0o755);
1246         tarball.add_file(cargomiri, "bin", 0o755);
1247         tarball.add_legal_and_readme_to("share/doc/miri");
1248         Some(tarball.generate())
1249     }
1250 }
1251
1252 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1253 pub struct Rustfmt {
1254     pub compiler: Compiler,
1255     pub target: TargetSelection,
1256 }
1257
1258 impl Step for Rustfmt {
1259     type Output = Option<GeneratedTarball>;
1260     const DEFAULT: bool = true;
1261     const ONLY_HOSTS: bool = true;
1262
1263     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1264         let default = should_build_extended_tool(&run.builder, "rustfmt");
1265         run.alias("rustfmt").default_condition(default)
1266     }
1267
1268     fn make_run(run: RunConfig<'_>) {
1269         run.builder.ensure(Rustfmt {
1270             compiler: run.builder.compiler_for(
1271                 run.builder.top_stage,
1272                 run.builder.config.build,
1273                 run.target,
1274             ),
1275             target: run.target,
1276         });
1277     }
1278
1279     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1280         let compiler = self.compiler;
1281         let target = self.target;
1282
1283         let rustfmt = builder
1284             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1285             .expect("rustfmt expected to build - essential tool");
1286         let cargofmt = builder
1287             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1288             .expect("cargo fmt expected to build - essential tool");
1289         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1290         tarball.set_overlay(OverlayKind::Rustfmt);
1291         tarball.add_file(rustfmt, "bin", 0o755);
1292         tarball.add_file(cargofmt, "bin", 0o755);
1293         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1294         Some(tarball.generate())
1295     }
1296 }
1297
1298 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1299 pub struct RustDemangler {
1300     pub compiler: Compiler,
1301     pub target: TargetSelection,
1302 }
1303
1304 impl Step for RustDemangler {
1305     type Output = Option<GeneratedTarball>;
1306     const DEFAULT: bool = true;
1307     const ONLY_HOSTS: bool = true;
1308
1309     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1310         // While other tools use `should_build_extended_tool` to decide whether to be run by
1311         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1312         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1313         // we run the step by default when only `extended = true`, and decide whether to actually
1314         // run it or not later.
1315         let default = run.builder.config.extended;
1316         run.alias("rust-demangler").default_condition(default)
1317     }
1318
1319     fn make_run(run: RunConfig<'_>) {
1320         run.builder.ensure(RustDemangler {
1321             compiler: run.builder.compiler_for(
1322                 run.builder.top_stage,
1323                 run.builder.config.build,
1324                 run.target,
1325             ),
1326             target: run.target,
1327         });
1328     }
1329
1330     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1331         let compiler = self.compiler;
1332         let target = self.target;
1333
1334         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1335         let condition = should_build_extended_tool(builder, "rust-demangler")
1336             || builder.config.profiler_enabled(target);
1337         if builder.config.extended && !condition {
1338             return None;
1339         }
1340
1341         let rust_demangler = builder
1342             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1343             .expect("rust-demangler expected to build - in-tree tool");
1344
1345         // Prepare the image directory
1346         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1347         tarball.set_overlay(OverlayKind::RustDemangler);
1348         tarball.is_preview(true);
1349         tarball.add_file(&rust_demangler, "bin", 0o755);
1350         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1351         Some(tarball.generate())
1352     }
1353 }
1354
1355 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1356 pub struct Extended {
1357     stage: u32,
1358     host: TargetSelection,
1359     target: TargetSelection,
1360 }
1361
1362 impl Step for Extended {
1363     type Output = ();
1364     const DEFAULT: bool = true;
1365     const ONLY_HOSTS: bool = true;
1366
1367     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1368         let builder = run.builder;
1369         run.alias("extended").default_condition(builder.config.extended)
1370     }
1371
1372     fn make_run(run: RunConfig<'_>) {
1373         run.builder.ensure(Extended {
1374             stage: run.builder.top_stage,
1375             host: run.builder.config.build,
1376             target: run.target,
1377         });
1378     }
1379
1380     /// Creates a combined installer for the specified target in the provided stage.
1381     fn run(self, builder: &Builder<'_>) {
1382         let target = self.target;
1383         let stage = self.stage;
1384         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1385
1386         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1387
1388         let mut tarballs = Vec::new();
1389         let mut built_tools = HashSet::new();
1390         macro_rules! add_component {
1391             ($name:expr => $step:expr) => {
1392                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1393                     tarballs.push(tarball);
1394                     built_tools.insert($name);
1395                 }
1396             };
1397         }
1398
1399         // When rust-std package split from rustc, we needed to ensure that during
1400         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1401         // the std files during uninstall. To do this ensure that rustc comes
1402         // before rust-std in the list below.
1403         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1404         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1405
1406         if target.ends_with("windows-gnu") {
1407             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1408         }
1409
1410         add_component!("rust-docs" => Docs { host: target });
1411         add_component!("rust-json-docs" => JsonDocs { host: target });
1412         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1413         add_component!("cargo" => Cargo { compiler, target });
1414         add_component!("rustfmt" => Rustfmt { compiler, target });
1415         add_component!("rls" => Rls { compiler, target });
1416         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1417         add_component!("llvm-components" => LlvmTools { target });
1418         add_component!("clippy" => Clippy { compiler, target });
1419         add_component!("miri" => Miri { compiler, target });
1420         add_component!("analysis" => Analysis { compiler, target });
1421
1422         let etc = builder.src.join("src/etc/installer");
1423
1424         // Avoid producing tarballs during a dry run.
1425         if builder.config.dry_run() {
1426             return;
1427         }
1428
1429         let tarball = Tarball::new(builder, "rust", &target.triple);
1430         let generated = tarball.combine(&tarballs);
1431
1432         let tmp = tmpdir(builder).join("combined-tarball");
1433         let work = generated.work_dir();
1434
1435         let mut license = String::new();
1436         license += &builder.read(&builder.src.join("COPYRIGHT"));
1437         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1438         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1439         license.push('\n');
1440         license.push('\n');
1441
1442         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1443         let mut rtf = rtf.to_string();
1444         rtf.push('\n');
1445         for line in license.lines() {
1446             rtf.push_str(line);
1447             rtf.push_str("\\line ");
1448         }
1449         rtf.push('}');
1450
1451         fn filter(contents: &str, marker: &str) -> String {
1452             let start = format!("tool-{}-start", marker);
1453             let end = format!("tool-{}-end", marker);
1454             let mut lines = Vec::new();
1455             let mut omitted = false;
1456             for line in contents.lines() {
1457                 if line.contains(&start) {
1458                     omitted = true;
1459                 } else if line.contains(&end) {
1460                     omitted = false;
1461                 } else if !omitted {
1462                     lines.push(line);
1463                 }
1464             }
1465
1466             lines.join("\n")
1467         }
1468
1469         let xform = |p: &Path| {
1470             let mut contents = t!(fs::read_to_string(p));
1471             for tool in &["rust-demangler"] {
1472                 if !built_tools.contains(tool) {
1473                     contents = filter(&contents, tool);
1474                 }
1475             }
1476             let ret = tmp.join(p.file_name().unwrap());
1477             t!(fs::write(&ret, &contents));
1478             ret
1479         };
1480
1481         if target.contains("apple-darwin") {
1482             builder.info("building pkg installer");
1483             let pkg = tmp.join("pkg");
1484             let _ = fs::remove_dir_all(&pkg);
1485
1486             let pkgbuild = |component: &str| {
1487                 let mut cmd = Command::new("pkgbuild");
1488                 cmd.arg("--identifier")
1489                     .arg(format!("org.rust-lang.{}", component))
1490                     .arg("--scripts")
1491                     .arg(pkg.join(component))
1492                     .arg("--nopayload")
1493                     .arg(pkg.join(component).with_extension("pkg"));
1494                 builder.run(&mut cmd);
1495             };
1496
1497             let prepare = |name: &str| {
1498                 builder.create_dir(&pkg.join(name));
1499                 builder.cp_r(
1500                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1501                     &pkg.join(name),
1502                 );
1503                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1504                 pkgbuild(name);
1505             };
1506             prepare("rustc");
1507             prepare("cargo");
1508             prepare("rust-std");
1509             prepare("rust-analysis");
1510             prepare("clippy");
1511             prepare("miri");
1512             prepare("rust-analyzer");
1513             for tool in &["rust-docs", "rust-demangler"] {
1514                 if built_tools.contains(tool) {
1515                     prepare(tool);
1516                 }
1517             }
1518             // create an 'uninstall' package
1519             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1520             pkgbuild("uninstall");
1521
1522             builder.create_dir(&pkg.join("res"));
1523             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1524             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1525             let mut cmd = Command::new("productbuild");
1526             cmd.arg("--distribution")
1527                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1528                 .arg("--resources")
1529                 .arg(pkg.join("res"))
1530                 .arg(distdir(builder).join(format!(
1531                     "{}-{}.pkg",
1532                     pkgname(builder, "rust"),
1533                     target.triple
1534                 )))
1535                 .arg("--package-path")
1536                 .arg(&pkg);
1537             let _time = timeit(builder);
1538             builder.run(&mut cmd);
1539         }
1540
1541         if target.contains("windows") {
1542             let exe = tmp.join("exe");
1543             let _ = fs::remove_dir_all(&exe);
1544
1545             let prepare = |name: &str| {
1546                 builder.create_dir(&exe.join(name));
1547                 let dir = if name == "rust-std" || name == "rust-analysis" {
1548                     format!("{}-{}", name, target.triple)
1549                 } else if name == "rust-analyzer" {
1550                     "rust-analyzer-preview".to_string()
1551                 } else if name == "rust-demangler" {
1552                     "rust-demangler-preview".to_string()
1553                 } else if name == "miri" {
1554                     "miri-preview".to_string()
1555                 } else {
1556                     name.to_string()
1557                 };
1558                 builder.cp_r(
1559                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1560                     &exe.join(name),
1561                 );
1562                 builder.remove(&exe.join(name).join("manifest.in"));
1563             };
1564             prepare("rustc");
1565             prepare("cargo");
1566             prepare("rust-analysis");
1567             prepare("rust-docs");
1568             prepare("rust-std");
1569             prepare("clippy");
1570             prepare("miri");
1571             prepare("rust-analyzer");
1572             for tool in &["rust-demangler"] {
1573                 if built_tools.contains(tool) {
1574                     prepare(tool);
1575                 }
1576             }
1577             if target.ends_with("windows-gnu") {
1578                 prepare("rust-mingw");
1579             }
1580
1581             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1582
1583             // Generate msi installer
1584             let wix_path = env::var_os("WIX")
1585                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1586             let wix = PathBuf::from(wix_path);
1587             let heat = wix.join("bin/heat.exe");
1588             let candle = wix.join("bin/candle.exe");
1589             let light = wix.join("bin/light.exe");
1590
1591             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1592             builder.run(
1593                 Command::new(&heat)
1594                     .current_dir(&exe)
1595                     .arg("dir")
1596                     .arg("rustc")
1597                     .args(&heat_flags)
1598                     .arg("-cg")
1599                     .arg("RustcGroup")
1600                     .arg("-dr")
1601                     .arg("Rustc")
1602                     .arg("-var")
1603                     .arg("var.RustcDir")
1604                     .arg("-out")
1605                     .arg(exe.join("RustcGroup.wxs")),
1606             );
1607             builder.run(
1608                 Command::new(&heat)
1609                     .current_dir(&exe)
1610                     .arg("dir")
1611                     .arg("rust-docs")
1612                     .args(&heat_flags)
1613                     .arg("-cg")
1614                     .arg("DocsGroup")
1615                     .arg("-dr")
1616                     .arg("Docs")
1617                     .arg("-var")
1618                     .arg("var.DocsDir")
1619                     .arg("-out")
1620                     .arg(exe.join("DocsGroup.wxs"))
1621                     .arg("-t")
1622                     .arg(etc.join("msi/squash-components.xsl")),
1623             );
1624             builder.run(
1625                 Command::new(&heat)
1626                     .current_dir(&exe)
1627                     .arg("dir")
1628                     .arg("cargo")
1629                     .args(&heat_flags)
1630                     .arg("-cg")
1631                     .arg("CargoGroup")
1632                     .arg("-dr")
1633                     .arg("Cargo")
1634                     .arg("-var")
1635                     .arg("var.CargoDir")
1636                     .arg("-out")
1637                     .arg(exe.join("CargoGroup.wxs"))
1638                     .arg("-t")
1639                     .arg(etc.join("msi/remove-duplicates.xsl")),
1640             );
1641             builder.run(
1642                 Command::new(&heat)
1643                     .current_dir(&exe)
1644                     .arg("dir")
1645                     .arg("rust-std")
1646                     .args(&heat_flags)
1647                     .arg("-cg")
1648                     .arg("StdGroup")
1649                     .arg("-dr")
1650                     .arg("Std")
1651                     .arg("-var")
1652                     .arg("var.StdDir")
1653                     .arg("-out")
1654                     .arg(exe.join("StdGroup.wxs")),
1655             );
1656             builder.run(
1657                 Command::new(&heat)
1658                     .current_dir(&exe)
1659                     .arg("dir")
1660                     .arg("rust-analyzer")
1661                     .args(&heat_flags)
1662                     .arg("-cg")
1663                     .arg("RustAnalyzerGroup")
1664                     .arg("-dr")
1665                     .arg("RustAnalyzer")
1666                     .arg("-var")
1667                     .arg("var.RustAnalyzerDir")
1668                     .arg("-out")
1669                     .arg(exe.join("RustAnalyzerGroup.wxs"))
1670                     .arg("-t")
1671                     .arg(etc.join("msi/remove-duplicates.xsl")),
1672             );
1673             builder.run(
1674                 Command::new(&heat)
1675                     .current_dir(&exe)
1676                     .arg("dir")
1677                     .arg("clippy")
1678                     .args(&heat_flags)
1679                     .arg("-cg")
1680                     .arg("ClippyGroup")
1681                     .arg("-dr")
1682                     .arg("Clippy")
1683                     .arg("-var")
1684                     .arg("var.ClippyDir")
1685                     .arg("-out")
1686                     .arg(exe.join("ClippyGroup.wxs"))
1687                     .arg("-t")
1688                     .arg(etc.join("msi/remove-duplicates.xsl")),
1689             );
1690             if built_tools.contains("rust-demangler") {
1691                 builder.run(
1692                     Command::new(&heat)
1693                         .current_dir(&exe)
1694                         .arg("dir")
1695                         .arg("rust-demangler")
1696                         .args(&heat_flags)
1697                         .arg("-cg")
1698                         .arg("RustDemanglerGroup")
1699                         .arg("-dr")
1700                         .arg("RustDemangler")
1701                         .arg("-var")
1702                         .arg("var.RustDemanglerDir")
1703                         .arg("-out")
1704                         .arg(exe.join("RustDemanglerGroup.wxs"))
1705                         .arg("-t")
1706                         .arg(etc.join("msi/remove-duplicates.xsl")),
1707                 );
1708             }
1709             builder.run(
1710                 Command::new(&heat)
1711                     .current_dir(&exe)
1712                     .arg("dir")
1713                     .arg("miri")
1714                     .args(&heat_flags)
1715                     .arg("-cg")
1716                     .arg("MiriGroup")
1717                     .arg("-dr")
1718                     .arg("Miri")
1719                     .arg("-var")
1720                     .arg("var.MiriDir")
1721                     .arg("-out")
1722                     .arg(exe.join("MiriGroup.wxs"))
1723                     .arg("-t")
1724                     .arg(etc.join("msi/remove-duplicates.xsl")),
1725             );
1726             builder.run(
1727                 Command::new(&heat)
1728                     .current_dir(&exe)
1729                     .arg("dir")
1730                     .arg("rust-analysis")
1731                     .args(&heat_flags)
1732                     .arg("-cg")
1733                     .arg("AnalysisGroup")
1734                     .arg("-dr")
1735                     .arg("Analysis")
1736                     .arg("-var")
1737                     .arg("var.AnalysisDir")
1738                     .arg("-out")
1739                     .arg(exe.join("AnalysisGroup.wxs"))
1740                     .arg("-t")
1741                     .arg(etc.join("msi/remove-duplicates.xsl")),
1742             );
1743             if target.ends_with("windows-gnu") {
1744                 builder.run(
1745                     Command::new(&heat)
1746                         .current_dir(&exe)
1747                         .arg("dir")
1748                         .arg("rust-mingw")
1749                         .args(&heat_flags)
1750                         .arg("-cg")
1751                         .arg("GccGroup")
1752                         .arg("-dr")
1753                         .arg("Gcc")
1754                         .arg("-var")
1755                         .arg("var.GccDir")
1756                         .arg("-out")
1757                         .arg(exe.join("GccGroup.wxs")),
1758                 );
1759             }
1760
1761             let candle = |input: &Path| {
1762                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1763                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1764                 let mut cmd = Command::new(&candle);
1765                 cmd.current_dir(&exe)
1766                     .arg("-nologo")
1767                     .arg("-dRustcDir=rustc")
1768                     .arg("-dDocsDir=rust-docs")
1769                     .arg("-dCargoDir=cargo")
1770                     .arg("-dStdDir=rust-std")
1771                     .arg("-dAnalysisDir=rust-analysis")
1772                     .arg("-dClippyDir=clippy")
1773                     .arg("-dMiriDir=miri")
1774                     .arg("-arch")
1775                     .arg(&arch)
1776                     .arg("-out")
1777                     .arg(&output)
1778                     .arg(&input);
1779                 add_env(builder, &mut cmd, target);
1780
1781                 if built_tools.contains("rust-demangler") {
1782                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1783                 }
1784                 if built_tools.contains("rust-analyzer") {
1785                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1786                 }
1787                 if target.ends_with("windows-gnu") {
1788                     cmd.arg("-dGccDir=rust-mingw");
1789                 }
1790                 builder.run(&mut cmd);
1791             };
1792             candle(&xform(&etc.join("msi/rust.wxs")));
1793             candle(&etc.join("msi/ui.wxs"));
1794             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1795             candle("RustcGroup.wxs".as_ref());
1796             candle("DocsGroup.wxs".as_ref());
1797             candle("CargoGroup.wxs".as_ref());
1798             candle("StdGroup.wxs".as_ref());
1799             candle("ClippyGroup.wxs".as_ref());
1800             candle("MiriGroup.wxs".as_ref());
1801             if built_tools.contains("rust-demangler") {
1802                 candle("RustDemanglerGroup.wxs".as_ref());
1803             }
1804             if built_tools.contains("rust-analyzer") {
1805                 candle("RustAnalyzerGroup.wxs".as_ref());
1806             }
1807             candle("AnalysisGroup.wxs".as_ref());
1808
1809             if target.ends_with("windows-gnu") {
1810                 candle("GccGroup.wxs".as_ref());
1811             }
1812
1813             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1814             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1815             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1816
1817             builder.info(&format!("building `msi` installer with {:?}", light));
1818             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1819             let mut cmd = Command::new(&light);
1820             cmd.arg("-nologo")
1821                 .arg("-ext")
1822                 .arg("WixUIExtension")
1823                 .arg("-ext")
1824                 .arg("WixUtilExtension")
1825                 .arg("-out")
1826                 .arg(exe.join(&filename))
1827                 .arg("rust.wixobj")
1828                 .arg("ui.wixobj")
1829                 .arg("rustwelcomedlg.wixobj")
1830                 .arg("RustcGroup.wixobj")
1831                 .arg("DocsGroup.wixobj")
1832                 .arg("CargoGroup.wixobj")
1833                 .arg("StdGroup.wixobj")
1834                 .arg("AnalysisGroup.wixobj")
1835                 .arg("ClippyGroup.wixobj")
1836                 .arg("MiriGroup.wixobj")
1837                 .current_dir(&exe);
1838
1839             if built_tools.contains("rust-analyzer") {
1840                 cmd.arg("RustAnalyzerGroup.wixobj");
1841             }
1842             if built_tools.contains("rust-demangler") {
1843                 cmd.arg("RustDemanglerGroup.wixobj");
1844             }
1845
1846             if target.ends_with("windows-gnu") {
1847                 cmd.arg("GccGroup.wixobj");
1848             }
1849             // ICE57 wrongly complains about the shortcuts
1850             cmd.arg("-sice:ICE57");
1851
1852             let _time = timeit(builder);
1853             builder.run(&mut cmd);
1854
1855             if !builder.config.dry_run() {
1856                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1857             }
1858         }
1859     }
1860 }
1861
1862 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1863     let mut parts = builder.version.split('.');
1864     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1865         .env("CFG_RELEASE_NUM", &builder.version)
1866         .env("CFG_RELEASE", builder.rust_release())
1867         .env("CFG_VER_MAJOR", parts.next().unwrap())
1868         .env("CFG_VER_MINOR", parts.next().unwrap())
1869         .env("CFG_VER_PATCH", parts.next().unwrap())
1870         .env("CFG_VER_BUILD", "0") // just needed to build
1871         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1872         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1873         .env("CFG_BUILD", target.triple)
1874         .env("CFG_CHANNEL", &builder.config.channel);
1875
1876     if target.contains("windows-gnullvm") {
1877         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1878     } else if target.contains("windows-gnu") {
1879         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1880     } else {
1881         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1882     }
1883 }
1884
1885 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1886 ///
1887 /// Returns whether the files were actually copied.
1888 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1889     if let Some(config) = builder.config.target_config.get(&target) {
1890         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1891             // If the LLVM was externally provided, then we don't currently copy
1892             // artifacts into the sysroot. This is not necessarily the right
1893             // choice (in particular, it will require the LLVM dylib to be in
1894             // the linker's load path at runtime), but the common use case for
1895             // external LLVMs is distribution provided LLVMs, and in that case
1896             // they're usually in the standard search path (e.g., /usr/lib) and
1897             // copying them here is going to cause problems as we may end up
1898             // with the wrong files and isn't what distributions want.
1899             //
1900             // This behavior may be revisited in the future though.
1901             //
1902             // If the LLVM is coming from ourselves (just from CI) though, we
1903             // still want to install it, as it otherwise won't be available.
1904             return false;
1905         }
1906     }
1907
1908     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1909     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1910     // clear why this is the case, though. llvm-config will emit the versioned
1911     // paths and we don't want those in the sysroot (as we're expecting
1912     // unversioned paths).
1913     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1914         let src_libdir = builder.llvm_out(target).join("lib");
1915         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1916         if llvm_dylib_path.exists() {
1917             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1918         }
1919         !builder.config.dry_run()
1920     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1921         let mut cmd = Command::new(llvm_config);
1922         cmd.arg("--libfiles");
1923         builder.verbose(&format!("running {:?}", cmd));
1924         let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) };
1925         let build_llvm_out = &builder.llvm_out(builder.config.build);
1926         let target_llvm_out = &builder.llvm_out(target);
1927         for file in files.trim_end().split(' ') {
1928             // If we're not using a custom LLVM, make sure we package for the target.
1929             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1930                 target_llvm_out.join(relative_path)
1931             } else {
1932                 PathBuf::from(file)
1933             };
1934             builder.install(&file, dst_libdir, 0o644);
1935         }
1936         !builder.config.dry_run()
1937     } else {
1938         false
1939     }
1940 }
1941
1942 /// Maybe add libLLVM.so to the target lib-dir for linking.
1943 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1944     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1945     // We do not need to copy LLVM files into the sysroot if it is not
1946     // dynamically linked; it is already included into librustc_llvm
1947     // statically.
1948     if builder.llvm_link_shared() {
1949         maybe_install_llvm(builder, target, &dst_libdir);
1950     }
1951 }
1952
1953 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1954 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1955     let dst_libdir =
1956         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1957     // We do not need to copy LLVM files into the sysroot if it is not
1958     // dynamically linked; it is already included into librustc_llvm
1959     // statically.
1960     if builder.llvm_link_shared() {
1961         maybe_install_llvm(builder, target, &dst_libdir);
1962     }
1963 }
1964
1965 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1966 pub struct LlvmTools {
1967     pub target: TargetSelection,
1968 }
1969
1970 impl Step for LlvmTools {
1971     type Output = Option<GeneratedTarball>;
1972     const ONLY_HOSTS: bool = true;
1973     const DEFAULT: bool = true;
1974
1975     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1976         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1977         // FIXME: allow using the names of the tools themselves?
1978         run.alias("llvm-tools").default_condition(default)
1979     }
1980
1981     fn make_run(run: RunConfig<'_>) {
1982         run.builder.ensure(LlvmTools { target: run.target });
1983     }
1984
1985     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1986         let target = self.target;
1987
1988         /* run only if llvm-config isn't used */
1989         if let Some(config) = builder.config.target_config.get(&target) {
1990             if let Some(ref _s) = config.llvm_config {
1991                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1992                 return None;
1993             }
1994         }
1995
1996         builder.ensure(crate::native::Llvm { target });
1997
1998         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1999         tarball.set_overlay(OverlayKind::LLVM);
2000         tarball.is_preview(true);
2001
2002         // Prepare the image directory
2003         let src_bindir = builder.llvm_out(target).join("bin");
2004         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2005         for tool in LLVM_TOOLS {
2006             let exe = src_bindir.join(exe(tool, target));
2007             tarball.add_file(&exe, &dst_bindir, 0o755);
2008         }
2009
2010         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2011         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2012         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2013         // compiler libraries.
2014         maybe_install_llvm_target(builder, target, tarball.image_dir());
2015
2016         Some(tarball.generate())
2017     }
2018 }
2019
2020 // Tarball intended for internal consumption to ease rustc/std development.
2021 //
2022 // Should not be considered stable by end users.
2023 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2024 pub struct RustDev {
2025     pub target: TargetSelection,
2026 }
2027
2028 impl Step for RustDev {
2029     type Output = Option<GeneratedTarball>;
2030     const DEFAULT: bool = true;
2031     const ONLY_HOSTS: bool = true;
2032
2033     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2034         run.alias("rust-dev")
2035     }
2036
2037     fn make_run(run: RunConfig<'_>) {
2038         run.builder.ensure(RustDev { target: run.target });
2039     }
2040
2041     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2042         let target = self.target;
2043
2044         /* run only if llvm-config isn't used */
2045         if let Some(config) = builder.config.target_config.get(&target) {
2046             if let Some(ref _s) = config.llvm_config {
2047                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2048                 return None;
2049             }
2050         }
2051
2052         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2053         tarball.set_overlay(OverlayKind::LLVM);
2054
2055         builder.ensure(crate::native::Llvm { target });
2056
2057         let src_bindir = builder.llvm_out(target).join("bin");
2058         // If updating this list, you likely want to change
2059         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2060         // will not pick up the extra file until LLVM gets bumped.
2061         for bin in &[
2062             "llvm-config",
2063             "llvm-ar",
2064             "llvm-objdump",
2065             "llvm-profdata",
2066             "llvm-bcanalyzer",
2067             "llvm-cov",
2068             "llvm-dwp",
2069             "llvm-nm",
2070             "llvm-dwarfdump",
2071             "llvm-dis",
2072             "llvm-tblgen",
2073         ] {
2074             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2075         }
2076
2077         // We don't build LLD on some platforms, so only add it if it exists
2078         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2079         if lld_path.exists() {
2080             tarball.add_file(lld_path, "bin", 0o755);
2081         }
2082
2083         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2084
2085         // Copy the include directory as well; needed mostly to build
2086         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2087         // just broadly useful to be able to link against the bundled LLVM.
2088         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2089
2090         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2091         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2092         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2093         // compiler libraries.
2094         let dst_libdir = tarball.image_dir().join("lib");
2095         maybe_install_llvm(builder, target, &dst_libdir);
2096         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2097         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2098
2099         Some(tarball.generate())
2100     }
2101 }
2102
2103 // Tarball intended for internal consumption to ease rustc/std development.
2104 //
2105 // Should not be considered stable by end users.
2106 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2107 pub struct Bootstrap {
2108     pub target: TargetSelection,
2109 }
2110
2111 impl Step for Bootstrap {
2112     type Output = Option<GeneratedTarball>;
2113     const DEFAULT: bool = false;
2114     const ONLY_HOSTS: bool = true;
2115
2116     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2117         run.alias("bootstrap")
2118     }
2119
2120     fn make_run(run: RunConfig<'_>) {
2121         run.builder.ensure(Bootstrap { target: run.target });
2122     }
2123
2124     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2125         let target = self.target;
2126
2127         let tarball = Tarball::new(builder, "bootstrap", &target.triple);
2128
2129         let bootstrap_outdir = &builder.bootstrap_out;
2130         for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
2131             tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
2132         }
2133
2134         Some(tarball.generate())
2135     }
2136 }
2137
2138 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2139 /// release process to avoid cloning the monorepo and building stuff.
2140 ///
2141 /// Should not be considered stable by end users.
2142 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2143 pub struct BuildManifest {
2144     pub target: TargetSelection,
2145 }
2146
2147 impl Step for BuildManifest {
2148     type Output = GeneratedTarball;
2149     const DEFAULT: bool = false;
2150     const ONLY_HOSTS: bool = true;
2151
2152     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2153         run.alias("build-manifest")
2154     }
2155
2156     fn make_run(run: RunConfig<'_>) {
2157         run.builder.ensure(BuildManifest { target: run.target });
2158     }
2159
2160     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2161         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2162
2163         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2164         tarball.add_file(&build_manifest, "bin", 0o755);
2165         tarball.generate()
2166     }
2167 }
2168
2169 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2170 ///
2171 /// Currently this is the PGO profile data.
2172 ///
2173 /// Should not be considered stable by end users.
2174 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2175 pub struct ReproducibleArtifacts {
2176     pub target: TargetSelection,
2177 }
2178
2179 impl Step for ReproducibleArtifacts {
2180     type Output = Option<GeneratedTarball>;
2181     const DEFAULT: bool = true;
2182     const ONLY_HOSTS: bool = true;
2183
2184     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2185         run.alias("reproducible-artifacts")
2186     }
2187
2188     fn make_run(run: RunConfig<'_>) {
2189         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2190     }
2191
2192     fn run(self, builder: &Builder<'_>) -> Self::Output {
2193         let mut added_anything = false;
2194         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2195         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2196             tarball.add_file(path, ".", 0o644);
2197             added_anything = true;
2198         }
2199         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2200             tarball.add_file(path, ".", 0o644);
2201             added_anything = true;
2202         }
2203         if let Some(path) = builder.config.llvm_bolt_profile_use.as_ref() {
2204             tarball.add_file(path, ".", 0o644);
2205             added_anything = true;
2206         }
2207         if added_anything { Some(tarball.generate()) } else { None }
2208     }
2209 }