src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  49     if !builder.config.extended {
  50         return false;
  51     }
  52     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  53 }
  54
  55 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  56 pub struct Docs {
  57     pub host: TargetSelection,
  58 }
  59
  60 impl Step for Docs {
  61     type Output = Option<GeneratedTarball>;
  62     const DEFAULT: bool = true;
  63
  64     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  65         let default = run.builder.config.docs;
  66         run.path("rust-docs").default_condition(default)
  67     }
  68
  69     fn make_run(run: RunConfig<'_>) {
  70         run.builder.ensure(Docs { host: run.target });
  71     }
  72
  73     /// Builds the `rust-docs` installer component.
  74     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  75         let host = self.host;
  76         builder.default_doc(&[]);
  77
  78         let dest = "share/doc/rust/html";
  79
  80         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  81         tarball.set_product_name("Rust Documentation");
  82         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  83         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  84         Some(tarball.generate())
  85     }
  86 }
  87
  88 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  89 pub struct RustcDocs {
  90     pub host: TargetSelection,
  91 }
  92
  93 impl Step for RustcDocs {
  94     type Output = Option<GeneratedTarball>;
  95     const DEFAULT: bool = true;
  96
  97     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  98         let builder = run.builder;
  99         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
 100     }
 101
 102     fn make_run(run: RunConfig<'_>) {
 103         run.builder.ensure(RustcDocs { host: run.target });
 104     }
 105
 106     /// Builds the `rustc-docs` installer component.
 107     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 108         let host = self.host;
 109         builder.default_doc(&[]);
 110
 111         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 112         tarball.set_product_name("Rustc Documentation");
 113         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 114         Some(tarball.generate())
 115     }
 116 }
 117
 118 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 119     let mut found = Vec::with_capacity(files.len());
 120
 121     for file in files {
 122         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 123
 124         if let Some(file_path) = file_path {
 125             found.push(file_path);
 126         } else {
 127             panic!("Could not find '{}' in {:?}", file, path);
 128         }
 129     }
 130
 131     found
 132 }
 133
 134 fn make_win_dist(
 135     rust_root: &Path,
 136     plat_root: &Path,
 137     target: TargetSelection,
 138     builder: &Builder<'_>,
 139 ) {
 140     //Ask gcc where it keeps its stuff
 141     let mut cmd = Command::new(builder.cc(target));
 142     cmd.arg("-print-search-dirs");
 143     let gcc_out = output(&mut cmd);
 144
 145     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 146     let mut lib_path = Vec::new();
 147
 148     for line in gcc_out.lines() {
 149         let idx = line.find(':').unwrap();
 150         let key = &line[..idx];
 151         let trim_chars: &[_] = &[' ', '='];
 152         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 153
 154         if key == "programs" {
 155             bin_path.extend(value);
 156         } else if key == "libraries" {
 157             lib_path.extend(value);
 158         }
 159     }
 160
 161     let compiler = if target == "i686-pc-windows-gnu" {
 162         "i686-w64-mingw32-gcc.exe"
 163     } else if target == "x86_64-pc-windows-gnu" {
 164         "x86_64-w64-mingw32-gcc.exe"
 165     } else {
 166         "gcc.exe"
 167     };
 168     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 169     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 170     if target.starts_with("i686-") {
 171         rustc_dlls.push("libgcc_s_dw2-1.dll");
 172     } else {
 173         rustc_dlls.push("libgcc_s_seh-1.dll");
 174     }
 175
 176     let target_libs = [
 177         //MinGW libs
 178         "libgcc.a",
 179         "libgcc_eh.a",
 180         "libgcc_s.a",
 181         "libm.a",
 182         "libmingw32.a",
 183         "libmingwex.a",
 184         "libstdc++.a",
 185         "libiconv.a",
 186         "libmoldname.a",
 187         "libpthread.a",
 188         //Windows import libs
 189         "libadvapi32.a",
 190         "libbcrypt.a",
 191         "libcomctl32.a",
 192         "libcomdlg32.a",
 193         "libcredui.a",
 194         "libcrypt32.a",
 195         "libdbghelp.a",
 196         "libgdi32.a",
 197         "libimagehlp.a",
 198         "libiphlpapi.a",
 199         "libkernel32.a",
 200         "libmsimg32.a",
 201         "libmsvcrt.a",
 202         "libodbc32.a",
 203         "libole32.a",
 204         "liboleaut32.a",
 205         "libopengl32.a",
 206         "libpsapi.a",
 207         "librpcrt4.a",
 208         "libsecur32.a",
 209         "libsetupapi.a",
 210         "libshell32.a",
 211         "libsynchronization.a",
 212         "libuser32.a",
 213         "libuserenv.a",
 214         "libuuid.a",
 215         "libwinhttp.a",
 216         "libwinmm.a",
 217         "libwinspool.a",
 218         "libws2_32.a",
 219         "libwsock32.a",
 220     ];
 221
 222     //Find mingw artifacts we want to bundle
 223     let target_tools = find_files(&target_tools, &bin_path);
 224     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 225     let target_libs = find_files(&target_libs, &lib_path);
 226
 227     // Copy runtime dlls next to rustc.exe
 228     let dist_bin_dir = rust_root.join("bin/");
 229     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 230     for src in rustc_dlls {
 231         builder.copy_to_folder(&src, &dist_bin_dir);
 232     }
 233
 234     //Copy platform tools to platform-specific bin directory
 235     let target_bin_dir = plat_root
 236         .join("lib")
 237         .join("rustlib")
 238         .join(target.triple)
 239         .join("bin")
 240         .join("self-contained");
 241     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 242     for src in target_tools {
 243         builder.copy_to_folder(&src, &target_bin_dir);
 244     }
 245
 246     // Warn windows-gnu users that the bundled GCC cannot compile C files
 247     builder.create(
 248         &target_bin_dir.join("GCC-WARNING.txt"),
 249         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 250          used as a linker. In order to be able to compile projects containing C code use \
 251          the GCC provided by MinGW or Cygwin.",
 252     );
 253
 254     //Copy platform libs to platform-specific lib directory
 255     let target_lib_dir = plat_root
 256         .join("lib")
 257         .join("rustlib")
 258         .join(target.triple)
 259         .join("lib")
 260         .join("self-contained");
 261     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 262     for src in target_libs {
 263         builder.copy_to_folder(&src, &target_lib_dir);
 264     }
 265 }
 266
 267 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 268 pub struct Mingw {
 269     pub host: TargetSelection,
 270 }
 271
 272 impl Step for Mingw {
 273     type Output = Option<GeneratedTarball>;
 274     const DEFAULT: bool = true;
 275
 276     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 277         run.path("rust-mingw")
 278     }
 279
 280     fn make_run(run: RunConfig<'_>) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Builds the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 289         let host = self.host;
 290         if !host.contains("pc-windows-gnu") {
 291             return None;
 292         }
 293
 294         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 295         tarball.set_product_name("Rust MinGW");
 296
 297         // The first argument is a "temporary directory" which is just
 298         // thrown away (this contains the runtime DLLs included in the rustc package
 299         // above) and the second argument is where to place all the MinGW components
 300         // (which is what we want).
 301         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 302
 303         Some(tarball.generate())
 304     }
 305 }
 306
 307 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 308 pub struct Rustc {
 309     pub compiler: Compiler,
 310 }
 311
 312 impl Step for Rustc {
 313     type Output = GeneratedTarball;
 314     const DEFAULT: bool = true;
 315     const ONLY_HOSTS: bool = true;
 316
 317     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 318         run.path("rustc")
 319     }
 320
 321     fn make_run(run: RunConfig<'_>) {
 322         run.builder
 323             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 324     }
 325
 326     /// Creates the `rustc` installer component.
 327     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 328         let compiler = self.compiler;
 329         let host = self.compiler.host;
 330
 331         let tarball = Tarball::new(builder, "rustc", &host.triple);
 332
 333         // Prepare the rustc "image", what will actually end up getting installed
 334         prepare_image(builder, compiler, tarball.image_dir());
 335
 336         // On MinGW we've got a few runtime DLL dependencies that we need to
 337         // include. The first argument to this script is where to put these DLLs
 338         // (the image we're creating), and the second argument is a junk directory
 339         // to ignore all other MinGW stuff the script creates.
 340         //
 341         // On 32-bit MinGW we're always including a DLL which needs some extra
 342         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 343         // anything requiring us to distribute a license, but it's likely the
 344         // install will *also* include the rust-mingw package, which also needs
 345         // licenses, so to be safe we just include it here in all MinGW packages.
 346         if host.contains("pc-windows-gnu") {
 347             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 348             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 349         }
 350
 351         return tarball.generate();
 352
 353         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 354             let host = compiler.host;
 355             let src = builder.sysroot(compiler);
 356
 357             // Copy rustc/rustdoc binaries
 358             t!(fs::create_dir_all(image.join("bin")));
 359             builder.cp_r(&src.join("bin"), &image.join("bin"));
 360
 361             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 362
 363             let libdir_relative = builder.libdir_relative(compiler);
 364
 365             // Copy runtime DLLs needed by the compiler
 366             if libdir_relative.to_str() != Some("bin") {
 367                 let libdir = builder.rustc_libdir(compiler);
 368                 for entry in builder.read_dir(&libdir) {
 369                     let name = entry.file_name();
 370                     if let Some(s) = name.to_str() {
 371                         if is_dylib(s) {
 372                             // Don't use custom libdir here because ^lib/ will be resolved again
 373                             // with installer
 374                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 375                         }
 376                     }
 377                 }
 378             }
 379
 380             // Copy over the codegen backends
 381             let backends_src = builder.sysroot_codegen_backends(compiler);
 382             let backends_rel = backends_src
 383                 .strip_prefix(&src)
 384                 .unwrap()
 385                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 386                 .unwrap();
 387             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 388             let backends_dst = image.join("lib").join(&backends_rel);
 389
 390             t!(fs::create_dir_all(&backends_dst));
 391             builder.cp_r(&backends_src, &backends_dst);
 392
 393             // Copy libLLVM.so to the lib dir as well, if needed. While not
 394             // technically needed by rustc itself it's needed by lots of other
 395             // components like the llvm tools and LLD. LLD is included below and
 396             // tools/LLDB come later, so let's just throw it in the rustc
 397             // component for now.
 398             maybe_install_llvm_runtime(builder, host, image);
 399
 400             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 401             t!(fs::create_dir_all(&dst_dir));
 402
 403             // Copy over lld if it's there
 404             if builder.config.lld_enabled {
 405                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 406                 let rust_lld = exe("rust-lld", compiler.host);
 407                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 408                 // for `-Z gcc-ld=lld`
 409                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 410                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 411                 t!(fs::create_dir(&gcc_lld_dst_dir));
 412                 for flavor in ["ld", "ld64"] {
 413                     let exe_name = exe(flavor, compiler.host);
 414                     builder
 415                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 416                 }
 417             }
 418
 419             // Man pages
 420             t!(fs::create_dir_all(image.join("share/man/man1")));
 421             let man_src = builder.src.join("src/doc/man");
 422             let man_dst = image.join("share/man/man1");
 423
 424             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 425             // to hardlink, and we don't want to edit the source templates
 426             for file_entry in builder.read_dir(&man_src) {
 427                 let page_src = file_entry.path();
 428                 let page_dst = man_dst.join(file_entry.file_name());
 429                 let src_text = t!(std::fs::read_to_string(&page_src));
 430                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 431                 t!(std::fs::write(&page_dst, &new_text));
 432                 t!(fs::copy(&page_src, &page_dst));
 433             }
 434
 435             // Debugger scripts
 436             builder
 437                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 438
 439             // Misc license info
 440             let cp = |file: &str| {
 441                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 442             };
 443             cp("COPYRIGHT");
 444             cp("LICENSE-APACHE");
 445             cp("LICENSE-MIT");
 446             cp("README.md");
 447         }
 448     }
 449 }
 450
 451 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 452 pub struct DebuggerScripts {
 453     pub sysroot: Interned<PathBuf>,
 454     pub host: TargetSelection,
 455 }
 456
 457 impl Step for DebuggerScripts {
 458     type Output = ();
 459
 460     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 461         run.path("src/lldb_batchmode.py")
 462     }
 463
 464     fn make_run(run: RunConfig<'_>) {
 465         run.builder.ensure(DebuggerScripts {
 466             sysroot: run
 467                 .builder
 468                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 469             host: run.target,
 470         });
 471     }
 472
 473     /// Copies debugger scripts for `target` into the `sysroot` specified.
 474     fn run(self, builder: &Builder<'_>) {
 475         let host = self.host;
 476         let sysroot = self.sysroot;
 477         let dst = sysroot.join("lib/rustlib/etc");
 478         t!(fs::create_dir_all(&dst));
 479         let cp_debugger_script = |file: &str| {
 480             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 481         };
 482         if host.contains("windows-msvc") {
 483             // windbg debugger scripts
 484             builder.install(
 485                 &builder.src.join("src/etc/rust-windbg.cmd"),
 486                 &sysroot.join("bin"),
 487                 0o755,
 488             );
 489
 490             cp_debugger_script("natvis/intrinsic.natvis");
 491             cp_debugger_script("natvis/liballoc.natvis");
 492             cp_debugger_script("natvis/libcore.natvis");
 493             cp_debugger_script("natvis/libstd.natvis");
 494         } else {
 495             cp_debugger_script("rust_types.py");
 496
 497             // gdb debugger scripts
 498             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 499             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 500
 501             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 502             cp_debugger_script("gdb_lookup.py");
 503             cp_debugger_script("gdb_providers.py");
 504
 505             // lldb debugger scripts
 506             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 507
 508             cp_debugger_script("lldb_lookup.py");
 509             cp_debugger_script("lldb_providers.py");
 510             cp_debugger_script("lldb_commands")
 511         }
 512     }
 513 }
 514
 515 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 516     // The only true set of target libraries came from the build triple, so
 517     // let's reduce redundant work by only producing archives from that host.
 518     if compiler.host != builder.config.build {
 519         builder.info("\tskipping, not a build host");
 520         true
 521     } else {
 522         false
 523     }
 524 }
 525
 526 /// Copy stamped files into an image's `target/lib` directory.
 527 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 528     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 529     let self_contained_dst = dst.join("self-contained");
 530     t!(fs::create_dir_all(&dst));
 531     t!(fs::create_dir_all(&self_contained_dst));
 532     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 533         if dependency_type == DependencyType::TargetSelfContained {
 534             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 535         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 536             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 537         }
 538     }
 539 }
 540
 541 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 542 pub struct Std {
 543     pub compiler: Compiler,
 544     pub target: TargetSelection,
 545 }
 546
 547 impl Step for Std {
 548     type Output = Option<GeneratedTarball>;
 549     const DEFAULT: bool = true;
 550
 551     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 552         run.path("rust-std")
 553     }
 554
 555     fn make_run(run: RunConfig<'_>) {
 556         run.builder.ensure(Std {
 557             compiler: run.builder.compiler_for(
 558                 run.builder.top_stage,
 559                 run.builder.config.build,
 560                 run.target,
 561             ),
 562             target: run.target,
 563         });
 564     }
 565
 566     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 567         let compiler = self.compiler;
 568         let target = self.target;
 569
 570         if skip_host_target_lib(builder, compiler) {
 571             return None;
 572         }
 573
 574         builder.ensure(compile::Std { compiler, target });
 575
 576         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 577         tarball.include_target_in_component_name(true);
 578
 579         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 580         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 581         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 582
 583         Some(tarball.generate())
 584     }
 585 }
 586
 587 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 588 pub struct RustcDev {
 589     pub compiler: Compiler,
 590     pub target: TargetSelection,
 591 }
 592
 593 impl Step for RustcDev {
 594     type Output = Option<GeneratedTarball>;
 595     const DEFAULT: bool = true;
 596     const ONLY_HOSTS: bool = true;
 597
 598     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 599         run.path("rustc-dev")
 600     }
 601
 602     fn make_run(run: RunConfig<'_>) {
 603         run.builder.ensure(RustcDev {
 604             compiler: run.builder.compiler_for(
 605                 run.builder.top_stage,
 606                 run.builder.config.build,
 607                 run.target,
 608             ),
 609             target: run.target,
 610         });
 611     }
 612
 613     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 614         let compiler = self.compiler;
 615         let target = self.target;
 616         if skip_host_target_lib(builder, compiler) {
 617             return None;
 618         }
 619
 620         builder.ensure(compile::Rustc { compiler, target });
 621
 622         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 623
 624         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 625         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 626         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 627
 628         let src_files = &["Cargo.lock"];
 629         // This is the reduced set of paths which will become the rustc-dev component
 630         // (essentially the compiler crates and all of their path dependencies).
 631         copy_src_dirs(
 632             builder,
 633             &builder.src,
 634             &["compiler"],
 635             &[],
 636             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 637         );
 638         // This particular crate is used as a build dependency of the above.
 639         copy_src_dirs(
 640             builder,
 641             &builder.src,
 642             &["src/build_helper"],
 643             &[],
 644             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 645         );
 646         for file in src_files {
 647             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 648         }
 649
 650         Some(tarball.generate())
 651     }
 652 }
 653
 654 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 655 pub struct Analysis {
 656     pub compiler: Compiler,
 657     pub target: TargetSelection,
 658 }
 659
 660 impl Step for Analysis {
 661     type Output = Option<GeneratedTarball>;
 662     const DEFAULT: bool = true;
 663
 664     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 665         let default = should_build_extended_tool(&run.builder, "analysis");
 666         run.path("rust-analysis").default_condition(default)
 667     }
 668
 669     fn make_run(run: RunConfig<'_>) {
 670         run.builder.ensure(Analysis {
 671             // Find the actual compiler (handling the full bootstrap option) which
 672             // produced the save-analysis data because that data isn't copied
 673             // through the sysroot uplifting.
 674             compiler: run.builder.compiler_for(
 675                 run.builder.top_stage,
 676                 run.builder.config.build,
 677                 run.target,
 678             ),
 679             target: run.target,
 680         });
 681     }
 682
 683     /// Creates a tarball of save-analysis metadata, if available.
 684     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 685         let compiler = self.compiler;
 686         let target = self.target;
 687         if compiler.host != builder.config.build {
 688             return None;
 689         }
 690
 691         builder.ensure(compile::Std { compiler, target });
 692         let src = builder
 693             .stage_out(compiler, Mode::Std)
 694             .join(target.triple)
 695             .join(builder.cargo_dir())
 696             .join("deps")
 697             .join("save-analysis");
 698
 699         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 700         tarball.include_target_in_component_name(true);
 701         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 702         Some(tarball.generate())
 703     }
 704 }
 705
 706 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 707 /// `dst_dir`.
 708 fn copy_src_dirs(
 709     builder: &Builder<'_>,
 710     base: &Path,
 711     src_dirs: &[&str],
 712     exclude_dirs: &[&str],
 713     dst_dir: &Path,
 714 ) {
 715     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 716         let spath = match path.to_str() {
 717             Some(path) => path,
 718             None => return false,
 719         };
 720         if spath.ends_with('~') || spath.ends_with(".pyc") {
 721             return false;
 722         }
 723
 724         const LLVM_PROJECTS: &[&str] = &[
 725             "llvm-project/clang",
 726             "llvm-project\\clang",
 727             "llvm-project/libunwind",
 728             "llvm-project\\libunwind",
 729             "llvm-project/lld",
 730             "llvm-project\\lld",
 731             "llvm-project/lldb",
 732             "llvm-project\\lldb",
 733             "llvm-project/llvm",
 734             "llvm-project\\llvm",
 735             "llvm-project/compiler-rt",
 736             "llvm-project\\compiler-rt",
 737             "llvm-project/cmake",
 738             "llvm-project\\cmake",
 739         ];
 740         if spath.contains("llvm-project")
 741             && !spath.ends_with("llvm-project")
 742             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 743         {
 744             return false;
 745         }
 746
 747         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 748         if LLVM_TEST.iter().any(|path| spath.contains(path))
 749             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 750         {
 751             return false;
 752         }
 753
 754         let full_path = Path::new(dir).join(path);
 755         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 756             return false;
 757         }
 758
 759         let excludes = [
 760             "CVS",
 761             "RCS",
 762             "SCCS",
 763             ".git",
 764             ".gitignore",
 765             ".gitmodules",
 766             ".gitattributes",
 767             ".cvsignore",
 768             ".svn",
 769             ".arch-ids",
 770             "{arch}",
 771             "=RELEASE-ID",
 772             "=meta-update",
 773             "=update",
 774             ".bzr",
 775             ".bzrignore",
 776             ".bzrtags",
 777             ".hg",
 778             ".hgignore",
 779             ".hgrags",
 780             "_darcs",
 781         ];
 782         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 783     }
 784
 785     // Copy the directories using our filter
 786     for item in src_dirs {
 787         let dst = &dst_dir.join(item);
 788         t!(fs::create_dir_all(dst));
 789         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 790     }
 791 }
 792
 793 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 794 pub struct Src;
 795
 796 impl Step for Src {
 797     /// The output path of the src installer tarball
 798     type Output = GeneratedTarball;
 799     const DEFAULT: bool = true;
 800     const ONLY_HOSTS: bool = true;
 801
 802     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 803         run.path("rust-src")
 804     }
 805
 806     fn make_run(run: RunConfig<'_>) {
 807         run.builder.ensure(Src);
 808     }
 809
 810     /// Creates the `rust-src` installer component
 811     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 812         let tarball = Tarball::new_targetless(builder, "rust-src");
 813
 814         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 815         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 816         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 817         // and fix them...
 818         //
 819         // NOTE: if you update the paths here, you also should update the "virtual" path
 820         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 821         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 822
 823         let src_files = ["Cargo.lock"];
 824         // This is the reduced set of paths which will become the rust-src component
 825         // (essentially libstd and all of its path dependencies).
 826         copy_src_dirs(
 827             builder,
 828             &builder.src,
 829             &["library", "src/llvm-project/libunwind"],
 830             &[
 831                 // not needed and contains symlinks which rustup currently
 832                 // chokes on when unpacking.
 833                 "library/backtrace/crates",
 834             ],
 835             &dst_src,
 836         );
 837         for file in src_files.iter() {
 838             builder.copy(&builder.src.join(file), &dst_src.join(file));
 839         }
 840
 841         tarball.generate()
 842     }
 843 }
 844
 845 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 846 pub struct PlainSourceTarball;
 847
 848 impl Step for PlainSourceTarball {
 849     /// Produces the location of the tarball generated
 850     type Output = GeneratedTarball;
 851     const DEFAULT: bool = true;
 852     const ONLY_HOSTS: bool = true;
 853
 854     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 855         let builder = run.builder;
 856         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 857     }
 858
 859     fn make_run(run: RunConfig<'_>) {
 860         run.builder.ensure(PlainSourceTarball);
 861     }
 862
 863     /// Creates the plain source tarball
 864     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 865         let tarball = Tarball::new(builder, "rustc", "src");
 866         let plain_dst_src = tarball.image_dir();
 867
 868         // This is the set of root paths which will become part of the source package
 869         let src_files = [
 870             "COPYRIGHT",
 871             "LICENSE-APACHE",
 872             "LICENSE-MIT",
 873             "CONTRIBUTING.md",
 874             "README.md",
 875             "RELEASES.md",
 876             "configure",
 877             "x.py",
 878             "config.toml.example",
 879             "Cargo.toml",
 880             "Cargo.lock",
 881         ];
 882         let src_dirs = ["src", "compiler", "library"];
 883
 884         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 885
 886         // Copy the files normally
 887         for item in &src_files {
 888             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 889         }
 890
 891         // Create the version file
 892         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 893         if let Some(sha) = builder.rust_sha() {
 894             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 895         }
 896
 897         // If we're building from git sources, we need to vendor a complete distribution.
 898         if builder.rust_info.is_git() {
 899             // Vendor all Cargo dependencies
 900             let mut cmd = Command::new(&builder.initial_cargo);
 901             cmd.arg("vendor")
 902                 .arg("--sync")
 903                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 904                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 905                 .current_dir(&plain_dst_src);
 906             builder.run(&mut cmd);
 907         }
 908
 909         tarball.bare()
 910     }
 911 }
 912
 913 // We have to run a few shell scripts, which choke quite a bit on both `\`
 914 // characters and on `C:\` paths, so normalize both of them away.
 915 pub fn sanitize_sh(path: &Path) -> String {
 916     let path = path.to_str().unwrap().replace("\\", "/");
 917     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 918
 919     fn unc_to_lfs(s: &str) -> &str {
 920         s.strip_prefix("//?/").unwrap_or(s)
 921     }
 922
 923     fn change_drive(s: &str) -> Option<String> {
 924         let mut ch = s.chars();
 925         let drive = ch.next().unwrap_or('C');
 926         if ch.next() != Some(':') {
 927             return None;
 928         }
 929         if ch.next() != Some('/') {
 930             return None;
 931         }
 932         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 933     }
 934 }
 935
 936 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 937 pub struct Cargo {
 938     pub compiler: Compiler,
 939     pub target: TargetSelection,
 940 }
 941
 942 impl Step for Cargo {
 943     type Output = Option<GeneratedTarball>;
 944     const DEFAULT: bool = true;
 945     const ONLY_HOSTS: bool = true;
 946
 947     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 948         let default = should_build_extended_tool(&run.builder, "cargo");
 949         run.path("cargo").default_condition(default)
 950     }
 951
 952     fn make_run(run: RunConfig<'_>) {
 953         run.builder.ensure(Cargo {
 954             compiler: run.builder.compiler_for(
 955                 run.builder.top_stage,
 956                 run.builder.config.build,
 957                 run.target,
 958             ),
 959             target: run.target,
 960         });
 961     }
 962
 963     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 964         let compiler = self.compiler;
 965         let target = self.target;
 966
 967         let cargo = builder.ensure(tool::Cargo { compiler, target });
 968         let src = builder.src.join("src/tools/cargo");
 969         let etc = src.join("src/etc");
 970
 971         // Prepare the image directory
 972         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 973         tarball.set_overlay(OverlayKind::Cargo);
 974
 975         tarball.add_file(&cargo, "bin", 0o755);
 976         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 977         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 978         tarball.add_dir(etc.join("man"), "share/man/man1");
 979         tarball.add_legal_and_readme_to("share/doc/cargo");
 980
 981         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 982             let dirent = dirent.expect("read dir entry");
 983             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 984                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 985             }
 986         }
 987
 988         Some(tarball.generate())
 989     }
 990 }
 991
 992 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 993 pub struct Rls {
 994     pub compiler: Compiler,
 995     pub target: TargetSelection,
 996 }
 997
 998 impl Step for Rls {
 999     type Output = Option<GeneratedTarball>;
1000     const ONLY_HOSTS: bool = true;
1001     const DEFAULT: bool = true;
1002
1003     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1004         let default = should_build_extended_tool(&run.builder, "rls");
1005         run.path("rls").default_condition(default)
1006     }
1007
1008     fn make_run(run: RunConfig<'_>) {
1009         run.builder.ensure(Rls {
1010             compiler: run.builder.compiler_for(
1011                 run.builder.top_stage,
1012                 run.builder.config.build,
1013                 run.target,
1014             ),
1015             target: run.target,
1016         });
1017     }
1018
1019     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1020         let compiler = self.compiler;
1021         let target = self.target;
1022
1023         let rls = builder
1024             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1025             .or_else(|| {
1026                 missing_tool("RLS", builder.build.config.missing_tools);
1027                 None
1028             })?;
1029
1030         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1031         tarball.set_overlay(OverlayKind::RLS);
1032         tarball.is_preview(true);
1033         tarball.add_file(rls, "bin", 0o755);
1034         tarball.add_legal_and_readme_to("share/doc/rls");
1035         Some(tarball.generate())
1036     }
1037 }
1038
1039 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1040 pub struct RustAnalyzer {
1041     pub compiler: Compiler,
1042     pub target: TargetSelection,
1043 }
1044
1045 impl Step for RustAnalyzer {
1046     type Output = Option<GeneratedTarball>;
1047     const DEFAULT: bool = true;
1048     const ONLY_HOSTS: bool = true;
1049
1050     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1051         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1052         run.path("rust-analyzer").default_condition(default)
1053     }
1054
1055     fn make_run(run: RunConfig<'_>) {
1056         run.builder.ensure(RustAnalyzer {
1057             compiler: run.builder.compiler_for(
1058                 run.builder.top_stage,
1059                 run.builder.config.build,
1060                 run.target,
1061             ),
1062             target: run.target,
1063         });
1064     }
1065
1066     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1067         // This prevents rust-analyzer from being built for "dist" or "install"
1068         // on the stable/beta channels. It is a nightly-only tool and should
1069         // not be included.
1070         if !builder.build.unstable_features() {
1071             return None;
1072         }
1073         let compiler = self.compiler;
1074         let target = self.target;
1075
1076         if target.contains("riscv64") {
1077             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1078             // to build. See #74813 for details.
1079             return None;
1080         }
1081
1082         let rust_analyzer = builder
1083             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1084             .expect("rust-analyzer always builds");
1085
1086         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1087         tarball.set_overlay(OverlayKind::RustAnalyzer);
1088         tarball.is_preview(true);
1089         tarball.add_file(rust_analyzer, "bin", 0o755);
1090         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1091         Some(tarball.generate())
1092     }
1093 }
1094
1095 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1096 pub struct Clippy {
1097     pub compiler: Compiler,
1098     pub target: TargetSelection,
1099 }
1100
1101 impl Step for Clippy {
1102     type Output = Option<GeneratedTarball>;
1103     const DEFAULT: bool = true;
1104     const ONLY_HOSTS: bool = true;
1105
1106     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1107         let default = should_build_extended_tool(&run.builder, "clippy");
1108         run.path("clippy").default_condition(default)
1109     }
1110
1111     fn make_run(run: RunConfig<'_>) {
1112         run.builder.ensure(Clippy {
1113             compiler: run.builder.compiler_for(
1114                 run.builder.top_stage,
1115                 run.builder.config.build,
1116                 run.target,
1117             ),
1118             target: run.target,
1119         });
1120     }
1121
1122     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1123         let compiler = self.compiler;
1124         let target = self.target;
1125
1126         // Prepare the image directory
1127         // We expect clippy to build, because we've exited this step above if tool
1128         // state for clippy isn't testing.
1129         let clippy = builder
1130             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1131             .expect("clippy expected to build - essential tool");
1132         let cargoclippy = builder
1133             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1134             .expect("clippy expected to build - essential tool");
1135
1136         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1137         tarball.set_overlay(OverlayKind::Clippy);
1138         tarball.is_preview(true);
1139         tarball.add_file(clippy, "bin", 0o755);
1140         tarball.add_file(cargoclippy, "bin", 0o755);
1141         tarball.add_legal_and_readme_to("share/doc/clippy");
1142         Some(tarball.generate())
1143     }
1144 }
1145
1146 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1147 pub struct Miri {
1148     pub compiler: Compiler,
1149     pub target: TargetSelection,
1150 }
1151
1152 impl Step for Miri {
1153     type Output = Option<GeneratedTarball>;
1154     const DEFAULT: bool = true;
1155     const ONLY_HOSTS: bool = true;
1156
1157     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1158         let default = should_build_extended_tool(&run.builder, "miri");
1159         run.path("miri").default_condition(default)
1160     }
1161
1162     fn make_run(run: RunConfig<'_>) {
1163         run.builder.ensure(Miri {
1164             compiler: run.builder.compiler_for(
1165                 run.builder.top_stage,
1166                 run.builder.config.build,
1167                 run.target,
1168             ),
1169             target: run.target,
1170         });
1171     }
1172
1173     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1174         // This prevents miri from being built for "dist" or "install"
1175         // on the stable/beta channels. It is a nightly-only tool and should
1176         // not be included.
1177         if !builder.build.unstable_features() {
1178             return None;
1179         }
1180         let compiler = self.compiler;
1181         let target = self.target;
1182
1183         let miri = builder
1184             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1185             .or_else(|| {
1186                 missing_tool("miri", builder.build.config.missing_tools);
1187                 None
1188             })?;
1189         let cargomiri = builder
1190             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1191             .or_else(|| {
1192                 missing_tool("cargo miri", builder.build.config.missing_tools);
1193                 None
1194             })?;
1195
1196         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1197         tarball.set_overlay(OverlayKind::Miri);
1198         tarball.is_preview(true);
1199         tarball.add_file(miri, "bin", 0o755);
1200         tarball.add_file(cargomiri, "bin", 0o755);
1201         tarball.add_legal_and_readme_to("share/doc/miri");
1202         Some(tarball.generate())
1203     }
1204 }
1205
1206 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1207 pub struct Rustfmt {
1208     pub compiler: Compiler,
1209     pub target: TargetSelection,
1210 }
1211
1212 impl Step for Rustfmt {
1213     type Output = Option<GeneratedTarball>;
1214     const DEFAULT: bool = true;
1215     const ONLY_HOSTS: bool = true;
1216
1217     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1218         let default = should_build_extended_tool(&run.builder, "rustfmt");
1219         run.path("rustfmt").default_condition(default)
1220     }
1221
1222     fn make_run(run: RunConfig<'_>) {
1223         run.builder.ensure(Rustfmt {
1224             compiler: run.builder.compiler_for(
1225                 run.builder.top_stage,
1226                 run.builder.config.build,
1227                 run.target,
1228             ),
1229             target: run.target,
1230         });
1231     }
1232
1233     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1234         let compiler = self.compiler;
1235         let target = self.target;
1236
1237         let rustfmt = builder
1238             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1239             .or_else(|| {
1240                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1241                 None
1242             })?;
1243         let cargofmt = builder
1244             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1245             .or_else(|| {
1246                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1247                 None
1248             })?;
1249
1250         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1251         tarball.set_overlay(OverlayKind::Rustfmt);
1252         tarball.is_preview(true);
1253         tarball.add_file(rustfmt, "bin", 0o755);
1254         tarball.add_file(cargofmt, "bin", 0o755);
1255         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1256         Some(tarball.generate())
1257     }
1258 }
1259
1260 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1261 pub struct RustDemangler {
1262     pub compiler: Compiler,
1263     pub target: TargetSelection,
1264 }
1265
1266 impl Step for RustDemangler {
1267     type Output = Option<GeneratedTarball>;
1268     const DEFAULT: bool = true;
1269     const ONLY_HOSTS: bool = true;
1270
1271     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1272         // While other tools use `should_build_extended_tool` to decide whether to be run by
1273         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1274         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1275         // we run the step by default when only `extended = true`, and decide whether to actually
1276         // run it or not later.
1277         let default = run.builder.config.extended;
1278         run.path("rust-demangler").default_condition(default)
1279     }
1280
1281     fn make_run(run: RunConfig<'_>) {
1282         run.builder.ensure(RustDemangler {
1283             compiler: run.builder.compiler_for(
1284                 run.builder.top_stage,
1285                 run.builder.config.build,
1286                 run.target,
1287             ),
1288             target: run.target,
1289         });
1290     }
1291
1292     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1293         let compiler = self.compiler;
1294         let target = self.target;
1295
1296         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1297         let condition = should_build_extended_tool(builder, "rust-demangler")
1298             || builder.config.profiler_enabled(target);
1299         if builder.config.extended && !condition {
1300             return None;
1301         }
1302
1303         let rust_demangler = builder
1304             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1305             .expect("rust-demangler expected to build - in-tree tool");
1306
1307         // Prepare the image directory
1308         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1309         tarball.set_overlay(OverlayKind::RustDemangler);
1310         tarball.is_preview(true);
1311         tarball.add_file(&rust_demangler, "bin", 0o755);
1312         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1313         Some(tarball.generate())
1314     }
1315 }
1316
1317 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1318 pub struct Extended {
1319     stage: u32,
1320     host: TargetSelection,
1321     target: TargetSelection,
1322 }
1323
1324 impl Step for Extended {
1325     type Output = ();
1326     const DEFAULT: bool = true;
1327     const ONLY_HOSTS: bool = true;
1328
1329     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1330         let builder = run.builder;
1331         run.path("extended").default_condition(builder.config.extended)
1332     }
1333
1334     fn make_run(run: RunConfig<'_>) {
1335         run.builder.ensure(Extended {
1336             stage: run.builder.top_stage,
1337             host: run.builder.config.build,
1338             target: run.target,
1339         });
1340     }
1341
1342     /// Creates a combined installer for the specified target in the provided stage.
1343     fn run(self, builder: &Builder<'_>) {
1344         let target = self.target;
1345         let stage = self.stage;
1346         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1347
1348         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1349
1350         let mut tarballs = Vec::new();
1351         let mut built_tools = HashSet::new();
1352         macro_rules! add_component {
1353             ($name:expr => $step:expr) => {
1354                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1355                     tarballs.push(tarball);
1356                     built_tools.insert($name);
1357                 }
1358             };
1359         }
1360
1361         // When rust-std package split from rustc, we needed to ensure that during
1362         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1363         // the std files during uninstall. To do this ensure that rustc comes
1364         // before rust-std in the list below.
1365         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1366         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1367
1368         if target.contains("windows-gnu") {
1369             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1370         }
1371
1372         add_component!("rust-docs" => Docs { host: target });
1373         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1374         add_component!("cargo" => Cargo { compiler, target });
1375         add_component!("rustfmt" => Rustfmt { compiler, target });
1376         add_component!("rls" => Rls { compiler, target });
1377         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1378         add_component!("llvm-components" => LlvmTools { target });
1379         add_component!("clippy" => Clippy { compiler, target });
1380         add_component!("miri" => Miri { compiler, target });
1381         add_component!("analysis" => Analysis { compiler, target });
1382
1383         let etc = builder.src.join("src/etc/installer");
1384
1385         // Avoid producing tarballs during a dry run.
1386         if builder.config.dry_run {
1387             return;
1388         }
1389
1390         let tarball = Tarball::new(builder, "rust", &target.triple);
1391         let generated = tarball.combine(&tarballs);
1392
1393         let tmp = tmpdir(builder).join("combined-tarball");
1394         let work = generated.work_dir();
1395
1396         let mut license = String::new();
1397         license += &builder.read(&builder.src.join("COPYRIGHT"));
1398         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1399         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1400         license.push('\n');
1401         license.push('\n');
1402
1403         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1404         let mut rtf = rtf.to_string();
1405         rtf.push('\n');
1406         for line in license.lines() {
1407             rtf.push_str(line);
1408             rtf.push_str("\\line ");
1409         }
1410         rtf.push('}');
1411
1412         fn filter(contents: &str, marker: &str) -> String {
1413             let start = format!("tool-{}-start", marker);
1414             let end = format!("tool-{}-end", marker);
1415             let mut lines = Vec::new();
1416             let mut omitted = false;
1417             for line in contents.lines() {
1418                 if line.contains(&start) {
1419                     omitted = true;
1420                 } else if line.contains(&end) {
1421                     omitted = false;
1422                 } else if !omitted {
1423                     lines.push(line);
1424                 }
1425             }
1426
1427             lines.join("\n")
1428         }
1429
1430         let xform = |p: &Path| {
1431             let mut contents = t!(fs::read_to_string(p));
1432             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1433                 if !built_tools.contains(tool) {
1434                     contents = filter(&contents, tool);
1435                 }
1436             }
1437             let ret = tmp.join(p.file_name().unwrap());
1438             t!(fs::write(&ret, &contents));
1439             ret
1440         };
1441
1442         if target.contains("apple-darwin") {
1443             builder.info("building pkg installer");
1444             let pkg = tmp.join("pkg");
1445             let _ = fs::remove_dir_all(&pkg);
1446
1447             let pkgbuild = |component: &str| {
1448                 let mut cmd = Command::new("pkgbuild");
1449                 cmd.arg("--identifier")
1450                     .arg(format!("org.rust-lang.{}", component))
1451                     .arg("--scripts")
1452                     .arg(pkg.join(component))
1453                     .arg("--nopayload")
1454                     .arg(pkg.join(component).with_extension("pkg"));
1455                 builder.run(&mut cmd);
1456             };
1457
1458             let prepare = |name: &str| {
1459                 builder.create_dir(&pkg.join(name));
1460                 builder.cp_r(
1461                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1462                     &pkg.join(name),
1463                 );
1464                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1465                 pkgbuild(name);
1466             };
1467             prepare("rustc");
1468             prepare("cargo");
1469             prepare("rust-std");
1470             prepare("rust-analysis");
1471             prepare("clippy");
1472             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1473                 if built_tools.contains(tool) {
1474                     prepare(tool);
1475                 }
1476             }
1477             // create an 'uninstall' package
1478             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1479             pkgbuild("uninstall");
1480
1481             builder.create_dir(&pkg.join("res"));
1482             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1483             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1484             let mut cmd = Command::new("productbuild");
1485             cmd.arg("--distribution")
1486                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1487                 .arg("--resources")
1488                 .arg(pkg.join("res"))
1489                 .arg(distdir(builder).join(format!(
1490                     "{}-{}.pkg",
1491                     pkgname(builder, "rust"),
1492                     target.triple
1493                 )))
1494                 .arg("--package-path")
1495                 .arg(&pkg);
1496             let _time = timeit(builder);
1497             builder.run(&mut cmd);
1498         }
1499
1500         if target.contains("windows") {
1501             let exe = tmp.join("exe");
1502             let _ = fs::remove_dir_all(&exe);
1503
1504             let prepare = |name: &str| {
1505                 builder.create_dir(&exe.join(name));
1506                 let dir = if name == "rust-std" || name == "rust-analysis" {
1507                     format!("{}-{}", name, target.triple)
1508                 } else if name == "rls" {
1509                     "rls-preview".to_string()
1510                 } else if name == "rust-analyzer" {
1511                     "rust-analyzer-preview".to_string()
1512                 } else if name == "clippy" {
1513                     "clippy-preview".to_string()
1514                 } else if name == "rust-demangler" {
1515                     "rust-demangler-preview".to_string()
1516                 } else if name == "miri" {
1517                     "miri-preview".to_string()
1518                 } else {
1519                     name.to_string()
1520                 };
1521                 builder.cp_r(
1522                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1523                     &exe.join(name),
1524                 );
1525                 builder.remove(&exe.join(name).join("manifest.in"));
1526             };
1527             prepare("rustc");
1528             prepare("cargo");
1529             prepare("rust-analysis");
1530             prepare("rust-docs");
1531             prepare("rust-std");
1532             prepare("clippy");
1533             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1534                 if built_tools.contains(tool) {
1535                     prepare(tool);
1536                 }
1537             }
1538             if target.contains("windows-gnu") {
1539                 prepare("rust-mingw");
1540             }
1541
1542             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1543
1544             // Generate msi installer
1545             let wix_path = env::var_os("WIX")
1546                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1547             let wix = PathBuf::from(wix_path);
1548             let heat = wix.join("bin/heat.exe");
1549             let candle = wix.join("bin/candle.exe");
1550             let light = wix.join("bin/light.exe");
1551
1552             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1553             builder.run(
1554                 Command::new(&heat)
1555                     .current_dir(&exe)
1556                     .arg("dir")
1557                     .arg("rustc")
1558                     .args(&heat_flags)
1559                     .arg("-cg")
1560                     .arg("RustcGroup")
1561                     .arg("-dr")
1562                     .arg("Rustc")
1563                     .arg("-var")
1564                     .arg("var.RustcDir")
1565                     .arg("-out")
1566                     .arg(exe.join("RustcGroup.wxs")),
1567             );
1568             builder.run(
1569                 Command::new(&heat)
1570                     .current_dir(&exe)
1571                     .arg("dir")
1572                     .arg("rust-docs")
1573                     .args(&heat_flags)
1574                     .arg("-cg")
1575                     .arg("DocsGroup")
1576                     .arg("-dr")
1577                     .arg("Docs")
1578                     .arg("-var")
1579                     .arg("var.DocsDir")
1580                     .arg("-out")
1581                     .arg(exe.join("DocsGroup.wxs"))
1582                     .arg("-t")
1583                     .arg(etc.join("msi/squash-components.xsl")),
1584             );
1585             builder.run(
1586                 Command::new(&heat)
1587                     .current_dir(&exe)
1588                     .arg("dir")
1589                     .arg("cargo")
1590                     .args(&heat_flags)
1591                     .arg("-cg")
1592                     .arg("CargoGroup")
1593                     .arg("-dr")
1594                     .arg("Cargo")
1595                     .arg("-var")
1596                     .arg("var.CargoDir")
1597                     .arg("-out")
1598                     .arg(exe.join("CargoGroup.wxs"))
1599                     .arg("-t")
1600                     .arg(etc.join("msi/remove-duplicates.xsl")),
1601             );
1602             builder.run(
1603                 Command::new(&heat)
1604                     .current_dir(&exe)
1605                     .arg("dir")
1606                     .arg("rust-std")
1607                     .args(&heat_flags)
1608                     .arg("-cg")
1609                     .arg("StdGroup")
1610                     .arg("-dr")
1611                     .arg("Std")
1612                     .arg("-var")
1613                     .arg("var.StdDir")
1614                     .arg("-out")
1615                     .arg(exe.join("StdGroup.wxs")),
1616             );
1617             if built_tools.contains("rls") {
1618                 builder.run(
1619                     Command::new(&heat)
1620                         .current_dir(&exe)
1621                         .arg("dir")
1622                         .arg("rls")
1623                         .args(&heat_flags)
1624                         .arg("-cg")
1625                         .arg("RlsGroup")
1626                         .arg("-dr")
1627                         .arg("Rls")
1628                         .arg("-var")
1629                         .arg("var.RlsDir")
1630                         .arg("-out")
1631                         .arg(exe.join("RlsGroup.wxs"))
1632                         .arg("-t")
1633                         .arg(etc.join("msi/remove-duplicates.xsl")),
1634                 );
1635             }
1636             if built_tools.contains("rust-analyzer") {
1637                 builder.run(
1638                     Command::new(&heat)
1639                         .current_dir(&exe)
1640                         .arg("dir")
1641                         .arg("rust-analyzer")
1642                         .args(&heat_flags)
1643                         .arg("-cg")
1644                         .arg("RustAnalyzerGroup")
1645                         .arg("-dr")
1646                         .arg("RustAnalyzer")
1647                         .arg("-var")
1648                         .arg("var.RustAnalyzerDir")
1649                         .arg("-out")
1650                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1651                         .arg("-t")
1652                         .arg(etc.join("msi/remove-duplicates.xsl")),
1653                 );
1654             }
1655             builder.run(
1656                 Command::new(&heat)
1657                     .current_dir(&exe)
1658                     .arg("dir")
1659                     .arg("clippy")
1660                     .args(&heat_flags)
1661                     .arg("-cg")
1662                     .arg("ClippyGroup")
1663                     .arg("-dr")
1664                     .arg("Clippy")
1665                     .arg("-var")
1666                     .arg("var.ClippyDir")
1667                     .arg("-out")
1668                     .arg(exe.join("ClippyGroup.wxs"))
1669                     .arg("-t")
1670                     .arg(etc.join("msi/remove-duplicates.xsl")),
1671             );
1672             if built_tools.contains("rust-demangler") {
1673                 builder.run(
1674                     Command::new(&heat)
1675                         .current_dir(&exe)
1676                         .arg("dir")
1677                         .arg("rust-demangler")
1678                         .args(&heat_flags)
1679                         .arg("-cg")
1680                         .arg("RustDemanglerGroup")
1681                         .arg("-dr")
1682                         .arg("RustDemangler")
1683                         .arg("-var")
1684                         .arg("var.RustDemanglerDir")
1685                         .arg("-out")
1686                         .arg(exe.join("RustDemanglerGroup.wxs"))
1687                         .arg("-t")
1688                         .arg(etc.join("msi/remove-duplicates.xsl")),
1689                 );
1690             }
1691             if built_tools.contains("miri") {
1692                 builder.run(
1693                     Command::new(&heat)
1694                         .current_dir(&exe)
1695                         .arg("dir")
1696                         .arg("miri")
1697                         .args(&heat_flags)
1698                         .arg("-cg")
1699                         .arg("MiriGroup")
1700                         .arg("-dr")
1701                         .arg("Miri")
1702                         .arg("-var")
1703                         .arg("var.MiriDir")
1704                         .arg("-out")
1705                         .arg(exe.join("MiriGroup.wxs"))
1706                         .arg("-t")
1707                         .arg(etc.join("msi/remove-duplicates.xsl")),
1708                 );
1709             }
1710             builder.run(
1711                 Command::new(&heat)
1712                     .current_dir(&exe)
1713                     .arg("dir")
1714                     .arg("rust-analysis")
1715                     .args(&heat_flags)
1716                     .arg("-cg")
1717                     .arg("AnalysisGroup")
1718                     .arg("-dr")
1719                     .arg("Analysis")
1720                     .arg("-var")
1721                     .arg("var.AnalysisDir")
1722                     .arg("-out")
1723                     .arg(exe.join("AnalysisGroup.wxs"))
1724                     .arg("-t")
1725                     .arg(etc.join("msi/remove-duplicates.xsl")),
1726             );
1727             if target.contains("windows-gnu") {
1728                 builder.run(
1729                     Command::new(&heat)
1730                         .current_dir(&exe)
1731                         .arg("dir")
1732                         .arg("rust-mingw")
1733                         .args(&heat_flags)
1734                         .arg("-cg")
1735                         .arg("GccGroup")
1736                         .arg("-dr")
1737                         .arg("Gcc")
1738                         .arg("-var")
1739                         .arg("var.GccDir")
1740                         .arg("-out")
1741                         .arg(exe.join("GccGroup.wxs")),
1742                 );
1743             }
1744
1745             let candle = |input: &Path| {
1746                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1747                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1748                 let mut cmd = Command::new(&candle);
1749                 cmd.current_dir(&exe)
1750                     .arg("-nologo")
1751                     .arg("-dRustcDir=rustc")
1752                     .arg("-dDocsDir=rust-docs")
1753                     .arg("-dCargoDir=cargo")
1754                     .arg("-dStdDir=rust-std")
1755                     .arg("-dAnalysisDir=rust-analysis")
1756                     .arg("-dClippyDir=clippy")
1757                     .arg("-arch")
1758                     .arg(&arch)
1759                     .arg("-out")
1760                     .arg(&output)
1761                     .arg(&input);
1762                 add_env(builder, &mut cmd, target);
1763
1764                 if built_tools.contains("rust-demangler") {
1765                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1766                 }
1767                 if built_tools.contains("rls") {
1768                     cmd.arg("-dRlsDir=rls");
1769                 }
1770                 if built_tools.contains("rust-analyzer") {
1771                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1772                 }
1773                 if built_tools.contains("miri") {
1774                     cmd.arg("-dMiriDir=miri");
1775                 }
1776                 if target.contains("windows-gnu") {
1777                     cmd.arg("-dGccDir=rust-mingw");
1778                 }
1779                 builder.run(&mut cmd);
1780             };
1781             candle(&xform(&etc.join("msi/rust.wxs")));
1782             candle(&etc.join("msi/ui.wxs"));
1783             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1784             candle("RustcGroup.wxs".as_ref());
1785             candle("DocsGroup.wxs".as_ref());
1786             candle("CargoGroup.wxs".as_ref());
1787             candle("StdGroup.wxs".as_ref());
1788             candle("ClippyGroup.wxs".as_ref());
1789             if built_tools.contains("rust-demangler") {
1790                 candle("RustDemanglerGroup.wxs".as_ref());
1791             }
1792             if built_tools.contains("rls") {
1793                 candle("RlsGroup.wxs".as_ref());
1794             }
1795             if built_tools.contains("rust-analyzer") {
1796                 candle("RustAnalyzerGroup.wxs".as_ref());
1797             }
1798             if built_tools.contains("miri") {
1799                 candle("MiriGroup.wxs".as_ref());
1800             }
1801             candle("AnalysisGroup.wxs".as_ref());
1802
1803             if target.contains("windows-gnu") {
1804                 candle("GccGroup.wxs".as_ref());
1805             }
1806
1807             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1808             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1809             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1810
1811             builder.info(&format!("building `msi` installer with {:?}", light));
1812             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1813             let mut cmd = Command::new(&light);
1814             cmd.arg("-nologo")
1815                 .arg("-ext")
1816                 .arg("WixUIExtension")
1817                 .arg("-ext")
1818                 .arg("WixUtilExtension")
1819                 .arg("-out")
1820                 .arg(exe.join(&filename))
1821                 .arg("rust.wixobj")
1822                 .arg("ui.wixobj")
1823                 .arg("rustwelcomedlg.wixobj")
1824                 .arg("RustcGroup.wixobj")
1825                 .arg("DocsGroup.wixobj")
1826                 .arg("CargoGroup.wixobj")
1827                 .arg("StdGroup.wixobj")
1828                 .arg("AnalysisGroup.wixobj")
1829                 .arg("ClippyGroup.wixobj")
1830                 .current_dir(&exe);
1831
1832             if built_tools.contains("rls") {
1833                 cmd.arg("RlsGroup.wixobj");
1834             }
1835             if built_tools.contains("rust-analyzer") {
1836                 cmd.arg("RustAnalyzerGroup.wixobj");
1837             }
1838             if built_tools.contains("rust-demangler") {
1839                 cmd.arg("RustDemanglerGroup.wixobj");
1840             }
1841             if built_tools.contains("miri") {
1842                 cmd.arg("MiriGroup.wixobj");
1843             }
1844
1845             if target.contains("windows-gnu") {
1846                 cmd.arg("GccGroup.wixobj");
1847             }
1848             // ICE57 wrongly complains about the shortcuts
1849             cmd.arg("-sice:ICE57");
1850
1851             let _time = timeit(builder);
1852             builder.run(&mut cmd);
1853
1854             if !builder.config.dry_run {
1855                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1856             }
1857         }
1858     }
1859 }
1860
1861 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1862     let mut parts = builder.version.split('.');
1863     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1864         .env("CFG_RELEASE_NUM", &builder.version)
1865         .env("CFG_RELEASE", builder.rust_release())
1866         .env("CFG_VER_MAJOR", parts.next().unwrap())
1867         .env("CFG_VER_MINOR", parts.next().unwrap())
1868         .env("CFG_VER_PATCH", parts.next().unwrap())
1869         .env("CFG_VER_BUILD", "0") // just needed to build
1870         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1871         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1872         .env("CFG_BUILD", target.triple)
1873         .env("CFG_CHANNEL", &builder.config.channel);
1874
1875     if target.contains("windows-gnu") {
1876         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1877     } else {
1878         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1879     }
1880 }
1881
1882 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1883 ///
1884
1885 /// Returns whether the files were actually copied.
1886 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1887     if let Some(config) = builder.config.target_config.get(&target) {
1888         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1889             // If the LLVM was externally provided, then we don't currently copy
1890             // artifacts into the sysroot. This is not necessarily the right
1891             // choice (in particular, it will require the LLVM dylib to be in
1892             // the linker's load path at runtime), but the common use case for
1893             // external LLVMs is distribution provided LLVMs, and in that case
1894             // they're usually in the standard search path (e.g., /usr/lib) and
1895             // copying them here is going to cause problems as we may end up
1896             // with the wrong files and isn't what distributions want.
1897             //
1898             // This behavior may be revisited in the future though.
1899             //
1900             // If the LLVM is coming from ourselves (just from CI) though, we
1901             // still want to install it, as it otherwise won't be available.
1902             return false;
1903         }
1904     }
1905
1906     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1907     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1908     // clear why this is the case, though. llvm-config will emit the versioned
1909     // paths and we don't want those in the sysroot (as we're expecting
1910     // unversioned paths).
1911     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1912         let src_libdir = builder.llvm_out(target).join("lib");
1913         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1914         if llvm_dylib_path.exists() {
1915             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1916         }
1917         !builder.config.dry_run
1918     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1919         let mut cmd = Command::new(llvm_config);
1920         cmd.arg("--libfiles");
1921         builder.verbose(&format!("running {:?}", cmd));
1922         let files = output(&mut cmd);
1923         let build_llvm_out = &builder.llvm_out(builder.config.build);
1924         let target_llvm_out = &builder.llvm_out(target);
1925         for file in files.trim_end().split(' ') {
1926             // If we're not using a custom LLVM, make sure we package for the target.
1927             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1928                 target_llvm_out.join(relative_path)
1929             } else {
1930                 PathBuf::from(file)
1931             };
1932             builder.install(&file, dst_libdir, 0o644);
1933         }
1934         !builder.config.dry_run
1935     } else {
1936         false
1937     }
1938 }
1939
1940 /// Maybe add libLLVM.so to the target lib-dir for linking.
1941 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1942     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1943     // We do not need to copy LLVM files into the sysroot if it is not
1944     // dynamically linked; it is already included into librustc_llvm
1945     // statically.
1946     if builder.config.llvm_link_shared {
1947         maybe_install_llvm(builder, target, &dst_libdir);
1948     }
1949 }
1950
1951 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1952 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1953     let dst_libdir =
1954         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1955     // We do not need to copy LLVM files into the sysroot if it is not
1956     // dynamically linked; it is already included into librustc_llvm
1957     // statically.
1958     if builder.config.llvm_link_shared {
1959         maybe_install_llvm(builder, target, &dst_libdir);
1960     }
1961 }
1962
1963 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1964 pub struct LlvmTools {
1965     pub target: TargetSelection,
1966 }
1967
1968 impl Step for LlvmTools {
1969     type Output = Option<GeneratedTarball>;
1970     const ONLY_HOSTS: bool = true;
1971     const DEFAULT: bool = true;
1972
1973     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1974         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1975         run.path("llvm-tools").default_condition(default)
1976     }
1977
1978     fn make_run(run: RunConfig<'_>) {
1979         run.builder.ensure(LlvmTools { target: run.target });
1980     }
1981
1982     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1983         let target = self.target;
1984
1985         /* run only if llvm-config isn't used */
1986         if let Some(config) = builder.config.target_config.get(&target) {
1987             if let Some(ref _s) = config.llvm_config {
1988                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1989                 return None;
1990             }
1991         }
1992
1993         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1994         tarball.set_overlay(OverlayKind::LLVM);
1995         tarball.is_preview(true);
1996
1997         // Prepare the image directory
1998         let src_bindir = builder.llvm_out(target).join("bin");
1999         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2000         for tool in LLVM_TOOLS {
2001             let exe = src_bindir.join(exe(tool, target));
2002             tarball.add_file(&exe, &dst_bindir, 0o755);
2003         }
2004
2005         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2006         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2007         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2008         // compiler libraries.
2009         maybe_install_llvm_target(builder, target, tarball.image_dir());
2010
2011         Some(tarball.generate())
2012     }
2013 }
2014
2015 // Tarball intended for internal consumption to ease rustc/std development.
2016 //
2017 // Should not be considered stable by end users.
2018 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2019 pub struct RustDev {
2020     pub target: TargetSelection,
2021 }
2022
2023 impl Step for RustDev {
2024     type Output = Option<GeneratedTarball>;
2025     const DEFAULT: bool = true;
2026     const ONLY_HOSTS: bool = true;
2027
2028     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2029         run.path("rust-dev")
2030     }
2031
2032     fn make_run(run: RunConfig<'_>) {
2033         run.builder.ensure(RustDev { target: run.target });
2034     }
2035
2036     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2037         let target = self.target;
2038
2039         /* run only if llvm-config isn't used */
2040         if let Some(config) = builder.config.target_config.get(&target) {
2041             if let Some(ref _s) = config.llvm_config {
2042                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2043                 return None;
2044             }
2045         }
2046
2047         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2048         tarball.set_overlay(OverlayKind::LLVM);
2049
2050         let src_bindir = builder.llvm_out(target).join("bin");
2051         for bin in &[
2052             "llvm-config",
2053             "llvm-ar",
2054             "llvm-objdump",
2055             "llvm-profdata",
2056             "llvm-bcanalyzer",
2057             "llvm-cov",
2058             "llvm-dwp",
2059             "llvm-nm",
2060         ] {
2061             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2062         }
2063
2064         // We don't build LLD on some platforms, so only add it if it exists
2065         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2066         if lld_path.exists() {
2067             tarball.add_file(lld_path, "bin", 0o755);
2068         }
2069
2070         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2071
2072         // Copy the include directory as well; needed mostly to build
2073         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2074         // just broadly useful to be able to link against the bundled LLVM.
2075         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2076
2077         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2078         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2079         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2080         // compiler libraries.
2081         let dst_libdir = tarball.image_dir().join("lib");
2082         maybe_install_llvm(builder, target, &dst_libdir);
2083         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2084         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2085
2086         Some(tarball.generate())
2087     }
2088 }
2089
2090 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2091 /// release process to avoid cloning the monorepo and building stuff.
2092 ///
2093 /// Should not be considered stable by end users.
2094 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2095 pub struct BuildManifest {
2096     pub target: TargetSelection,
2097 }
2098
2099 impl Step for BuildManifest {
2100     type Output = GeneratedTarball;
2101     const DEFAULT: bool = false;
2102     const ONLY_HOSTS: bool = true;
2103
2104     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2105         run.path("build-manifest")
2106     }
2107
2108     fn make_run(run: RunConfig<'_>) {
2109         run.builder.ensure(BuildManifest { target: run.target });
2110     }
2111
2112     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2113         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2114
2115         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2116         tarball.add_file(&build_manifest, "bin", 0o755);
2117         tarball.generate()
2118     }
2119 }
2120
2121 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2122 ///
2123 /// Currently this is the PGO profile data.
2124 ///
2125 /// Should not be considered stable by end users.
2126 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2127 pub struct ReproducibleArtifacts {
2128     pub target: TargetSelection,
2129 }
2130
2131 impl Step for ReproducibleArtifacts {
2132     type Output = Option<GeneratedTarball>;
2133     const DEFAULT: bool = true;
2134     const ONLY_HOSTS: bool = true;
2135
2136     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2137         run.path("reproducible-artifacts")
2138     }
2139
2140     fn make_run(run: RunConfig<'_>) {
2141         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2142     }
2143
2144     fn run(self, builder: &Builder<'_>) -> Self::Output {
2145         let mut added_anything = false;
2146         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2147         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2148             tarball.add_file(path, ".", 0o644);
2149             added_anything = true;
2150         }
2151         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2152             tarball.add_file(path, ".", 0o644);
2153             added_anything = true;
2154         }
2155         if added_anything { Some(tarball.generate()) } else { None }
2156     }
2157 }