src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let exe = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 405             }
 406
 407             // Copy over llvm-dwp if it's there
 408             let exe = exe("rust-llvm-dwp", compiler.host);
 409             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 410
 411             // Man pages
 412             t!(fs::create_dir_all(image.join("share/man/man1")));
 413             let man_src = builder.src.join("src/doc/man");
 414             let man_dst = image.join("share/man/man1");
 415
 416             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 417             let time = env::var("SOURCE_DATE_EPOCH")
 418                 .map(|timestamp| {
 419                     let epoch = timestamp
 420                         .parse()
 421                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 422                         .unwrap();
 423
 424                     time::at(Timespec::new(epoch, 0))
 425                 })
 426                 .unwrap_or_else(|_| time::now());
 427
 428             let month_year = t!(time::strftime("%B %Y", &time));
 429             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 430             // to hardlink, and we don't want to edit the source templates
 431             for file_entry in builder.read_dir(&man_src) {
 432                 let page_src = file_entry.path();
 433                 let page_dst = man_dst.join(file_entry.file_name());
 434                 t!(fs::copy(&page_src, &page_dst));
 435                 // template in month/year and version number
 436                 builder.replace_in_file(
 437                     &page_dst,
 438                     &[
 439                         ("<INSERT DATE HERE>", &month_year),
 440                         ("<INSERT VERSION HERE>", &builder.version),
 441                     ],
 442                 );
 443             }
 444
 445             // Debugger scripts
 446             builder
 447                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 448
 449             // Misc license info
 450             let cp = |file: &str| {
 451                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 452             };
 453             cp("COPYRIGHT");
 454             cp("LICENSE-APACHE");
 455             cp("LICENSE-MIT");
 456             cp("README.md");
 457         }
 458     }
 459 }
 460
 461 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 462 pub struct DebuggerScripts {
 463     pub sysroot: Interned<PathBuf>,
 464     pub host: TargetSelection,
 465 }
 466
 467 impl Step for DebuggerScripts {
 468     type Output = ();
 469
 470     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 471         run.path("src/lldb_batchmode.py")
 472     }
 473
 474     fn make_run(run: RunConfig<'_>) {
 475         run.builder.ensure(DebuggerScripts {
 476             sysroot: run
 477                 .builder
 478                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 479             host: run.target,
 480         });
 481     }
 482
 483     /// Copies debugger scripts for `target` into the `sysroot` specified.
 484     fn run(self, builder: &Builder<'_>) {
 485         let host = self.host;
 486         let sysroot = self.sysroot;
 487         let dst = sysroot.join("lib/rustlib/etc");
 488         t!(fs::create_dir_all(&dst));
 489         let cp_debugger_script = |file: &str| {
 490             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 491         };
 492         if host.contains("windows-msvc") {
 493             // windbg debugger scripts
 494             builder.install(
 495                 &builder.src.join("src/etc/rust-windbg.cmd"),
 496                 &sysroot.join("bin"),
 497                 0o755,
 498             );
 499
 500             cp_debugger_script("natvis/intrinsic.natvis");
 501             cp_debugger_script("natvis/liballoc.natvis");
 502             cp_debugger_script("natvis/libcore.natvis");
 503             cp_debugger_script("natvis/libstd.natvis");
 504         } else {
 505             cp_debugger_script("rust_types.py");
 506
 507             // gdb debugger scripts
 508             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 509             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 510
 511             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 512             cp_debugger_script("gdb_lookup.py");
 513             cp_debugger_script("gdb_providers.py");
 514
 515             // lldb debugger scripts
 516             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 517
 518             cp_debugger_script("lldb_lookup.py");
 519             cp_debugger_script("lldb_providers.py");
 520             cp_debugger_script("lldb_commands")
 521         }
 522     }
 523 }
 524
 525 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 526     // The only true set of target libraries came from the build triple, so
 527     // let's reduce redundant work by only producing archives from that host.
 528     if compiler.host != builder.config.build {
 529         builder.info("\tskipping, not a build host");
 530         true
 531     } else {
 532         false
 533     }
 534 }
 535
 536 /// Copy stamped files into an image's `target/lib` directory.
 537 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 538     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 539     let self_contained_dst = dst.join("self-contained");
 540     t!(fs::create_dir_all(&dst));
 541     t!(fs::create_dir_all(&self_contained_dst));
 542     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 543         if dependency_type == DependencyType::TargetSelfContained {
 544             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 545         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 546             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 547         }
 548     }
 549 }
 550
 551 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 552 pub struct Std {
 553     pub compiler: Compiler,
 554     pub target: TargetSelection,
 555 }
 556
 557 impl Step for Std {
 558     type Output = Option<GeneratedTarball>;
 559     const DEFAULT: bool = true;
 560
 561     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 562         run.path("library/std")
 563     }
 564
 565     fn make_run(run: RunConfig<'_>) {
 566         run.builder.ensure(Std {
 567             compiler: run.builder.compiler_for(
 568                 run.builder.top_stage,
 569                 run.builder.config.build,
 570                 run.target,
 571             ),
 572             target: run.target,
 573         });
 574     }
 575
 576     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 577         let compiler = self.compiler;
 578         let target = self.target;
 579
 580         if skip_host_target_lib(builder, compiler) {
 581             return None;
 582         }
 583
 584         builder.ensure(compile::Std { compiler, target });
 585
 586         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 587         tarball.include_target_in_component_name(true);
 588
 589         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 590         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 591         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 592
 593         Some(tarball.generate())
 594     }
 595 }
 596
 597 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 598 pub struct RustcDev {
 599     pub compiler: Compiler,
 600     pub target: TargetSelection,
 601 }
 602
 603 impl Step for RustcDev {
 604     type Output = Option<GeneratedTarball>;
 605     const DEFAULT: bool = true;
 606     const ONLY_HOSTS: bool = true;
 607
 608     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 609         run.path("rustc-dev")
 610     }
 611
 612     fn make_run(run: RunConfig<'_>) {
 613         run.builder.ensure(RustcDev {
 614             compiler: run.builder.compiler_for(
 615                 run.builder.top_stage,
 616                 run.builder.config.build,
 617                 run.target,
 618             ),
 619             target: run.target,
 620         });
 621     }
 622
 623     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 624         let compiler = self.compiler;
 625         let target = self.target;
 626         if skip_host_target_lib(builder, compiler) {
 627             return None;
 628         }
 629
 630         builder.ensure(compile::Rustc { compiler, target });
 631
 632         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 633
 634         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 635         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 636         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 637
 638         let src_files = &["Cargo.lock"];
 639         // This is the reduced set of paths which will become the rustc-dev component
 640         // (essentially the compiler crates and all of their path dependencies).
 641         copy_src_dirs(
 642             builder,
 643             &builder.src,
 644             &["compiler"],
 645             &[],
 646             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 647         );
 648         // This particular crate is used as a build dependency of the above.
 649         copy_src_dirs(
 650             builder,
 651             &builder.src,
 652             &["src/build_helper"],
 653             &[],
 654             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 655         );
 656         for file in src_files {
 657             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 658         }
 659
 660         Some(tarball.generate())
 661     }
 662 }
 663
 664 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 665 pub struct Analysis {
 666     pub compiler: Compiler,
 667     pub target: TargetSelection,
 668 }
 669
 670 impl Step for Analysis {
 671     type Output = Option<GeneratedTarball>;
 672     const DEFAULT: bool = true;
 673
 674     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 675         let builder = run.builder;
 676         run.path("analysis").default_condition(builder.config.extended)
 677     }
 678
 679     fn make_run(run: RunConfig<'_>) {
 680         run.builder.ensure(Analysis {
 681             // Find the actual compiler (handling the full bootstrap option) which
 682             // produced the save-analysis data because that data isn't copied
 683             // through the sysroot uplifting.
 684             compiler: run.builder.compiler_for(
 685                 run.builder.top_stage,
 686                 run.builder.config.build,
 687                 run.target,
 688             ),
 689             target: run.target,
 690         });
 691     }
 692
 693     /// Creates a tarball of save-analysis metadata, if available.
 694     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 695         let compiler = self.compiler;
 696         let target = self.target;
 697         assert!(builder.config.extended);
 698         if compiler.host != builder.config.build {
 699             return None;
 700         }
 701
 702         builder.ensure(compile::Std { compiler, target });
 703         let src = builder
 704             .stage_out(compiler, Mode::Std)
 705             .join(target.triple)
 706             .join(builder.cargo_dir())
 707             .join("deps")
 708             .join("save-analysis");
 709
 710         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 711         tarball.include_target_in_component_name(true);
 712         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 713         Some(tarball.generate())
 714     }
 715 }
 716
 717 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 718 /// `dst_dir`.
 719 fn copy_src_dirs(
 720     builder: &Builder<'_>,
 721     base: &Path,
 722     src_dirs: &[&str],
 723     exclude_dirs: &[&str],
 724     dst_dir: &Path,
 725 ) {
 726     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 727         let spath = match path.to_str() {
 728             Some(path) => path,
 729             None => return false,
 730         };
 731         if spath.ends_with('~') || spath.ends_with(".pyc") {
 732             return false;
 733         }
 734
 735         const LLVM_PROJECTS: &[&str] = &[
 736             "llvm-project/clang",
 737             "llvm-project\\clang",
 738             "llvm-project/libunwind",
 739             "llvm-project\\libunwind",
 740             "llvm-project/lld",
 741             "llvm-project\\lld",
 742             "llvm-project/lldb",
 743             "llvm-project\\lldb",
 744             "llvm-project/llvm",
 745             "llvm-project\\llvm",
 746             "llvm-project/compiler-rt",
 747             "llvm-project\\compiler-rt",
 748         ];
 749         if spath.contains("llvm-project")
 750             && !spath.ends_with("llvm-project")
 751             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 752         {
 753             return false;
 754         }
 755
 756         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 757         if LLVM_TEST.iter().any(|path| spath.contains(path))
 758             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 759         {
 760             return false;
 761         }
 762
 763         let full_path = Path::new(dir).join(path);
 764         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 765             return false;
 766         }
 767
 768         let excludes = [
 769             "CVS",
 770             "RCS",
 771             "SCCS",
 772             ".git",
 773             ".gitignore",
 774             ".gitmodules",
 775             ".gitattributes",
 776             ".cvsignore",
 777             ".svn",
 778             ".arch-ids",
 779             "{arch}",
 780             "=RELEASE-ID",
 781             "=meta-update",
 782             "=update",
 783             ".bzr",
 784             ".bzrignore",
 785             ".bzrtags",
 786             ".hg",
 787             ".hgignore",
 788             ".hgrags",
 789             "_darcs",
 790         ];
 791         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 792     }
 793
 794     // Copy the directories using our filter
 795     for item in src_dirs {
 796         let dst = &dst_dir.join(item);
 797         t!(fs::create_dir_all(dst));
 798         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 799     }
 800 }
 801
 802 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 803 pub struct Src;
 804
 805 impl Step for Src {
 806     /// The output path of the src installer tarball
 807     type Output = GeneratedTarball;
 808     const DEFAULT: bool = true;
 809     const ONLY_HOSTS: bool = true;
 810
 811     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 812         run.path("src")
 813     }
 814
 815     fn make_run(run: RunConfig<'_>) {
 816         run.builder.ensure(Src);
 817     }
 818
 819     /// Creates the `rust-src` installer component
 820     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 821         let tarball = Tarball::new_targetless(builder, "rust-src");
 822
 823         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 824         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 825         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 826         // and fix them...
 827         //
 828         // NOTE: if you update the paths here, you also should update the "virtual" path
 829         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 830         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 831
 832         let src_files = ["Cargo.lock"];
 833         // This is the reduced set of paths which will become the rust-src component
 834         // (essentially libstd and all of its path dependencies).
 835         copy_src_dirs(
 836             builder,
 837             &builder.src,
 838             &["library", "src/llvm-project/libunwind"],
 839             &[
 840                 // not needed and contains symlinks which rustup currently
 841                 // chokes on when unpacking.
 842                 "library/backtrace/crates",
 843             ],
 844             &dst_src,
 845         );
 846         for file in src_files.iter() {
 847             builder.copy(&builder.src.join(file), &dst_src.join(file));
 848         }
 849
 850         tarball.generate()
 851     }
 852 }
 853
 854 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 855 pub struct PlainSourceTarball;
 856
 857 impl Step for PlainSourceTarball {
 858     /// Produces the location of the tarball generated
 859     type Output = GeneratedTarball;
 860     const DEFAULT: bool = true;
 861     const ONLY_HOSTS: bool = true;
 862
 863     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 864         let builder = run.builder;
 865         run.path("src").default_condition(builder.config.rust_dist_src)
 866     }
 867
 868     fn make_run(run: RunConfig<'_>) {
 869         run.builder.ensure(PlainSourceTarball);
 870     }
 871
 872     /// Creates the plain source tarball
 873     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 874         let tarball = Tarball::new(builder, "rustc", "src");
 875         let plain_dst_src = tarball.image_dir();
 876
 877         // This is the set of root paths which will become part of the source package
 878         let src_files = [
 879             "COPYRIGHT",
 880             "LICENSE-APACHE",
 881             "LICENSE-MIT",
 882             "CONTRIBUTING.md",
 883             "README.md",
 884             "RELEASES.md",
 885             "configure",
 886             "x.py",
 887             "config.toml.example",
 888             "Cargo.toml",
 889             "Cargo.lock",
 890         ];
 891         let src_dirs = ["src", "compiler", "library"];
 892
 893         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 894
 895         // Copy the files normally
 896         for item in &src_files {
 897             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 898         }
 899
 900         // Create the version file
 901         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 902         if let Some(sha) = builder.rust_sha() {
 903             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 904         }
 905
 906         // If we're building from git sources, we need to vendor a complete distribution.
 907         if builder.rust_info.is_git() {
 908             // Vendor all Cargo dependencies
 909             let mut cmd = Command::new(&builder.initial_cargo);
 910             cmd.arg("vendor")
 911                 .arg("--sync")
 912                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 913                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 914                 .current_dir(&plain_dst_src);
 915             builder.run(&mut cmd);
 916         }
 917
 918         tarball.bare()
 919     }
 920 }
 921
 922 // We have to run a few shell scripts, which choke quite a bit on both `\`
 923 // characters and on `C:\` paths, so normalize both of them away.
 924 pub fn sanitize_sh(path: &Path) -> String {
 925     let path = path.to_str().unwrap().replace("\\", "/");
 926     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 927
 928     fn unc_to_lfs(s: &str) -> &str {
 929         s.strip_prefix("//?/").unwrap_or(s)
 930     }
 931
 932     fn change_drive(s: &str) -> Option<String> {
 933         let mut ch = s.chars();
 934         let drive = ch.next().unwrap_or('C');
 935         if ch.next() != Some(':') {
 936             return None;
 937         }
 938         if ch.next() != Some('/') {
 939             return None;
 940         }
 941         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 942     }
 943 }
 944
 945 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 946 pub struct Cargo {
 947     pub compiler: Compiler,
 948     pub target: TargetSelection,
 949 }
 950
 951 impl Step for Cargo {
 952     type Output = GeneratedTarball;
 953     const ONLY_HOSTS: bool = true;
 954
 955     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 956         run.path("cargo")
 957     }
 958
 959     fn make_run(run: RunConfig<'_>) {
 960         run.builder.ensure(Cargo {
 961             compiler: run.builder.compiler_for(
 962                 run.builder.top_stage,
 963                 run.builder.config.build,
 964                 run.target,
 965             ),
 966             target: run.target,
 967         });
 968     }
 969
 970     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 971         let compiler = self.compiler;
 972         let target = self.target;
 973
 974         let cargo = builder.ensure(tool::Cargo { compiler, target });
 975         let src = builder.src.join("src/tools/cargo");
 976         let etc = src.join("src/etc");
 977
 978         // Prepare the image directory
 979         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 980         tarball.set_overlay(OverlayKind::Cargo);
 981
 982         tarball.add_file(&cargo, "bin", 0o755);
 983         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 984         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 985         tarball.add_dir(etc.join("man"), "share/man/man1");
 986         tarball.add_legal_and_readme_to("share/doc/cargo");
 987
 988         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 989             let dirent = dirent.expect("read dir entry");
 990             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 991                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 992             }
 993         }
 994
 995         tarball.generate()
 996     }
 997 }
 998
 999 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1000 pub struct Rls {
1001     pub compiler: Compiler,
1002     pub target: TargetSelection,
1003 }
1004
1005 impl Step for Rls {
1006     type Output = Option<GeneratedTarball>;
1007     const ONLY_HOSTS: bool = true;
1008
1009     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1010         run.path("rls")
1011     }
1012
1013     fn make_run(run: RunConfig<'_>) {
1014         run.builder.ensure(Rls {
1015             compiler: run.builder.compiler_for(
1016                 run.builder.top_stage,
1017                 run.builder.config.build,
1018                 run.target,
1019             ),
1020             target: run.target,
1021         });
1022     }
1023
1024     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1025         let compiler = self.compiler;
1026         let target = self.target;
1027         assert!(builder.config.extended);
1028
1029         let rls = builder
1030             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1031             .or_else(|| {
1032                 missing_tool("RLS", builder.build.config.missing_tools);
1033                 None
1034             })?;
1035
1036         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1037         tarball.set_overlay(OverlayKind::RLS);
1038         tarball.is_preview(true);
1039         tarball.add_file(rls, "bin", 0o755);
1040         tarball.add_legal_and_readme_to("share/doc/rls");
1041         Some(tarball.generate())
1042     }
1043 }
1044
1045 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1046 pub struct RustAnalyzer {
1047     pub compiler: Compiler,
1048     pub target: TargetSelection,
1049 }
1050
1051 impl Step for RustAnalyzer {
1052     type Output = Option<GeneratedTarball>;
1053     const ONLY_HOSTS: bool = true;
1054
1055     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1056         run.path("rust-analyzer")
1057     }
1058
1059     fn make_run(run: RunConfig<'_>) {
1060         run.builder.ensure(RustAnalyzer {
1061             compiler: run.builder.compiler_for(
1062                 run.builder.top_stage,
1063                 run.builder.config.build,
1064                 run.target,
1065             ),
1066             target: run.target,
1067         });
1068     }
1069
1070     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1071         let compiler = self.compiler;
1072         let target = self.target;
1073         assert!(builder.config.extended);
1074
1075         if target.contains("riscv64") {
1076             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1077             // to build. See #74813 for details.
1078             return None;
1079         }
1080
1081         let rust_analyzer = builder
1082             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1083             .expect("rust-analyzer always builds");
1084
1085         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1086         tarball.set_overlay(OverlayKind::RustAnalyzer);
1087         tarball.is_preview(true);
1088         tarball.add_file(rust_analyzer, "bin", 0o755);
1089         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1090         Some(tarball.generate())
1091     }
1092 }
1093
1094 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1095 pub struct Clippy {
1096     pub compiler: Compiler,
1097     pub target: TargetSelection,
1098 }
1099
1100 impl Step for Clippy {
1101     type Output = GeneratedTarball;
1102     const ONLY_HOSTS: bool = true;
1103
1104     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1105         run.path("clippy")
1106     }
1107
1108     fn make_run(run: RunConfig<'_>) {
1109         run.builder.ensure(Clippy {
1110             compiler: run.builder.compiler_for(
1111                 run.builder.top_stage,
1112                 run.builder.config.build,
1113                 run.target,
1114             ),
1115             target: run.target,
1116         });
1117     }
1118
1119     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1120         let compiler = self.compiler;
1121         let target = self.target;
1122         assert!(builder.config.extended);
1123
1124         // Prepare the image directory
1125         // We expect clippy to build, because we've exited this step above if tool
1126         // state for clippy isn't testing.
1127         let clippy = builder
1128             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1129             .expect("clippy expected to build - essential tool");
1130         let cargoclippy = builder
1131             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1132             .expect("clippy expected to build - essential tool");
1133
1134         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1135         tarball.set_overlay(OverlayKind::Clippy);
1136         tarball.is_preview(true);
1137         tarball.add_file(clippy, "bin", 0o755);
1138         tarball.add_file(cargoclippy, "bin", 0o755);
1139         tarball.add_legal_and_readme_to("share/doc/clippy");
1140         tarball.generate()
1141     }
1142 }
1143
1144 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1145 pub struct Miri {
1146     pub compiler: Compiler,
1147     pub target: TargetSelection,
1148 }
1149
1150 impl Step for Miri {
1151     type Output = Option<GeneratedTarball>;
1152     const ONLY_HOSTS: bool = true;
1153
1154     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1155         run.path("miri")
1156     }
1157
1158     fn make_run(run: RunConfig<'_>) {
1159         run.builder.ensure(Miri {
1160             compiler: run.builder.compiler_for(
1161                 run.builder.top_stage,
1162                 run.builder.config.build,
1163                 run.target,
1164             ),
1165             target: run.target,
1166         });
1167     }
1168
1169     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1170         let compiler = self.compiler;
1171         let target = self.target;
1172         assert!(builder.config.extended);
1173
1174         let miri = builder
1175             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1176             .or_else(|| {
1177                 missing_tool("miri", builder.build.config.missing_tools);
1178                 None
1179             })?;
1180         let cargomiri = builder
1181             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1182             .or_else(|| {
1183                 missing_tool("cargo miri", builder.build.config.missing_tools);
1184                 None
1185             })?;
1186
1187         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1188         tarball.set_overlay(OverlayKind::Miri);
1189         tarball.is_preview(true);
1190         tarball.add_file(miri, "bin", 0o755);
1191         tarball.add_file(cargomiri, "bin", 0o755);
1192         tarball.add_legal_and_readme_to("share/doc/miri");
1193         Some(tarball.generate())
1194     }
1195 }
1196
1197 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1198 pub struct Rustfmt {
1199     pub compiler: Compiler,
1200     pub target: TargetSelection,
1201 }
1202
1203 impl Step for Rustfmt {
1204     type Output = Option<GeneratedTarball>;
1205     const ONLY_HOSTS: bool = true;
1206
1207     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1208         run.path("rustfmt")
1209     }
1210
1211     fn make_run(run: RunConfig<'_>) {
1212         run.builder.ensure(Rustfmt {
1213             compiler: run.builder.compiler_for(
1214                 run.builder.top_stage,
1215                 run.builder.config.build,
1216                 run.target,
1217             ),
1218             target: run.target,
1219         });
1220     }
1221
1222     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1223         let compiler = self.compiler;
1224         let target = self.target;
1225
1226         let rustfmt = builder
1227             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1228             .or_else(|| {
1229                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1230                 None
1231             })?;
1232         let cargofmt = builder
1233             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1234             .or_else(|| {
1235                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1236                 None
1237             })?;
1238
1239         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1240         tarball.set_overlay(OverlayKind::Rustfmt);
1241         tarball.is_preview(true);
1242         tarball.add_file(rustfmt, "bin", 0o755);
1243         tarball.add_file(cargofmt, "bin", 0o755);
1244         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1245         Some(tarball.generate())
1246     }
1247 }
1248
1249 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1250 pub struct RustDemangler {
1251     pub compiler: Compiler,
1252     pub target: TargetSelection,
1253 }
1254
1255 impl Step for RustDemangler {
1256     type Output = Option<GeneratedTarball>;
1257     const ONLY_HOSTS: bool = true;
1258
1259     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1260         run.path("rust-demangler")
1261     }
1262
1263     fn make_run(run: RunConfig<'_>) {
1264         run.builder.ensure(RustDemangler {
1265             compiler: run.builder.compiler_for(
1266                 run.builder.top_stage,
1267                 run.builder.config.build,
1268                 run.target,
1269             ),
1270             target: run.target,
1271         });
1272     }
1273
1274     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1275         let compiler = self.compiler;
1276         let target = self.target;
1277         assert!(builder.config.extended);
1278
1279         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1280         let profiler = builder.config.profiler_enabled(target);
1281         if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
1282             return None;
1283         }
1284
1285         let rust_demangler = builder
1286             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1287             .expect("rust-demangler expected to build - in-tree tool");
1288
1289         // Prepare the image directory
1290         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1291         tarball.set_overlay(OverlayKind::RustDemangler);
1292         tarball.is_preview(true);
1293         tarball.add_file(&rust_demangler, "bin", 0o755);
1294         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1295         Some(tarball.generate())
1296     }
1297 }
1298
1299 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1300 pub struct Extended {
1301     stage: u32,
1302     host: TargetSelection,
1303     target: TargetSelection,
1304 }
1305
1306 impl Step for Extended {
1307     type Output = ();
1308     const DEFAULT: bool = true;
1309     const ONLY_HOSTS: bool = true;
1310
1311     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1312         let builder = run.builder;
1313         run.path("extended").default_condition(builder.config.extended)
1314     }
1315
1316     fn make_run(run: RunConfig<'_>) {
1317         run.builder.ensure(Extended {
1318             stage: run.builder.top_stage,
1319             host: run.builder.config.build,
1320             target: run.target,
1321         });
1322     }
1323
1324     /// Creates a combined installer for the specified target in the provided stage.
1325     fn run(self, builder: &Builder<'_>) {
1326         let target = self.target;
1327         let stage = self.stage;
1328         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1329
1330         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1331
1332         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1333         let cargo_installer = builder.ensure(Cargo { compiler, target });
1334         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1335         let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
1336         let rls_installer = builder.ensure(Rls { compiler, target });
1337         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1338         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1339         let clippy_installer = builder.ensure(Clippy { compiler, target });
1340         let miri_installer = builder.ensure(Miri { compiler, target });
1341         let mingw_installer = builder.ensure(Mingw { host: target });
1342         let analysis_installer = builder.ensure(Analysis { compiler, target });
1343
1344         let docs_installer = builder.ensure(Docs { host: target });
1345         let std_installer = builder.ensure(Std { compiler, target });
1346
1347         let etc = builder.src.join("src/etc/installer");
1348
1349         // Avoid producing tarballs during a dry run.
1350         if builder.config.dry_run {
1351             return;
1352         }
1353
1354         // When rust-std package split from rustc, we needed to ensure that during
1355         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1356         // the std files during uninstall. To do this ensure that rustc comes
1357         // before rust-std in the list below.
1358         let mut tarballs = Vec::new();
1359         tarballs.push(rustc_installer);
1360         tarballs.push(cargo_installer);
1361         tarballs.push(clippy_installer);
1362         tarballs.extend(rust_demangler_installer.clone());
1363         tarballs.extend(rls_installer.clone());
1364         tarballs.extend(rust_analyzer_installer.clone());
1365         tarballs.extend(miri_installer.clone());
1366         tarballs.extend(rustfmt_installer.clone());
1367         tarballs.extend(llvm_tools_installer);
1368         if let Some(analysis_installer) = analysis_installer {
1369             tarballs.push(analysis_installer);
1370         }
1371         tarballs.push(std_installer.expect("missing std"));
1372         if let Some(docs_installer) = docs_installer {
1373             tarballs.push(docs_installer);
1374         }
1375         if target.contains("pc-windows-gnu") {
1376             tarballs.push(mingw_installer.unwrap());
1377         }
1378
1379         let tarball = Tarball::new(builder, "rust", &target.triple);
1380         let generated = tarball.combine(&tarballs);
1381
1382         let tmp = tmpdir(builder).join("combined-tarball");
1383         let work = generated.work_dir();
1384
1385         let mut license = String::new();
1386         license += &builder.read(&builder.src.join("COPYRIGHT"));
1387         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1388         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1389         license.push('\n');
1390         license.push('\n');
1391
1392         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1393         let mut rtf = rtf.to_string();
1394         rtf.push('\n');
1395         for line in license.lines() {
1396             rtf.push_str(line);
1397             rtf.push_str("\\line ");
1398         }
1399         rtf.push('}');
1400
1401         fn filter(contents: &str, marker: &str) -> String {
1402             let start = format!("tool-{}-start", marker);
1403             let end = format!("tool-{}-end", marker);
1404             let mut lines = Vec::new();
1405             let mut omitted = false;
1406             for line in contents.lines() {
1407                 if line.contains(&start) {
1408                     omitted = true;
1409                 } else if line.contains(&end) {
1410                     omitted = false;
1411                 } else if !omitted {
1412                     lines.push(line);
1413                 }
1414             }
1415
1416             lines.join("\n")
1417         }
1418
1419         let xform = |p: &Path| {
1420             let mut contents = t!(fs::read_to_string(p));
1421             if rust_demangler_installer.is_none() {
1422                 contents = filter(&contents, "rust-demangler");
1423             }
1424             if rls_installer.is_none() {
1425                 contents = filter(&contents, "rls");
1426             }
1427             if rust_analyzer_installer.is_none() {
1428                 contents = filter(&contents, "rust-analyzer");
1429             }
1430             if miri_installer.is_none() {
1431                 contents = filter(&contents, "miri");
1432             }
1433             if rustfmt_installer.is_none() {
1434                 contents = filter(&contents, "rustfmt");
1435             }
1436             let ret = tmp.join(p.file_name().unwrap());
1437             t!(fs::write(&ret, &contents));
1438             ret
1439         };
1440
1441         if target.contains("apple-darwin") {
1442             builder.info("building pkg installer");
1443             let pkg = tmp.join("pkg");
1444             let _ = fs::remove_dir_all(&pkg);
1445
1446             let pkgbuild = |component: &str| {
1447                 let mut cmd = Command::new("pkgbuild");
1448                 cmd.arg("--identifier")
1449                     .arg(format!("org.rust-lang.{}", component))
1450                     .arg("--scripts")
1451                     .arg(pkg.join(component))
1452                     .arg("--nopayload")
1453                     .arg(pkg.join(component).with_extension("pkg"));
1454                 builder.run(&mut cmd);
1455             };
1456
1457             let prepare = |name: &str| {
1458                 builder.create_dir(&pkg.join(name));
1459                 builder.cp_r(
1460                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1461                     &pkg.join(name),
1462                 );
1463                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1464                 pkgbuild(name);
1465             };
1466             prepare("rustc");
1467             prepare("cargo");
1468             prepare("rust-docs");
1469             prepare("rust-std");
1470             prepare("rust-analysis");
1471             prepare("clippy");
1472             if rust_demangler_installer.is_some() {
1473                 prepare("rust-demangler");
1474             }
1475             if rls_installer.is_some() {
1476                 prepare("rls");
1477             }
1478             if rust_analyzer_installer.is_some() {
1479                 prepare("rust-analyzer");
1480             }
1481             if miri_installer.is_some() {
1482                 prepare("miri");
1483             }
1484
1485             // create an 'uninstall' package
1486             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1487             pkgbuild("uninstall");
1488
1489             builder.create_dir(&pkg.join("res"));
1490             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1491             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1492             let mut cmd = Command::new("productbuild");
1493             cmd.arg("--distribution")
1494                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1495                 .arg("--resources")
1496                 .arg(pkg.join("res"))
1497                 .arg(distdir(builder).join(format!(
1498                     "{}-{}.pkg",
1499                     pkgname(builder, "rust"),
1500                     target.triple
1501                 )))
1502                 .arg("--package-path")
1503                 .arg(&pkg);
1504             let _time = timeit(builder);
1505             builder.run(&mut cmd);
1506         }
1507
1508         if target.contains("windows") {
1509             let exe = tmp.join("exe");
1510             let _ = fs::remove_dir_all(&exe);
1511
1512             let prepare = |name: &str| {
1513                 builder.create_dir(&exe.join(name));
1514                 let dir = if name == "rust-std" || name == "rust-analysis" {
1515                     format!("{}-{}", name, target.triple)
1516                 } else if name == "rls" {
1517                     "rls-preview".to_string()
1518                 } else if name == "rust-analyzer" {
1519                     "rust-analyzer-preview".to_string()
1520                 } else if name == "clippy" {
1521                     "clippy-preview".to_string()
1522                 } else if name == "rust-demangler" {
1523                     "rust-demangler-preview".to_string()
1524                 } else if name == "miri" {
1525                     "miri-preview".to_string()
1526                 } else {
1527                     name.to_string()
1528                 };
1529                 builder.cp_r(
1530                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1531                     &exe.join(name),
1532                 );
1533                 builder.remove(&exe.join(name).join("manifest.in"));
1534             };
1535             prepare("rustc");
1536             prepare("cargo");
1537             prepare("rust-analysis");
1538             prepare("rust-docs");
1539             prepare("rust-std");
1540             prepare("clippy");
1541             if rust_demangler_installer.is_some() {
1542                 prepare("rust-demangler");
1543             }
1544             if rls_installer.is_some() {
1545                 prepare("rls");
1546             }
1547             if rust_analyzer_installer.is_some() {
1548                 prepare("rust-analyzer");
1549             }
1550             if miri_installer.is_some() {
1551                 prepare("miri");
1552             }
1553             if target.contains("windows-gnu") {
1554                 prepare("rust-mingw");
1555             }
1556
1557             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1558
1559             // Generate msi installer
1560             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1561             let heat = wix.join("bin/heat.exe");
1562             let candle = wix.join("bin/candle.exe");
1563             let light = wix.join("bin/light.exe");
1564
1565             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1566             builder.run(
1567                 Command::new(&heat)
1568                     .current_dir(&exe)
1569                     .arg("dir")
1570                     .arg("rustc")
1571                     .args(&heat_flags)
1572                     .arg("-cg")
1573                     .arg("RustcGroup")
1574                     .arg("-dr")
1575                     .arg("Rustc")
1576                     .arg("-var")
1577                     .arg("var.RustcDir")
1578                     .arg("-out")
1579                     .arg(exe.join("RustcGroup.wxs")),
1580             );
1581             builder.run(
1582                 Command::new(&heat)
1583                     .current_dir(&exe)
1584                     .arg("dir")
1585                     .arg("rust-docs")
1586                     .args(&heat_flags)
1587                     .arg("-cg")
1588                     .arg("DocsGroup")
1589                     .arg("-dr")
1590                     .arg("Docs")
1591                     .arg("-var")
1592                     .arg("var.DocsDir")
1593                     .arg("-out")
1594                     .arg(exe.join("DocsGroup.wxs"))
1595                     .arg("-t")
1596                     .arg(etc.join("msi/squash-components.xsl")),
1597             );
1598             builder.run(
1599                 Command::new(&heat)
1600                     .current_dir(&exe)
1601                     .arg("dir")
1602                     .arg("cargo")
1603                     .args(&heat_flags)
1604                     .arg("-cg")
1605                     .arg("CargoGroup")
1606                     .arg("-dr")
1607                     .arg("Cargo")
1608                     .arg("-var")
1609                     .arg("var.CargoDir")
1610                     .arg("-out")
1611                     .arg(exe.join("CargoGroup.wxs"))
1612                     .arg("-t")
1613                     .arg(etc.join("msi/remove-duplicates.xsl")),
1614             );
1615             builder.run(
1616                 Command::new(&heat)
1617                     .current_dir(&exe)
1618                     .arg("dir")
1619                     .arg("rust-std")
1620                     .args(&heat_flags)
1621                     .arg("-cg")
1622                     .arg("StdGroup")
1623                     .arg("-dr")
1624                     .arg("Std")
1625                     .arg("-var")
1626                     .arg("var.StdDir")
1627                     .arg("-out")
1628                     .arg(exe.join("StdGroup.wxs")),
1629             );
1630             if rls_installer.is_some() {
1631                 builder.run(
1632                     Command::new(&heat)
1633                         .current_dir(&exe)
1634                         .arg("dir")
1635                         .arg("rls")
1636                         .args(&heat_flags)
1637                         .arg("-cg")
1638                         .arg("RlsGroup")
1639                         .arg("-dr")
1640                         .arg("Rls")
1641                         .arg("-var")
1642                         .arg("var.RlsDir")
1643                         .arg("-out")
1644                         .arg(exe.join("RlsGroup.wxs"))
1645                         .arg("-t")
1646                         .arg(etc.join("msi/remove-duplicates.xsl")),
1647                 );
1648             }
1649             if rust_analyzer_installer.is_some() {
1650                 builder.run(
1651                     Command::new(&heat)
1652                         .current_dir(&exe)
1653                         .arg("dir")
1654                         .arg("rust-analyzer")
1655                         .args(&heat_flags)
1656                         .arg("-cg")
1657                         .arg("RustAnalyzerGroup")
1658                         .arg("-dr")
1659                         .arg("RustAnalyzer")
1660                         .arg("-var")
1661                         .arg("var.RustAnalyzerDir")
1662                         .arg("-out")
1663                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1664                         .arg("-t")
1665                         .arg(etc.join("msi/remove-duplicates.xsl")),
1666                 );
1667             }
1668             builder.run(
1669                 Command::new(&heat)
1670                     .current_dir(&exe)
1671                     .arg("dir")
1672                     .arg("clippy")
1673                     .args(&heat_flags)
1674                     .arg("-cg")
1675                     .arg("ClippyGroup")
1676                     .arg("-dr")
1677                     .arg("Clippy")
1678                     .arg("-var")
1679                     .arg("var.ClippyDir")
1680                     .arg("-out")
1681                     .arg(exe.join("ClippyGroup.wxs"))
1682                     .arg("-t")
1683                     .arg(etc.join("msi/remove-duplicates.xsl")),
1684             );
1685             if rust_demangler_installer.is_some() {
1686                 builder.run(
1687                     Command::new(&heat)
1688                         .current_dir(&exe)
1689                         .arg("dir")
1690                         .arg("rust-demangler")
1691                         .args(&heat_flags)
1692                         .arg("-cg")
1693                         .arg("RustDemanglerGroup")
1694                         .arg("-dr")
1695                         .arg("RustDemangler")
1696                         .arg("-var")
1697                         .arg("var.RustDemanglerDir")
1698                         .arg("-out")
1699                         .arg(exe.join("RustDemanglerGroup.wxs"))
1700                         .arg("-t")
1701                         .arg(etc.join("msi/remove-duplicates.xsl")),
1702                 );
1703             }
1704             if miri_installer.is_some() {
1705                 builder.run(
1706                     Command::new(&heat)
1707                         .current_dir(&exe)
1708                         .arg("dir")
1709                         .arg("miri")
1710                         .args(&heat_flags)
1711                         .arg("-cg")
1712                         .arg("MiriGroup")
1713                         .arg("-dr")
1714                         .arg("Miri")
1715                         .arg("-var")
1716                         .arg("var.MiriDir")
1717                         .arg("-out")
1718                         .arg(exe.join("MiriGroup.wxs"))
1719                         .arg("-t")
1720                         .arg(etc.join("msi/remove-duplicates.xsl")),
1721                 );
1722             }
1723             builder.run(
1724                 Command::new(&heat)
1725                     .current_dir(&exe)
1726                     .arg("dir")
1727                     .arg("rust-analysis")
1728                     .args(&heat_flags)
1729                     .arg("-cg")
1730                     .arg("AnalysisGroup")
1731                     .arg("-dr")
1732                     .arg("Analysis")
1733                     .arg("-var")
1734                     .arg("var.AnalysisDir")
1735                     .arg("-out")
1736                     .arg(exe.join("AnalysisGroup.wxs"))
1737                     .arg("-t")
1738                     .arg(etc.join("msi/remove-duplicates.xsl")),
1739             );
1740             if target.contains("windows-gnu") {
1741                 builder.run(
1742                     Command::new(&heat)
1743                         .current_dir(&exe)
1744                         .arg("dir")
1745                         .arg("rust-mingw")
1746                         .args(&heat_flags)
1747                         .arg("-cg")
1748                         .arg("GccGroup")
1749                         .arg("-dr")
1750                         .arg("Gcc")
1751                         .arg("-var")
1752                         .arg("var.GccDir")
1753                         .arg("-out")
1754                         .arg(exe.join("GccGroup.wxs")),
1755                 );
1756             }
1757
1758             let candle = |input: &Path| {
1759                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1760                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1761                 let mut cmd = Command::new(&candle);
1762                 cmd.current_dir(&exe)
1763                     .arg("-nologo")
1764                     .arg("-dRustcDir=rustc")
1765                     .arg("-dDocsDir=rust-docs")
1766                     .arg("-dCargoDir=cargo")
1767                     .arg("-dStdDir=rust-std")
1768                     .arg("-dAnalysisDir=rust-analysis")
1769                     .arg("-dClippyDir=clippy")
1770                     .arg("-arch")
1771                     .arg(&arch)
1772                     .arg("-out")
1773                     .arg(&output)
1774                     .arg(&input);
1775                 add_env(builder, &mut cmd, target);
1776
1777                 if rust_demangler_installer.is_some() {
1778                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1779                 }
1780                 if rls_installer.is_some() {
1781                     cmd.arg("-dRlsDir=rls");
1782                 }
1783                 if rust_analyzer_installer.is_some() {
1784                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1785                 }
1786                 if miri_installer.is_some() {
1787                     cmd.arg("-dMiriDir=miri");
1788                 }
1789                 if target.contains("windows-gnu") {
1790                     cmd.arg("-dGccDir=rust-mingw");
1791                 }
1792                 builder.run(&mut cmd);
1793             };
1794             candle(&xform(&etc.join("msi/rust.wxs")));
1795             candle(&etc.join("msi/ui.wxs"));
1796             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1797             candle("RustcGroup.wxs".as_ref());
1798             candle("DocsGroup.wxs".as_ref());
1799             candle("CargoGroup.wxs".as_ref());
1800             candle("StdGroup.wxs".as_ref());
1801             candle("ClippyGroup.wxs".as_ref());
1802             if rust_demangler_installer.is_some() {
1803                 candle("RustDemanglerGroup.wxs".as_ref());
1804             }
1805             if rls_installer.is_some() {
1806                 candle("RlsGroup.wxs".as_ref());
1807             }
1808             if rust_analyzer_installer.is_some() {
1809                 candle("RustAnalyzerGroup.wxs".as_ref());
1810             }
1811             if miri_installer.is_some() {
1812                 candle("MiriGroup.wxs".as_ref());
1813             }
1814             candle("AnalysisGroup.wxs".as_ref());
1815
1816             if target.contains("windows-gnu") {
1817                 candle("GccGroup.wxs".as_ref());
1818             }
1819
1820             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1821             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1822             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1823
1824             builder.info(&format!("building `msi` installer with {:?}", light));
1825             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1826             let mut cmd = Command::new(&light);
1827             cmd.arg("-nologo")
1828                 .arg("-ext")
1829                 .arg("WixUIExtension")
1830                 .arg("-ext")
1831                 .arg("WixUtilExtension")
1832                 .arg("-out")
1833                 .arg(exe.join(&filename))
1834                 .arg("rust.wixobj")
1835                 .arg("ui.wixobj")
1836                 .arg("rustwelcomedlg.wixobj")
1837                 .arg("RustcGroup.wixobj")
1838                 .arg("DocsGroup.wixobj")
1839                 .arg("CargoGroup.wixobj")
1840                 .arg("StdGroup.wixobj")
1841                 .arg("AnalysisGroup.wixobj")
1842                 .arg("ClippyGroup.wixobj")
1843                 .current_dir(&exe);
1844
1845             if rls_installer.is_some() {
1846                 cmd.arg("RlsGroup.wixobj");
1847             }
1848             if rust_analyzer_installer.is_some() {
1849                 cmd.arg("RustAnalyzerGroup.wixobj");
1850             }
1851             if rust_demangler_installer.is_some() {
1852                 cmd.arg("RustDemanglerGroup.wixobj");
1853             }
1854             if miri_installer.is_some() {
1855                 cmd.arg("MiriGroup.wixobj");
1856             }
1857
1858             if target.contains("windows-gnu") {
1859                 cmd.arg("GccGroup.wixobj");
1860             }
1861             // ICE57 wrongly complains about the shortcuts
1862             cmd.arg("-sice:ICE57");
1863
1864             let _time = timeit(builder);
1865             builder.run(&mut cmd);
1866
1867             if !builder.config.dry_run {
1868                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1869             }
1870         }
1871     }
1872 }
1873
1874 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1875     let mut parts = builder.version.split('.');
1876     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1877         .env("CFG_RELEASE_NUM", &builder.version)
1878         .env("CFG_RELEASE", builder.rust_release())
1879         .env("CFG_VER_MAJOR", parts.next().unwrap())
1880         .env("CFG_VER_MINOR", parts.next().unwrap())
1881         .env("CFG_VER_PATCH", parts.next().unwrap())
1882         .env("CFG_VER_BUILD", "0") // just needed to build
1883         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1884         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1885         .env("CFG_BUILD", target.triple)
1886         .env("CFG_CHANNEL", &builder.config.channel);
1887
1888     if target.contains("windows-gnu") {
1889         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1890     } else {
1891         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1892     }
1893
1894     if target.contains("x86_64") {
1895         cmd.env("CFG_PLATFORM", "x64");
1896     } else {
1897         cmd.env("CFG_PLATFORM", "x86");
1898     }
1899 }
1900
1901 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1902 ///
1903
1904 /// Returns whether the files were actually copied.
1905 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1906     if let Some(config) = builder.config.target_config.get(&target) {
1907         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1908             // If the LLVM was externally provided, then we don't currently copy
1909             // artifacts into the sysroot. This is not necessarily the right
1910             // choice (in particular, it will require the LLVM dylib to be in
1911             // the linker's load path at runtime), but the common use case for
1912             // external LLVMs is distribution provided LLVMs, and in that case
1913             // they're usually in the standard search path (e.g., /usr/lib) and
1914             // copying them here is going to cause problems as we may end up
1915             // with the wrong files and isn't what distributions want.
1916             //
1917             // This behavior may be revisited in the future though.
1918             //
1919             // If the LLVM is coming from ourselves (just from CI) though, we
1920             // still want to install it, as it otherwise won't be available.
1921             return false;
1922         }
1923     }
1924
1925     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1926     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1927     // clear why this is the case, though. llvm-config will emit the versioned
1928     // paths and we don't want those in the sysroot (as we're expecting
1929     // unversioned paths).
1930     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1931         let src_libdir = builder.llvm_out(target).join("lib");
1932         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1933         if llvm_dylib_path.exists() {
1934             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1935         }
1936         !builder.config.dry_run
1937     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1938         let mut cmd = Command::new(llvm_config);
1939         cmd.arg("--libfiles");
1940         builder.verbose(&format!("running {:?}", cmd));
1941         let files = output(&mut cmd);
1942         for file in files.trim_end().split(' ') {
1943             builder.install(Path::new(file), dst_libdir, 0o644);
1944         }
1945         !builder.config.dry_run
1946     } else {
1947         false
1948     }
1949 }
1950
1951 /// Maybe add libLLVM.so to the target lib-dir for linking.
1952 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1953     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1954     // We do not need to copy LLVM files into the sysroot if it is not
1955     // dynamically linked; it is already included into librustc_llvm
1956     // statically.
1957     if builder.config.llvm_link_shared {
1958         maybe_install_llvm(builder, target, &dst_libdir);
1959     }
1960 }
1961
1962 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1963 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1964     let dst_libdir =
1965         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1966     // We do not need to copy LLVM files into the sysroot if it is not
1967     // dynamically linked; it is already included into librustc_llvm
1968     // statically.
1969     if builder.config.llvm_link_shared {
1970         maybe_install_llvm(builder, target, &dst_libdir);
1971     }
1972 }
1973
1974 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1975 pub struct LlvmTools {
1976     pub target: TargetSelection,
1977 }
1978
1979 impl Step for LlvmTools {
1980     type Output = Option<GeneratedTarball>;
1981     const ONLY_HOSTS: bool = true;
1982
1983     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1984         run.path("llvm-tools")
1985     }
1986
1987     fn make_run(run: RunConfig<'_>) {
1988         run.builder.ensure(LlvmTools { target: run.target });
1989     }
1990
1991     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1992         let target = self.target;
1993         assert!(builder.config.extended);
1994
1995         /* run only if llvm-config isn't used */
1996         if let Some(config) = builder.config.target_config.get(&target) {
1997             if let Some(ref _s) = config.llvm_config {
1998                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1999                 return None;
2000             }
2001         }
2002
2003         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2004         tarball.set_overlay(OverlayKind::LLVM);
2005         tarball.is_preview(true);
2006
2007         // Prepare the image directory
2008         let src_bindir = builder.llvm_out(target).join("bin");
2009         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2010         for tool in LLVM_TOOLS {
2011             let exe = src_bindir.join(exe(tool, target));
2012             tarball.add_file(&exe, &dst_bindir, 0o755);
2013         }
2014
2015         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2016         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2017         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2018         // compiler libraries.
2019         maybe_install_llvm_target(builder, target, tarball.image_dir());
2020
2021         Some(tarball.generate())
2022     }
2023 }
2024
2025 // Tarball intended for internal consumption to ease rustc/std development.
2026 //
2027 // Should not be considered stable by end users.
2028 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2029 pub struct RustDev {
2030     pub target: TargetSelection,
2031 }
2032
2033 impl Step for RustDev {
2034     type Output = Option<GeneratedTarball>;
2035     const DEFAULT: bool = true;
2036     const ONLY_HOSTS: bool = true;
2037
2038     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2039         run.path("rust-dev")
2040     }
2041
2042     fn make_run(run: RunConfig<'_>) {
2043         run.builder.ensure(RustDev { target: run.target });
2044     }
2045
2046     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2047         let target = self.target;
2048
2049         /* run only if llvm-config isn't used */
2050         if let Some(config) = builder.config.target_config.get(&target) {
2051             if let Some(ref _s) = config.llvm_config {
2052                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2053                 return None;
2054             }
2055         }
2056
2057         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2058         tarball.set_overlay(OverlayKind::LLVM);
2059
2060         let src_bindir = builder.llvm_out(target).join("bin");
2061         for bin in &[
2062             "llvm-config",
2063             "llvm-ar",
2064             "llvm-objdump",
2065             "llvm-profdata",
2066             "llvm-bcanalyzer",
2067             "llvm-cov",
2068             "llvm-dwp",
2069         ] {
2070             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2071         }
2072         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2073
2074         // Copy the include directory as well; needed mostly to build
2075         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2076         // just broadly useful to be able to link against the bundled LLVM.
2077         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2078
2079         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2080         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2081         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2082         // compiler libraries.
2083         let dst_libdir = tarball.image_dir().join("lib");
2084         maybe_install_llvm(builder, target, &dst_libdir);
2085         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2086         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2087
2088         Some(tarball.generate())
2089     }
2090 }
2091
2092 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2093 /// release process to avoid cloning the monorepo and building stuff.
2094 ///
2095 /// Should not be considered stable by end users.
2096 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2097 pub struct BuildManifest {
2098     pub target: TargetSelection,
2099 }
2100
2101 impl Step for BuildManifest {
2102     type Output = GeneratedTarball;
2103     const DEFAULT: bool = false;
2104     const ONLY_HOSTS: bool = true;
2105
2106     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2107         run.path("src/tools/build-manifest")
2108     }
2109
2110     fn make_run(run: RunConfig<'_>) {
2111         run.builder.ensure(BuildManifest { target: run.target });
2112     }
2113
2114     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2115         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2116
2117         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2118         tarball.add_file(&build_manifest, "bin", 0o755);
2119         tarball.generate()
2120     }
2121 }
2122
2123 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2124 ///
2125 /// Currently this is the PGO profile data.
2126 ///
2127 /// Should not be considered stable by end users.
2128 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2129 pub struct ReproducibleArtifacts {
2130     pub target: TargetSelection,
2131 }
2132
2133 impl Step for ReproducibleArtifacts {
2134     type Output = Option<GeneratedTarball>;
2135     const DEFAULT: bool = true;
2136     const ONLY_HOSTS: bool = true;
2137
2138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2139         run.path("reproducible")
2140     }
2141
2142     fn make_run(run: RunConfig<'_>) {
2143         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2144     }
2145
2146     fn run(self, builder: &Builder<'_>) -> Self::Output {
2147         let path = builder.config.rust_profile_use.as_ref()?;
2148
2149         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2150         tarball.add_file(path, ".", 0o644);
2151         Some(tarball.generate())
2152     }
2153 }