src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let rust_lld = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 405                 // for `-Z gcc-ld=lld`
 406                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 407                 t!(fs::create_dir(&gcc_lld_dir));
 408                 builder.copy(&src_dir.join(&rust_lld), &gcc_lld_dir.join(exe("ld", compiler.host)));
 409             }
 410
 411             // Copy over llvm-dwp if it's there
 412             let exe = exe("rust-llvm-dwp", compiler.host);
 413             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 414
 415             // Man pages
 416             t!(fs::create_dir_all(image.join("share/man/man1")));
 417             let man_src = builder.src.join("src/doc/man");
 418             let man_dst = image.join("share/man/man1");
 419
 420             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 421             let time = env::var("SOURCE_DATE_EPOCH")
 422                 .map(|timestamp| {
 423                     let epoch = timestamp
 424                         .parse()
 425                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 426                         .unwrap();
 427
 428                     time::at(Timespec::new(epoch, 0))
 429                 })
 430                 .unwrap_or_else(|_| time::now());
 431
 432             let month_year = t!(time::strftime("%B %Y", &time));
 433             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 434             // to hardlink, and we don't want to edit the source templates
 435             for file_entry in builder.read_dir(&man_src) {
 436                 let page_src = file_entry.path();
 437                 let page_dst = man_dst.join(file_entry.file_name());
 438                 t!(fs::copy(&page_src, &page_dst));
 439                 // template in month/year and version number
 440                 builder.replace_in_file(
 441                     &page_dst,
 442                     &[
 443                         ("<INSERT DATE HERE>", &month_year),
 444                         ("<INSERT VERSION HERE>", &builder.version),
 445                     ],
 446                 );
 447             }
 448
 449             // Debugger scripts
 450             builder
 451                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 452
 453             // Misc license info
 454             let cp = |file: &str| {
 455                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 456             };
 457             cp("COPYRIGHT");
 458             cp("LICENSE-APACHE");
 459             cp("LICENSE-MIT");
 460             cp("README.md");
 461         }
 462     }
 463 }
 464
 465 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 466 pub struct DebuggerScripts {
 467     pub sysroot: Interned<PathBuf>,
 468     pub host: TargetSelection,
 469 }
 470
 471 impl Step for DebuggerScripts {
 472     type Output = ();
 473
 474     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 475         run.path("src/lldb_batchmode.py")
 476     }
 477
 478     fn make_run(run: RunConfig<'_>) {
 479         run.builder.ensure(DebuggerScripts {
 480             sysroot: run
 481                 .builder
 482                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 483             host: run.target,
 484         });
 485     }
 486
 487     /// Copies debugger scripts for `target` into the `sysroot` specified.
 488     fn run(self, builder: &Builder<'_>) {
 489         let host = self.host;
 490         let sysroot = self.sysroot;
 491         let dst = sysroot.join("lib/rustlib/etc");
 492         t!(fs::create_dir_all(&dst));
 493         let cp_debugger_script = |file: &str| {
 494             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 495         };
 496         if host.contains("windows-msvc") {
 497             // windbg debugger scripts
 498             builder.install(
 499                 &builder.src.join("src/etc/rust-windbg.cmd"),
 500                 &sysroot.join("bin"),
 501                 0o755,
 502             );
 503
 504             cp_debugger_script("natvis/intrinsic.natvis");
 505             cp_debugger_script("natvis/liballoc.natvis");
 506             cp_debugger_script("natvis/libcore.natvis");
 507             cp_debugger_script("natvis/libstd.natvis");
 508         } else {
 509             cp_debugger_script("rust_types.py");
 510
 511             // gdb debugger scripts
 512             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 513             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 514
 515             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 516             cp_debugger_script("gdb_lookup.py");
 517             cp_debugger_script("gdb_providers.py");
 518
 519             // lldb debugger scripts
 520             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 521
 522             cp_debugger_script("lldb_lookup.py");
 523             cp_debugger_script("lldb_providers.py");
 524             cp_debugger_script("lldb_commands")
 525         }
 526     }
 527 }
 528
 529 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 530     // The only true set of target libraries came from the build triple, so
 531     // let's reduce redundant work by only producing archives from that host.
 532     if compiler.host != builder.config.build {
 533         builder.info("\tskipping, not a build host");
 534         true
 535     } else {
 536         false
 537     }
 538 }
 539
 540 /// Copy stamped files into an image's `target/lib` directory.
 541 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 542     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 543     let self_contained_dst = dst.join("self-contained");
 544     t!(fs::create_dir_all(&dst));
 545     t!(fs::create_dir_all(&self_contained_dst));
 546     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 547         if dependency_type == DependencyType::TargetSelfContained {
 548             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 549         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 550             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: TargetSelection,
 559 }
 560
 561 impl Step for Std {
 562     type Output = Option<GeneratedTarball>;
 563     const DEFAULT: bool = true;
 564
 565     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 566         run.path("library/std")
 567     }
 568
 569     fn make_run(run: RunConfig<'_>) {
 570         run.builder.ensure(Std {
 571             compiler: run.builder.compiler_for(
 572                 run.builder.top_stage,
 573                 run.builder.config.build,
 574                 run.target,
 575             ),
 576             target: run.target,
 577         });
 578     }
 579
 580     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 581         let compiler = self.compiler;
 582         let target = self.target;
 583
 584         if skip_host_target_lib(builder, compiler) {
 585             return None;
 586         }
 587
 588         builder.ensure(compile::Std { compiler, target });
 589
 590         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 591         tarball.include_target_in_component_name(true);
 592
 593         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 594         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 595         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 596
 597         Some(tarball.generate())
 598     }
 599 }
 600
 601 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 602 pub struct RustcDev {
 603     pub compiler: Compiler,
 604     pub target: TargetSelection,
 605 }
 606
 607 impl Step for RustcDev {
 608     type Output = Option<GeneratedTarball>;
 609     const DEFAULT: bool = true;
 610     const ONLY_HOSTS: bool = true;
 611
 612     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 613         run.path("rustc-dev")
 614     }
 615
 616     fn make_run(run: RunConfig<'_>) {
 617         run.builder.ensure(RustcDev {
 618             compiler: run.builder.compiler_for(
 619                 run.builder.top_stage,
 620                 run.builder.config.build,
 621                 run.target,
 622             ),
 623             target: run.target,
 624         });
 625     }
 626
 627     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 628         let compiler = self.compiler;
 629         let target = self.target;
 630         if skip_host_target_lib(builder, compiler) {
 631             return None;
 632         }
 633
 634         builder.ensure(compile::Rustc { compiler, target });
 635
 636         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 637
 638         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 639         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 640         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 641
 642         let src_files = &["Cargo.lock"];
 643         // This is the reduced set of paths which will become the rustc-dev component
 644         // (essentially the compiler crates and all of their path dependencies).
 645         copy_src_dirs(
 646             builder,
 647             &builder.src,
 648             &["compiler"],
 649             &[],
 650             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 651         );
 652         // This particular crate is used as a build dependency of the above.
 653         copy_src_dirs(
 654             builder,
 655             &builder.src,
 656             &["src/build_helper"],
 657             &[],
 658             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 659         );
 660         for file in src_files {
 661             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 662         }
 663
 664         Some(tarball.generate())
 665     }
 666 }
 667
 668 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 669 pub struct Analysis {
 670     pub compiler: Compiler,
 671     pub target: TargetSelection,
 672 }
 673
 674 impl Step for Analysis {
 675     type Output = Option<GeneratedTarball>;
 676     const DEFAULT: bool = true;
 677
 678     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 679         let builder = run.builder;
 680         run.path("analysis").default_condition(builder.config.extended)
 681     }
 682
 683     fn make_run(run: RunConfig<'_>) {
 684         run.builder.ensure(Analysis {
 685             // Find the actual compiler (handling the full bootstrap option) which
 686             // produced the save-analysis data because that data isn't copied
 687             // through the sysroot uplifting.
 688             compiler: run.builder.compiler_for(
 689                 run.builder.top_stage,
 690                 run.builder.config.build,
 691                 run.target,
 692             ),
 693             target: run.target,
 694         });
 695     }
 696
 697     /// Creates a tarball of save-analysis metadata, if available.
 698     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 699         let compiler = self.compiler;
 700         let target = self.target;
 701         assert!(builder.config.extended);
 702         if compiler.host != builder.config.build {
 703             return None;
 704         }
 705
 706         builder.ensure(compile::Std { compiler, target });
 707         let src = builder
 708             .stage_out(compiler, Mode::Std)
 709             .join(target.triple)
 710             .join(builder.cargo_dir())
 711             .join("deps")
 712             .join("save-analysis");
 713
 714         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 715         tarball.include_target_in_component_name(true);
 716         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 717         Some(tarball.generate())
 718     }
 719 }
 720
 721 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 722 /// `dst_dir`.
 723 fn copy_src_dirs(
 724     builder: &Builder<'_>,
 725     base: &Path,
 726     src_dirs: &[&str],
 727     exclude_dirs: &[&str],
 728     dst_dir: &Path,
 729 ) {
 730     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 731         let spath = match path.to_str() {
 732             Some(path) => path,
 733             None => return false,
 734         };
 735         if spath.ends_with('~') || spath.ends_with(".pyc") {
 736             return false;
 737         }
 738
 739         const LLVM_PROJECTS: &[&str] = &[
 740             "llvm-project/clang",
 741             "llvm-project\\clang",
 742             "llvm-project/libunwind",
 743             "llvm-project\\libunwind",
 744             "llvm-project/lld",
 745             "llvm-project\\lld",
 746             "llvm-project/lldb",
 747             "llvm-project\\lldb",
 748             "llvm-project/llvm",
 749             "llvm-project\\llvm",
 750             "llvm-project/compiler-rt",
 751             "llvm-project\\compiler-rt",
 752         ];
 753         if spath.contains("llvm-project")
 754             && !spath.ends_with("llvm-project")
 755             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 756         {
 757             return false;
 758         }
 759
 760         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 761         if LLVM_TEST.iter().any(|path| spath.contains(path))
 762             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 763         {
 764             return false;
 765         }
 766
 767         let full_path = Path::new(dir).join(path);
 768         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 769             return false;
 770         }
 771
 772         let excludes = [
 773             "CVS",
 774             "RCS",
 775             "SCCS",
 776             ".git",
 777             ".gitignore",
 778             ".gitmodules",
 779             ".gitattributes",
 780             ".cvsignore",
 781             ".svn",
 782             ".arch-ids",
 783             "{arch}",
 784             "=RELEASE-ID",
 785             "=meta-update",
 786             "=update",
 787             ".bzr",
 788             ".bzrignore",
 789             ".bzrtags",
 790             ".hg",
 791             ".hgignore",
 792             ".hgrags",
 793             "_darcs",
 794         ];
 795         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 796     }
 797
 798     // Copy the directories using our filter
 799     for item in src_dirs {
 800         let dst = &dst_dir.join(item);
 801         t!(fs::create_dir_all(dst));
 802         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 803     }
 804 }
 805
 806 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 807 pub struct Src;
 808
 809 impl Step for Src {
 810     /// The output path of the src installer tarball
 811     type Output = GeneratedTarball;
 812     const DEFAULT: bool = true;
 813     const ONLY_HOSTS: bool = true;
 814
 815     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 816         run.path("src")
 817     }
 818
 819     fn make_run(run: RunConfig<'_>) {
 820         run.builder.ensure(Src);
 821     }
 822
 823     /// Creates the `rust-src` installer component
 824     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 825         let tarball = Tarball::new_targetless(builder, "rust-src");
 826
 827         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 828         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 829         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 830         // and fix them...
 831         //
 832         // NOTE: if you update the paths here, you also should update the "virtual" path
 833         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 834         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 835
 836         let src_files = ["Cargo.lock"];
 837         // This is the reduced set of paths which will become the rust-src component
 838         // (essentially libstd and all of its path dependencies).
 839         copy_src_dirs(
 840             builder,
 841             &builder.src,
 842             &["library", "src/llvm-project/libunwind"],
 843             &[
 844                 // not needed and contains symlinks which rustup currently
 845                 // chokes on when unpacking.
 846                 "library/backtrace/crates",
 847             ],
 848             &dst_src,
 849         );
 850         for file in src_files.iter() {
 851             builder.copy(&builder.src.join(file), &dst_src.join(file));
 852         }
 853
 854         tarball.generate()
 855     }
 856 }
 857
 858 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 859 pub struct PlainSourceTarball;
 860
 861 impl Step for PlainSourceTarball {
 862     /// Produces the location of the tarball generated
 863     type Output = GeneratedTarball;
 864     const DEFAULT: bool = true;
 865     const ONLY_HOSTS: bool = true;
 866
 867     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 868         let builder = run.builder;
 869         run.path("src").default_condition(builder.config.rust_dist_src)
 870     }
 871
 872     fn make_run(run: RunConfig<'_>) {
 873         run.builder.ensure(PlainSourceTarball);
 874     }
 875
 876     /// Creates the plain source tarball
 877     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 878         let tarball = Tarball::new(builder, "rustc", "src");
 879         let plain_dst_src = tarball.image_dir();
 880
 881         // This is the set of root paths which will become part of the source package
 882         let src_files = [
 883             "COPYRIGHT",
 884             "LICENSE-APACHE",
 885             "LICENSE-MIT",
 886             "CONTRIBUTING.md",
 887             "README.md",
 888             "RELEASES.md",
 889             "configure",
 890             "x.py",
 891             "config.toml.example",
 892             "Cargo.toml",
 893             "Cargo.lock",
 894         ];
 895         let src_dirs = ["src", "compiler", "library"];
 896
 897         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 898
 899         // Copy the files normally
 900         for item in &src_files {
 901             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 902         }
 903
 904         // Create the version file
 905         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 906         if let Some(sha) = builder.rust_sha() {
 907             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 908         }
 909
 910         // If we're building from git sources, we need to vendor a complete distribution.
 911         if builder.rust_info.is_git() {
 912             // Vendor all Cargo dependencies
 913             let mut cmd = Command::new(&builder.initial_cargo);
 914             cmd.arg("vendor")
 915                 .arg("--sync")
 916                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 917                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 918                 .current_dir(&plain_dst_src);
 919             builder.run(&mut cmd);
 920         }
 921
 922         tarball.bare()
 923     }
 924 }
 925
 926 // We have to run a few shell scripts, which choke quite a bit on both `\`
 927 // characters and on `C:\` paths, so normalize both of them away.
 928 pub fn sanitize_sh(path: &Path) -> String {
 929     let path = path.to_str().unwrap().replace("\\", "/");
 930     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 931
 932     fn unc_to_lfs(s: &str) -> &str {
 933         s.strip_prefix("//?/").unwrap_or(s)
 934     }
 935
 936     fn change_drive(s: &str) -> Option<String> {
 937         let mut ch = s.chars();
 938         let drive = ch.next().unwrap_or('C');
 939         if ch.next() != Some(':') {
 940             return None;
 941         }
 942         if ch.next() != Some('/') {
 943             return None;
 944         }
 945         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 946     }
 947 }
 948
 949 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 950 pub struct Cargo {
 951     pub compiler: Compiler,
 952     pub target: TargetSelection,
 953 }
 954
 955 impl Step for Cargo {
 956     type Output = GeneratedTarball;
 957     const ONLY_HOSTS: bool = true;
 958
 959     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 960         run.path("cargo")
 961     }
 962
 963     fn make_run(run: RunConfig<'_>) {
 964         run.builder.ensure(Cargo {
 965             compiler: run.builder.compiler_for(
 966                 run.builder.top_stage,
 967                 run.builder.config.build,
 968                 run.target,
 969             ),
 970             target: run.target,
 971         });
 972     }
 973
 974     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 975         let compiler = self.compiler;
 976         let target = self.target;
 977
 978         let cargo = builder.ensure(tool::Cargo { compiler, target });
 979         let src = builder.src.join("src/tools/cargo");
 980         let etc = src.join("src/etc");
 981
 982         // Prepare the image directory
 983         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 984         tarball.set_overlay(OverlayKind::Cargo);
 985
 986         tarball.add_file(&cargo, "bin", 0o755);
 987         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 988         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 989         tarball.add_dir(etc.join("man"), "share/man/man1");
 990         tarball.add_legal_and_readme_to("share/doc/cargo");
 991
 992         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 993             let dirent = dirent.expect("read dir entry");
 994             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 995                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 996             }
 997         }
 998
 999         tarball.generate()
1000     }
1001 }
1002
1003 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1004 pub struct Rls {
1005     pub compiler: Compiler,
1006     pub target: TargetSelection,
1007 }
1008
1009 impl Step for Rls {
1010     type Output = Option<GeneratedTarball>;
1011     const ONLY_HOSTS: bool = true;
1012
1013     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1014         run.path("rls")
1015     }
1016
1017     fn make_run(run: RunConfig<'_>) {
1018         run.builder.ensure(Rls {
1019             compiler: run.builder.compiler_for(
1020                 run.builder.top_stage,
1021                 run.builder.config.build,
1022                 run.target,
1023             ),
1024             target: run.target,
1025         });
1026     }
1027
1028     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1029         let compiler = self.compiler;
1030         let target = self.target;
1031         assert!(builder.config.extended);
1032
1033         let rls = builder
1034             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1035             .or_else(|| {
1036                 missing_tool("RLS", builder.build.config.missing_tools);
1037                 None
1038             })?;
1039
1040         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1041         tarball.set_overlay(OverlayKind::RLS);
1042         tarball.is_preview(true);
1043         tarball.add_file(rls, "bin", 0o755);
1044         tarball.add_legal_and_readme_to("share/doc/rls");
1045         Some(tarball.generate())
1046     }
1047 }
1048
1049 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1050 pub struct RustAnalyzer {
1051     pub compiler: Compiler,
1052     pub target: TargetSelection,
1053 }
1054
1055 impl Step for RustAnalyzer {
1056     type Output = Option<GeneratedTarball>;
1057     const ONLY_HOSTS: bool = true;
1058
1059     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1060         run.path("rust-analyzer")
1061     }
1062
1063     fn make_run(run: RunConfig<'_>) {
1064         run.builder.ensure(RustAnalyzer {
1065             compiler: run.builder.compiler_for(
1066                 run.builder.top_stage,
1067                 run.builder.config.build,
1068                 run.target,
1069             ),
1070             target: run.target,
1071         });
1072     }
1073
1074     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1075         // This prevents rust-analyzer from being built for "dist" or "install"
1076         // on the stable/beta channels. It is a nightly-only tool and should
1077         // not be included.
1078         if !builder.build.unstable_features() {
1079             return None;
1080         }
1081         let compiler = self.compiler;
1082         let target = self.target;
1083         assert!(builder.config.extended);
1084
1085         if target.contains("riscv64") {
1086             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1087             // to build. See #74813 for details.
1088             return None;
1089         }
1090
1091         let rust_analyzer = builder
1092             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1093             .expect("rust-analyzer always builds");
1094
1095         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1096         tarball.set_overlay(OverlayKind::RustAnalyzer);
1097         tarball.is_preview(true);
1098         tarball.add_file(rust_analyzer, "bin", 0o755);
1099         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1100         Some(tarball.generate())
1101     }
1102 }
1103
1104 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1105 pub struct Clippy {
1106     pub compiler: Compiler,
1107     pub target: TargetSelection,
1108 }
1109
1110 impl Step for Clippy {
1111     type Output = GeneratedTarball;
1112     const ONLY_HOSTS: bool = true;
1113
1114     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1115         run.path("clippy")
1116     }
1117
1118     fn make_run(run: RunConfig<'_>) {
1119         run.builder.ensure(Clippy {
1120             compiler: run.builder.compiler_for(
1121                 run.builder.top_stage,
1122                 run.builder.config.build,
1123                 run.target,
1124             ),
1125             target: run.target,
1126         });
1127     }
1128
1129     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1130         let compiler = self.compiler;
1131         let target = self.target;
1132         assert!(builder.config.extended);
1133
1134         // Prepare the image directory
1135         // We expect clippy to build, because we've exited this step above if tool
1136         // state for clippy isn't testing.
1137         let clippy = builder
1138             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1139             .expect("clippy expected to build - essential tool");
1140         let cargoclippy = builder
1141             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1142             .expect("clippy expected to build - essential tool");
1143
1144         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1145         tarball.set_overlay(OverlayKind::Clippy);
1146         tarball.is_preview(true);
1147         tarball.add_file(clippy, "bin", 0o755);
1148         tarball.add_file(cargoclippy, "bin", 0o755);
1149         tarball.add_legal_and_readme_to("share/doc/clippy");
1150         tarball.generate()
1151     }
1152 }
1153
1154 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1155 pub struct Miri {
1156     pub compiler: Compiler,
1157     pub target: TargetSelection,
1158 }
1159
1160 impl Step for Miri {
1161     type Output = Option<GeneratedTarball>;
1162     const ONLY_HOSTS: bool = true;
1163
1164     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1165         run.path("miri")
1166     }
1167
1168     fn make_run(run: RunConfig<'_>) {
1169         run.builder.ensure(Miri {
1170             compiler: run.builder.compiler_for(
1171                 run.builder.top_stage,
1172                 run.builder.config.build,
1173                 run.target,
1174             ),
1175             target: run.target,
1176         });
1177     }
1178
1179     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1180         // This prevents miri from being built for "dist" or "install"
1181         // on the stable/beta channels. It is a nightly-only tool and should
1182         // not be included.
1183         if !builder.build.unstable_features() {
1184             return None;
1185         }
1186         let compiler = self.compiler;
1187         let target = self.target;
1188         assert!(builder.config.extended);
1189
1190         let miri = builder
1191             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1192             .or_else(|| {
1193                 missing_tool("miri", builder.build.config.missing_tools);
1194                 None
1195             })?;
1196         let cargomiri = builder
1197             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1198             .or_else(|| {
1199                 missing_tool("cargo miri", builder.build.config.missing_tools);
1200                 None
1201             })?;
1202
1203         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1204         tarball.set_overlay(OverlayKind::Miri);
1205         tarball.is_preview(true);
1206         tarball.add_file(miri, "bin", 0o755);
1207         tarball.add_file(cargomiri, "bin", 0o755);
1208         tarball.add_legal_and_readme_to("share/doc/miri");
1209         Some(tarball.generate())
1210     }
1211 }
1212
1213 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1214 pub struct Rustfmt {
1215     pub compiler: Compiler,
1216     pub target: TargetSelection,
1217 }
1218
1219 impl Step for Rustfmt {
1220     type Output = Option<GeneratedTarball>;
1221     const ONLY_HOSTS: bool = true;
1222
1223     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1224         run.path("rustfmt")
1225     }
1226
1227     fn make_run(run: RunConfig<'_>) {
1228         run.builder.ensure(Rustfmt {
1229             compiler: run.builder.compiler_for(
1230                 run.builder.top_stage,
1231                 run.builder.config.build,
1232                 run.target,
1233             ),
1234             target: run.target,
1235         });
1236     }
1237
1238     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1239         let compiler = self.compiler;
1240         let target = self.target;
1241
1242         let rustfmt = builder
1243             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1244             .or_else(|| {
1245                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1246                 None
1247             })?;
1248         let cargofmt = builder
1249             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1250             .or_else(|| {
1251                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1252                 None
1253             })?;
1254
1255         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1256         tarball.set_overlay(OverlayKind::Rustfmt);
1257         tarball.is_preview(true);
1258         tarball.add_file(rustfmt, "bin", 0o755);
1259         tarball.add_file(cargofmt, "bin", 0o755);
1260         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1261         Some(tarball.generate())
1262     }
1263 }
1264
1265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1266 pub struct RustDemangler {
1267     pub compiler: Compiler,
1268     pub target: TargetSelection,
1269 }
1270
1271 impl Step for RustDemangler {
1272     type Output = Option<GeneratedTarball>;
1273     const ONLY_HOSTS: bool = true;
1274
1275     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1276         run.path("rust-demangler")
1277     }
1278
1279     fn make_run(run: RunConfig<'_>) {
1280         run.builder.ensure(RustDemangler {
1281             compiler: run.builder.compiler_for(
1282                 run.builder.top_stage,
1283                 run.builder.config.build,
1284                 run.target,
1285             ),
1286             target: run.target,
1287         });
1288     }
1289
1290     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1291         let compiler = self.compiler;
1292         let target = self.target;
1293         assert!(builder.config.extended);
1294
1295         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1296         let profiler = builder.config.profiler_enabled(target);
1297         if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
1298             return None;
1299         }
1300
1301         let rust_demangler = builder
1302             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1303             .expect("rust-demangler expected to build - in-tree tool");
1304
1305         // Prepare the image directory
1306         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1307         tarball.set_overlay(OverlayKind::RustDemangler);
1308         tarball.is_preview(true);
1309         tarball.add_file(&rust_demangler, "bin", 0o755);
1310         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1311         Some(tarball.generate())
1312     }
1313 }
1314
1315 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1316 pub struct Extended {
1317     stage: u32,
1318     host: TargetSelection,
1319     target: TargetSelection,
1320 }
1321
1322 impl Step for Extended {
1323     type Output = ();
1324     const DEFAULT: bool = true;
1325     const ONLY_HOSTS: bool = true;
1326
1327     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1328         let builder = run.builder;
1329         run.path("extended").default_condition(builder.config.extended)
1330     }
1331
1332     fn make_run(run: RunConfig<'_>) {
1333         run.builder.ensure(Extended {
1334             stage: run.builder.top_stage,
1335             host: run.builder.config.build,
1336             target: run.target,
1337         });
1338     }
1339
1340     /// Creates a combined installer for the specified target in the provided stage.
1341     fn run(self, builder: &Builder<'_>) {
1342         let target = self.target;
1343         let stage = self.stage;
1344         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1345
1346         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1347
1348         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1349         let cargo_installer = builder.ensure(Cargo { compiler, target });
1350         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1351         let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
1352         let rls_installer = builder.ensure(Rls { compiler, target });
1353         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1354         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1355         let clippy_installer = builder.ensure(Clippy { compiler, target });
1356         let miri_installer = builder.ensure(Miri { compiler, target });
1357         let mingw_installer = builder.ensure(Mingw { host: target });
1358         let analysis_installer = builder.ensure(Analysis { compiler, target });
1359
1360         let docs_installer = builder.ensure(Docs { host: target });
1361         let std_installer = builder.ensure(Std { compiler, target });
1362
1363         let etc = builder.src.join("src/etc/installer");
1364
1365         // Avoid producing tarballs during a dry run.
1366         if builder.config.dry_run {
1367             return;
1368         }
1369
1370         // When rust-std package split from rustc, we needed to ensure that during
1371         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1372         // the std files during uninstall. To do this ensure that rustc comes
1373         // before rust-std in the list below.
1374         let mut tarballs = Vec::new();
1375         tarballs.push(rustc_installer);
1376         tarballs.push(cargo_installer);
1377         tarballs.push(clippy_installer);
1378         tarballs.extend(rust_demangler_installer.clone());
1379         tarballs.extend(rls_installer.clone());
1380         tarballs.extend(rust_analyzer_installer.clone());
1381         tarballs.extend(miri_installer.clone());
1382         tarballs.extend(rustfmt_installer.clone());
1383         tarballs.extend(llvm_tools_installer);
1384         if let Some(analysis_installer) = analysis_installer {
1385             tarballs.push(analysis_installer);
1386         }
1387         tarballs.push(std_installer.expect("missing std"));
1388         if let Some(docs_installer) = docs_installer {
1389             tarballs.push(docs_installer);
1390         }
1391         if target.contains("pc-windows-gnu") {
1392             tarballs.push(mingw_installer.unwrap());
1393         }
1394
1395         let tarball = Tarball::new(builder, "rust", &target.triple);
1396         let generated = tarball.combine(&tarballs);
1397
1398         let tmp = tmpdir(builder).join("combined-tarball");
1399         let work = generated.work_dir();
1400
1401         let mut license = String::new();
1402         license += &builder.read(&builder.src.join("COPYRIGHT"));
1403         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1404         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1405         license.push('\n');
1406         license.push('\n');
1407
1408         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1409         let mut rtf = rtf.to_string();
1410         rtf.push('\n');
1411         for line in license.lines() {
1412             rtf.push_str(line);
1413             rtf.push_str("\\line ");
1414         }
1415         rtf.push('}');
1416
1417         fn filter(contents: &str, marker: &str) -> String {
1418             let start = format!("tool-{}-start", marker);
1419             let end = format!("tool-{}-end", marker);
1420             let mut lines = Vec::new();
1421             let mut omitted = false;
1422             for line in contents.lines() {
1423                 if line.contains(&start) {
1424                     omitted = true;
1425                 } else if line.contains(&end) {
1426                     omitted = false;
1427                 } else if !omitted {
1428                     lines.push(line);
1429                 }
1430             }
1431
1432             lines.join("\n")
1433         }
1434
1435         let xform = |p: &Path| {
1436             let mut contents = t!(fs::read_to_string(p));
1437             if rust_demangler_installer.is_none() {
1438                 contents = filter(&contents, "rust-demangler");
1439             }
1440             if rls_installer.is_none() {
1441                 contents = filter(&contents, "rls");
1442             }
1443             if rust_analyzer_installer.is_none() {
1444                 contents = filter(&contents, "rust-analyzer");
1445             }
1446             if miri_installer.is_none() {
1447                 contents = filter(&contents, "miri");
1448             }
1449             if rustfmt_installer.is_none() {
1450                 contents = filter(&contents, "rustfmt");
1451             }
1452             let ret = tmp.join(p.file_name().unwrap());
1453             t!(fs::write(&ret, &contents));
1454             ret
1455         };
1456
1457         if target.contains("apple-darwin") {
1458             builder.info("building pkg installer");
1459             let pkg = tmp.join("pkg");
1460             let _ = fs::remove_dir_all(&pkg);
1461
1462             let pkgbuild = |component: &str| {
1463                 let mut cmd = Command::new("pkgbuild");
1464                 cmd.arg("--identifier")
1465                     .arg(format!("org.rust-lang.{}", component))
1466                     .arg("--scripts")
1467                     .arg(pkg.join(component))
1468                     .arg("--nopayload")
1469                     .arg(pkg.join(component).with_extension("pkg"));
1470                 builder.run(&mut cmd);
1471             };
1472
1473             let prepare = |name: &str| {
1474                 builder.create_dir(&pkg.join(name));
1475                 builder.cp_r(
1476                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1477                     &pkg.join(name),
1478                 );
1479                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1480                 pkgbuild(name);
1481             };
1482             prepare("rustc");
1483             prepare("cargo");
1484             prepare("rust-docs");
1485             prepare("rust-std");
1486             prepare("rust-analysis");
1487             prepare("clippy");
1488             if rust_demangler_installer.is_some() {
1489                 prepare("rust-demangler");
1490             }
1491             if rls_installer.is_some() {
1492                 prepare("rls");
1493             }
1494             if rust_analyzer_installer.is_some() {
1495                 prepare("rust-analyzer");
1496             }
1497             if miri_installer.is_some() {
1498                 prepare("miri");
1499             }
1500
1501             // create an 'uninstall' package
1502             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1503             pkgbuild("uninstall");
1504
1505             builder.create_dir(&pkg.join("res"));
1506             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1507             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1508             let mut cmd = Command::new("productbuild");
1509             cmd.arg("--distribution")
1510                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1511                 .arg("--resources")
1512                 .arg(pkg.join("res"))
1513                 .arg(distdir(builder).join(format!(
1514                     "{}-{}.pkg",
1515                     pkgname(builder, "rust"),
1516                     target.triple
1517                 )))
1518                 .arg("--package-path")
1519                 .arg(&pkg);
1520             let _time = timeit(builder);
1521             builder.run(&mut cmd);
1522         }
1523
1524         if target.contains("windows") {
1525             let exe = tmp.join("exe");
1526             let _ = fs::remove_dir_all(&exe);
1527
1528             let prepare = |name: &str| {
1529                 builder.create_dir(&exe.join(name));
1530                 let dir = if name == "rust-std" || name == "rust-analysis" {
1531                     format!("{}-{}", name, target.triple)
1532                 } else if name == "rls" {
1533                     "rls-preview".to_string()
1534                 } else if name == "rust-analyzer" {
1535                     "rust-analyzer-preview".to_string()
1536                 } else if name == "clippy" {
1537                     "clippy-preview".to_string()
1538                 } else if name == "rust-demangler" {
1539                     "rust-demangler-preview".to_string()
1540                 } else if name == "miri" {
1541                     "miri-preview".to_string()
1542                 } else {
1543                     name.to_string()
1544                 };
1545                 builder.cp_r(
1546                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1547                     &exe.join(name),
1548                 );
1549                 builder.remove(&exe.join(name).join("manifest.in"));
1550             };
1551             prepare("rustc");
1552             prepare("cargo");
1553             prepare("rust-analysis");
1554             prepare("rust-docs");
1555             prepare("rust-std");
1556             prepare("clippy");
1557             if rust_demangler_installer.is_some() {
1558                 prepare("rust-demangler");
1559             }
1560             if rls_installer.is_some() {
1561                 prepare("rls");
1562             }
1563             if rust_analyzer_installer.is_some() {
1564                 prepare("rust-analyzer");
1565             }
1566             if miri_installer.is_some() {
1567                 prepare("miri");
1568             }
1569             if target.contains("windows-gnu") {
1570                 prepare("rust-mingw");
1571             }
1572
1573             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1574
1575             // Generate msi installer
1576             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1577             let heat = wix.join("bin/heat.exe");
1578             let candle = wix.join("bin/candle.exe");
1579             let light = wix.join("bin/light.exe");
1580
1581             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1582             builder.run(
1583                 Command::new(&heat)
1584                     .current_dir(&exe)
1585                     .arg("dir")
1586                     .arg("rustc")
1587                     .args(&heat_flags)
1588                     .arg("-cg")
1589                     .arg("RustcGroup")
1590                     .arg("-dr")
1591                     .arg("Rustc")
1592                     .arg("-var")
1593                     .arg("var.RustcDir")
1594                     .arg("-out")
1595                     .arg(exe.join("RustcGroup.wxs")),
1596             );
1597             builder.run(
1598                 Command::new(&heat)
1599                     .current_dir(&exe)
1600                     .arg("dir")
1601                     .arg("rust-docs")
1602                     .args(&heat_flags)
1603                     .arg("-cg")
1604                     .arg("DocsGroup")
1605                     .arg("-dr")
1606                     .arg("Docs")
1607                     .arg("-var")
1608                     .arg("var.DocsDir")
1609                     .arg("-out")
1610                     .arg(exe.join("DocsGroup.wxs"))
1611                     .arg("-t")
1612                     .arg(etc.join("msi/squash-components.xsl")),
1613             );
1614             builder.run(
1615                 Command::new(&heat)
1616                     .current_dir(&exe)
1617                     .arg("dir")
1618                     .arg("cargo")
1619                     .args(&heat_flags)
1620                     .arg("-cg")
1621                     .arg("CargoGroup")
1622                     .arg("-dr")
1623                     .arg("Cargo")
1624                     .arg("-var")
1625                     .arg("var.CargoDir")
1626                     .arg("-out")
1627                     .arg(exe.join("CargoGroup.wxs"))
1628                     .arg("-t")
1629                     .arg(etc.join("msi/remove-duplicates.xsl")),
1630             );
1631             builder.run(
1632                 Command::new(&heat)
1633                     .current_dir(&exe)
1634                     .arg("dir")
1635                     .arg("rust-std")
1636                     .args(&heat_flags)
1637                     .arg("-cg")
1638                     .arg("StdGroup")
1639                     .arg("-dr")
1640                     .arg("Std")
1641                     .arg("-var")
1642                     .arg("var.StdDir")
1643                     .arg("-out")
1644                     .arg(exe.join("StdGroup.wxs")),
1645             );
1646             if rls_installer.is_some() {
1647                 builder.run(
1648                     Command::new(&heat)
1649                         .current_dir(&exe)
1650                         .arg("dir")
1651                         .arg("rls")
1652                         .args(&heat_flags)
1653                         .arg("-cg")
1654                         .arg("RlsGroup")
1655                         .arg("-dr")
1656                         .arg("Rls")
1657                         .arg("-var")
1658                         .arg("var.RlsDir")
1659                         .arg("-out")
1660                         .arg(exe.join("RlsGroup.wxs"))
1661                         .arg("-t")
1662                         .arg(etc.join("msi/remove-duplicates.xsl")),
1663                 );
1664             }
1665             if rust_analyzer_installer.is_some() {
1666                 builder.run(
1667                     Command::new(&heat)
1668                         .current_dir(&exe)
1669                         .arg("dir")
1670                         .arg("rust-analyzer")
1671                         .args(&heat_flags)
1672                         .arg("-cg")
1673                         .arg("RustAnalyzerGroup")
1674                         .arg("-dr")
1675                         .arg("RustAnalyzer")
1676                         .arg("-var")
1677                         .arg("var.RustAnalyzerDir")
1678                         .arg("-out")
1679                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1680                         .arg("-t")
1681                         .arg(etc.join("msi/remove-duplicates.xsl")),
1682                 );
1683             }
1684             builder.run(
1685                 Command::new(&heat)
1686                     .current_dir(&exe)
1687                     .arg("dir")
1688                     .arg("clippy")
1689                     .args(&heat_flags)
1690                     .arg("-cg")
1691                     .arg("ClippyGroup")
1692                     .arg("-dr")
1693                     .arg("Clippy")
1694                     .arg("-var")
1695                     .arg("var.ClippyDir")
1696                     .arg("-out")
1697                     .arg(exe.join("ClippyGroup.wxs"))
1698                     .arg("-t")
1699                     .arg(etc.join("msi/remove-duplicates.xsl")),
1700             );
1701             if rust_demangler_installer.is_some() {
1702                 builder.run(
1703                     Command::new(&heat)
1704                         .current_dir(&exe)
1705                         .arg("dir")
1706                         .arg("rust-demangler")
1707                         .args(&heat_flags)
1708                         .arg("-cg")
1709                         .arg("RustDemanglerGroup")
1710                         .arg("-dr")
1711                         .arg("RustDemangler")
1712                         .arg("-var")
1713                         .arg("var.RustDemanglerDir")
1714                         .arg("-out")
1715                         .arg(exe.join("RustDemanglerGroup.wxs"))
1716                         .arg("-t")
1717                         .arg(etc.join("msi/remove-duplicates.xsl")),
1718                 );
1719             }
1720             if miri_installer.is_some() {
1721                 builder.run(
1722                     Command::new(&heat)
1723                         .current_dir(&exe)
1724                         .arg("dir")
1725                         .arg("miri")
1726                         .args(&heat_flags)
1727                         .arg("-cg")
1728                         .arg("MiriGroup")
1729                         .arg("-dr")
1730                         .arg("Miri")
1731                         .arg("-var")
1732                         .arg("var.MiriDir")
1733                         .arg("-out")
1734                         .arg(exe.join("MiriGroup.wxs"))
1735                         .arg("-t")
1736                         .arg(etc.join("msi/remove-duplicates.xsl")),
1737                 );
1738             }
1739             builder.run(
1740                 Command::new(&heat)
1741                     .current_dir(&exe)
1742                     .arg("dir")
1743                     .arg("rust-analysis")
1744                     .args(&heat_flags)
1745                     .arg("-cg")
1746                     .arg("AnalysisGroup")
1747                     .arg("-dr")
1748                     .arg("Analysis")
1749                     .arg("-var")
1750                     .arg("var.AnalysisDir")
1751                     .arg("-out")
1752                     .arg(exe.join("AnalysisGroup.wxs"))
1753                     .arg("-t")
1754                     .arg(etc.join("msi/remove-duplicates.xsl")),
1755             );
1756             if target.contains("windows-gnu") {
1757                 builder.run(
1758                     Command::new(&heat)
1759                         .current_dir(&exe)
1760                         .arg("dir")
1761                         .arg("rust-mingw")
1762                         .args(&heat_flags)
1763                         .arg("-cg")
1764                         .arg("GccGroup")
1765                         .arg("-dr")
1766                         .arg("Gcc")
1767                         .arg("-var")
1768                         .arg("var.GccDir")
1769                         .arg("-out")
1770                         .arg(exe.join("GccGroup.wxs")),
1771                 );
1772             }
1773
1774             let candle = |input: &Path| {
1775                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1776                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1777                 let mut cmd = Command::new(&candle);
1778                 cmd.current_dir(&exe)
1779                     .arg("-nologo")
1780                     .arg("-dRustcDir=rustc")
1781                     .arg("-dDocsDir=rust-docs")
1782                     .arg("-dCargoDir=cargo")
1783                     .arg("-dStdDir=rust-std")
1784                     .arg("-dAnalysisDir=rust-analysis")
1785                     .arg("-dClippyDir=clippy")
1786                     .arg("-arch")
1787                     .arg(&arch)
1788                     .arg("-out")
1789                     .arg(&output)
1790                     .arg(&input);
1791                 add_env(builder, &mut cmd, target);
1792
1793                 if rust_demangler_installer.is_some() {
1794                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1795                 }
1796                 if rls_installer.is_some() {
1797                     cmd.arg("-dRlsDir=rls");
1798                 }
1799                 if rust_analyzer_installer.is_some() {
1800                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1801                 }
1802                 if miri_installer.is_some() {
1803                     cmd.arg("-dMiriDir=miri");
1804                 }
1805                 if target.contains("windows-gnu") {
1806                     cmd.arg("-dGccDir=rust-mingw");
1807                 }
1808                 builder.run(&mut cmd);
1809             };
1810             candle(&xform(&etc.join("msi/rust.wxs")));
1811             candle(&etc.join("msi/ui.wxs"));
1812             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1813             candle("RustcGroup.wxs".as_ref());
1814             candle("DocsGroup.wxs".as_ref());
1815             candle("CargoGroup.wxs".as_ref());
1816             candle("StdGroup.wxs".as_ref());
1817             candle("ClippyGroup.wxs".as_ref());
1818             if rust_demangler_installer.is_some() {
1819                 candle("RustDemanglerGroup.wxs".as_ref());
1820             }
1821             if rls_installer.is_some() {
1822                 candle("RlsGroup.wxs".as_ref());
1823             }
1824             if rust_analyzer_installer.is_some() {
1825                 candle("RustAnalyzerGroup.wxs".as_ref());
1826             }
1827             if miri_installer.is_some() {
1828                 candle("MiriGroup.wxs".as_ref());
1829             }
1830             candle("AnalysisGroup.wxs".as_ref());
1831
1832             if target.contains("windows-gnu") {
1833                 candle("GccGroup.wxs".as_ref());
1834             }
1835
1836             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1837             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1838             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1839
1840             builder.info(&format!("building `msi` installer with {:?}", light));
1841             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1842             let mut cmd = Command::new(&light);
1843             cmd.arg("-nologo")
1844                 .arg("-ext")
1845                 .arg("WixUIExtension")
1846                 .arg("-ext")
1847                 .arg("WixUtilExtension")
1848                 .arg("-out")
1849                 .arg(exe.join(&filename))
1850                 .arg("rust.wixobj")
1851                 .arg("ui.wixobj")
1852                 .arg("rustwelcomedlg.wixobj")
1853                 .arg("RustcGroup.wixobj")
1854                 .arg("DocsGroup.wixobj")
1855                 .arg("CargoGroup.wixobj")
1856                 .arg("StdGroup.wixobj")
1857                 .arg("AnalysisGroup.wixobj")
1858                 .arg("ClippyGroup.wixobj")
1859                 .current_dir(&exe);
1860
1861             if rls_installer.is_some() {
1862                 cmd.arg("RlsGroup.wixobj");
1863             }
1864             if rust_analyzer_installer.is_some() {
1865                 cmd.arg("RustAnalyzerGroup.wixobj");
1866             }
1867             if rust_demangler_installer.is_some() {
1868                 cmd.arg("RustDemanglerGroup.wixobj");
1869             }
1870             if miri_installer.is_some() {
1871                 cmd.arg("MiriGroup.wixobj");
1872             }
1873
1874             if target.contains("windows-gnu") {
1875                 cmd.arg("GccGroup.wixobj");
1876             }
1877             // ICE57 wrongly complains about the shortcuts
1878             cmd.arg("-sice:ICE57");
1879
1880             let _time = timeit(builder);
1881             builder.run(&mut cmd);
1882
1883             if !builder.config.dry_run {
1884                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1885             }
1886         }
1887     }
1888 }
1889
1890 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1891     let mut parts = builder.version.split('.');
1892     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1893         .env("CFG_RELEASE_NUM", &builder.version)
1894         .env("CFG_RELEASE", builder.rust_release())
1895         .env("CFG_VER_MAJOR", parts.next().unwrap())
1896         .env("CFG_VER_MINOR", parts.next().unwrap())
1897         .env("CFG_VER_PATCH", parts.next().unwrap())
1898         .env("CFG_VER_BUILD", "0") // just needed to build
1899         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1900         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1901         .env("CFG_BUILD", target.triple)
1902         .env("CFG_CHANNEL", &builder.config.channel);
1903
1904     if target.contains("windows-gnu") {
1905         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1906     } else {
1907         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1908     }
1909
1910     if target.contains("x86_64") {
1911         cmd.env("CFG_PLATFORM", "x64");
1912     } else {
1913         cmd.env("CFG_PLATFORM", "x86");
1914     }
1915 }
1916
1917 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1918 ///
1919
1920 /// Returns whether the files were actually copied.
1921 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1922     if let Some(config) = builder.config.target_config.get(&target) {
1923         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1924             // If the LLVM was externally provided, then we don't currently copy
1925             // artifacts into the sysroot. This is not necessarily the right
1926             // choice (in particular, it will require the LLVM dylib to be in
1927             // the linker's load path at runtime), but the common use case for
1928             // external LLVMs is distribution provided LLVMs, and in that case
1929             // they're usually in the standard search path (e.g., /usr/lib) and
1930             // copying them here is going to cause problems as we may end up
1931             // with the wrong files and isn't what distributions want.
1932             //
1933             // This behavior may be revisited in the future though.
1934             //
1935             // If the LLVM is coming from ourselves (just from CI) though, we
1936             // still want to install it, as it otherwise won't be available.
1937             return false;
1938         }
1939     }
1940
1941     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1942     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1943     // clear why this is the case, though. llvm-config will emit the versioned
1944     // paths and we don't want those in the sysroot (as we're expecting
1945     // unversioned paths).
1946     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1947         let src_libdir = builder.llvm_out(target).join("lib");
1948         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1949         if llvm_dylib_path.exists() {
1950             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1951         }
1952         !builder.config.dry_run
1953     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1954         let mut cmd = Command::new(llvm_config);
1955         cmd.arg("--libfiles");
1956         builder.verbose(&format!("running {:?}", cmd));
1957         let files = output(&mut cmd);
1958         let build_llvm_out = &builder.llvm_out(builder.config.build);
1959         let target_llvm_out = &builder.llvm_out(target);
1960         for file in files.trim_end().split(' ') {
1961             // If we're not using a custom LLVM, make sure we package for the target.
1962             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1963                 target_llvm_out.join(relative_path)
1964             } else {
1965                 PathBuf::from(file)
1966             };
1967             builder.install(&file, dst_libdir, 0o644);
1968         }
1969         !builder.config.dry_run
1970     } else {
1971         false
1972     }
1973 }
1974
1975 /// Maybe add libLLVM.so to the target lib-dir for linking.
1976 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1977     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1978     // We do not need to copy LLVM files into the sysroot if it is not
1979     // dynamically linked; it is already included into librustc_llvm
1980     // statically.
1981     if builder.config.llvm_link_shared {
1982         maybe_install_llvm(builder, target, &dst_libdir);
1983     }
1984 }
1985
1986 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1987 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1988     let dst_libdir =
1989         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1990     // We do not need to copy LLVM files into the sysroot if it is not
1991     // dynamically linked; it is already included into librustc_llvm
1992     // statically.
1993     if builder.config.llvm_link_shared {
1994         maybe_install_llvm(builder, target, &dst_libdir);
1995     }
1996 }
1997
1998 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1999 pub struct LlvmTools {
2000     pub target: TargetSelection,
2001 }
2002
2003 impl Step for LlvmTools {
2004     type Output = Option<GeneratedTarball>;
2005     const ONLY_HOSTS: bool = true;
2006
2007     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2008         run.path("llvm-tools")
2009     }
2010
2011     fn make_run(run: RunConfig<'_>) {
2012         run.builder.ensure(LlvmTools { target: run.target });
2013     }
2014
2015     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2016         let target = self.target;
2017         assert!(builder.config.extended);
2018
2019         /* run only if llvm-config isn't used */
2020         if let Some(config) = builder.config.target_config.get(&target) {
2021             if let Some(ref _s) = config.llvm_config {
2022                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2023                 return None;
2024             }
2025         }
2026
2027         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2028         tarball.set_overlay(OverlayKind::LLVM);
2029         tarball.is_preview(true);
2030
2031         // Prepare the image directory
2032         let src_bindir = builder.llvm_out(target).join("bin");
2033         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2034         for tool in LLVM_TOOLS {
2035             let exe = src_bindir.join(exe(tool, target));
2036             tarball.add_file(&exe, &dst_bindir, 0o755);
2037         }
2038
2039         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2040         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2041         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2042         // compiler libraries.
2043         maybe_install_llvm_target(builder, target, tarball.image_dir());
2044
2045         Some(tarball.generate())
2046     }
2047 }
2048
2049 // Tarball intended for internal consumption to ease rustc/std development.
2050 //
2051 // Should not be considered stable by end users.
2052 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2053 pub struct RustDev {
2054     pub target: TargetSelection,
2055 }
2056
2057 impl Step for RustDev {
2058     type Output = Option<GeneratedTarball>;
2059     const DEFAULT: bool = true;
2060     const ONLY_HOSTS: bool = true;
2061
2062     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2063         run.path("rust-dev")
2064     }
2065
2066     fn make_run(run: RunConfig<'_>) {
2067         run.builder.ensure(RustDev { target: run.target });
2068     }
2069
2070     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2071         let target = self.target;
2072
2073         /* run only if llvm-config isn't used */
2074         if let Some(config) = builder.config.target_config.get(&target) {
2075             if let Some(ref _s) = config.llvm_config {
2076                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2077                 return None;
2078             }
2079         }
2080
2081         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2082         tarball.set_overlay(OverlayKind::LLVM);
2083
2084         let src_bindir = builder.llvm_out(target).join("bin");
2085         for bin in &[
2086             "llvm-config",
2087             "llvm-ar",
2088             "llvm-objdump",
2089             "llvm-profdata",
2090             "llvm-bcanalyzer",
2091             "llvm-cov",
2092             "llvm-dwp",
2093         ] {
2094             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2095         }
2096         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2097
2098         // Copy the include directory as well; needed mostly to build
2099         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2100         // just broadly useful to be able to link against the bundled LLVM.
2101         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2102
2103         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2104         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2105         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2106         // compiler libraries.
2107         let dst_libdir = tarball.image_dir().join("lib");
2108         maybe_install_llvm(builder, target, &dst_libdir);
2109         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2110         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2111
2112         Some(tarball.generate())
2113     }
2114 }
2115
2116 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2117 /// release process to avoid cloning the monorepo and building stuff.
2118 ///
2119 /// Should not be considered stable by end users.
2120 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2121 pub struct BuildManifest {
2122     pub target: TargetSelection,
2123 }
2124
2125 impl Step for BuildManifest {
2126     type Output = GeneratedTarball;
2127     const DEFAULT: bool = false;
2128     const ONLY_HOSTS: bool = true;
2129
2130     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2131         run.path("src/tools/build-manifest")
2132     }
2133
2134     fn make_run(run: RunConfig<'_>) {
2135         run.builder.ensure(BuildManifest { target: run.target });
2136     }
2137
2138     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2139         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2140
2141         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2142         tarball.add_file(&build_manifest, "bin", 0o755);
2143         tarball.generate()
2144     }
2145 }
2146
2147 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2148 ///
2149 /// Currently this is the PGO profile data.
2150 ///
2151 /// Should not be considered stable by end users.
2152 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2153 pub struct ReproducibleArtifacts {
2154     pub target: TargetSelection,
2155 }
2156
2157 impl Step for ReproducibleArtifacts {
2158     type Output = Option<GeneratedTarball>;
2159     const DEFAULT: bool = true;
2160     const ONLY_HOSTS: bool = true;
2161
2162     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2163         run.path("reproducible")
2164     }
2165
2166     fn make_run(run: RunConfig<'_>) {
2167         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2168     }
2169
2170     fn run(self, builder: &Builder<'_>) -> Self::Output {
2171         let path = builder.config.rust_profile_use.as_ref()?;
2172
2173         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2174         tarball.add_file(path, ".", 0o644);
2175         Some(tarball.generate())
2176     }
2177 }