src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.path("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.path("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.contains("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.path("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.contains("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 for flavor in ["ld", "ld64"] {
 411                     let exe_name = exe(flavor, compiler.host);
 412                     builder
 413                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 414                 }
 415             }
 416
 417             // Man pages
 418             t!(fs::create_dir_all(image.join("share/man/man1")));
 419             let man_src = builder.src.join("src/doc/man");
 420             let man_dst = image.join("share/man/man1");
 421
 422             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 423             // to hardlink, and we don't want to edit the source templates
 424             for file_entry in builder.read_dir(&man_src) {
 425                 let page_src = file_entry.path();
 426                 let page_dst = man_dst.join(file_entry.file_name());
 427                 let src_text = t!(std::fs::read_to_string(&page_src));
 428                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 429                 t!(std::fs::write(&page_dst, &new_text));
 430                 t!(fs::copy(&page_src, &page_dst));
 431             }
 432
 433             // Debugger scripts
 434             builder
 435                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 436
 437             // Misc license info
 438             let cp = |file: &str| {
 439                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 440             };
 441             cp("COPYRIGHT");
 442             cp("LICENSE-APACHE");
 443             cp("LICENSE-MIT");
 444             cp("README.md");
 445         }
 446     }
 447 }
 448
 449 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 450 pub struct DebuggerScripts {
 451     pub sysroot: Interned<PathBuf>,
 452     pub host: TargetSelection,
 453 }
 454
 455 impl Step for DebuggerScripts {
 456     type Output = ();
 457
 458     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 459         run.path("src/lldb_batchmode.py")
 460     }
 461
 462     fn make_run(run: RunConfig<'_>) {
 463         run.builder.ensure(DebuggerScripts {
 464             sysroot: run
 465                 .builder
 466                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 467             host: run.target,
 468         });
 469     }
 470
 471     /// Copies debugger scripts for `target` into the `sysroot` specified.
 472     fn run(self, builder: &Builder<'_>) {
 473         let host = self.host;
 474         let sysroot = self.sysroot;
 475         let dst = sysroot.join("lib/rustlib/etc");
 476         t!(fs::create_dir_all(&dst));
 477         let cp_debugger_script = |file: &str| {
 478             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 479         };
 480         if host.contains("windows-msvc") {
 481             // windbg debugger scripts
 482             builder.install(
 483                 &builder.src.join("src/etc/rust-windbg.cmd"),
 484                 &sysroot.join("bin"),
 485                 0o755,
 486             );
 487
 488             cp_debugger_script("natvis/intrinsic.natvis");
 489             cp_debugger_script("natvis/liballoc.natvis");
 490             cp_debugger_script("natvis/libcore.natvis");
 491             cp_debugger_script("natvis/libstd.natvis");
 492         } else {
 493             cp_debugger_script("rust_types.py");
 494
 495             // gdb debugger scripts
 496             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 497             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 498
 499             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 500             cp_debugger_script("gdb_lookup.py");
 501             cp_debugger_script("gdb_providers.py");
 502
 503             // lldb debugger scripts
 504             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 505
 506             cp_debugger_script("lldb_lookup.py");
 507             cp_debugger_script("lldb_providers.py");
 508             cp_debugger_script("lldb_commands")
 509         }
 510     }
 511 }
 512
 513 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 514     // The only true set of target libraries came from the build triple, so
 515     // let's reduce redundant work by only producing archives from that host.
 516     if compiler.host != builder.config.build {
 517         builder.info("\tskipping, not a build host");
 518         true
 519     } else {
 520         false
 521     }
 522 }
 523
 524 /// Copy stamped files into an image's `target/lib` directory.
 525 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 526     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 527     let self_contained_dst = dst.join("self-contained");
 528     t!(fs::create_dir_all(&dst));
 529     t!(fs::create_dir_all(&self_contained_dst));
 530     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 531         if dependency_type == DependencyType::TargetSelfContained {
 532             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 533         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 534             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: TargetSelection,
 543 }
 544
 545 impl Step for Std {
 546     type Output = Option<GeneratedTarball>;
 547     const DEFAULT: bool = true;
 548
 549     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 550         run.path("rust-std")
 551     }
 552
 553     fn make_run(run: RunConfig<'_>) {
 554         run.builder.ensure(Std {
 555             compiler: run.builder.compiler_for(
 556                 run.builder.top_stage,
 557                 run.builder.config.build,
 558                 run.target,
 559             ),
 560             target: run.target,
 561         });
 562     }
 563
 564     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 565         let compiler = self.compiler;
 566         let target = self.target;
 567
 568         if skip_host_target_lib(builder, compiler) {
 569             return None;
 570         }
 571
 572         builder.ensure(compile::Std { compiler, target });
 573
 574         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 575         tarball.include_target_in_component_name(true);
 576
 577         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 578         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 579         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 580
 581         Some(tarball.generate())
 582     }
 583 }
 584
 585 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 586 pub struct RustcDev {
 587     pub compiler: Compiler,
 588     pub target: TargetSelection,
 589 }
 590
 591 impl Step for RustcDev {
 592     type Output = Option<GeneratedTarball>;
 593     const DEFAULT: bool = true;
 594     const ONLY_HOSTS: bool = true;
 595
 596     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 597         run.path("rustc-dev")
 598     }
 599
 600     fn make_run(run: RunConfig<'_>) {
 601         run.builder.ensure(RustcDev {
 602             compiler: run.builder.compiler_for(
 603                 run.builder.top_stage,
 604                 run.builder.config.build,
 605                 run.target,
 606             ),
 607             target: run.target,
 608         });
 609     }
 610
 611     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 612         let compiler = self.compiler;
 613         let target = self.target;
 614         if skip_host_target_lib(builder, compiler) {
 615             return None;
 616         }
 617
 618         builder.ensure(compile::Rustc { compiler, target });
 619
 620         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 621
 622         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 623         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 624         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 625
 626         let src_files = &["Cargo.lock"];
 627         // This is the reduced set of paths which will become the rustc-dev component
 628         // (essentially the compiler crates and all of their path dependencies).
 629         copy_src_dirs(
 630             builder,
 631             &builder.src,
 632             &["compiler"],
 633             &[],
 634             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 635         );
 636         for file in src_files {
 637             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 638         }
 639
 640         Some(tarball.generate())
 641     }
 642 }
 643
 644 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 645 pub struct Analysis {
 646     pub compiler: Compiler,
 647     pub target: TargetSelection,
 648 }
 649
 650 impl Step for Analysis {
 651     type Output = Option<GeneratedTarball>;
 652     const DEFAULT: bool = true;
 653
 654     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 655         let default = should_build_extended_tool(&run.builder, "analysis");
 656         run.path("rust-analysis").default_condition(default)
 657     }
 658
 659     fn make_run(run: RunConfig<'_>) {
 660         run.builder.ensure(Analysis {
 661             // Find the actual compiler (handling the full bootstrap option) which
 662             // produced the save-analysis data because that data isn't copied
 663             // through the sysroot uplifting.
 664             compiler: run.builder.compiler_for(
 665                 run.builder.top_stage,
 666                 run.builder.config.build,
 667                 run.target,
 668             ),
 669             target: run.target,
 670         });
 671     }
 672
 673     /// Creates a tarball of save-analysis metadata, if available.
 674     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 675         let compiler = self.compiler;
 676         let target = self.target;
 677         if compiler.host != builder.config.build {
 678             return None;
 679         }
 680
 681         builder.ensure(compile::Std { compiler, target });
 682         let src = builder
 683             .stage_out(compiler, Mode::Std)
 684             .join(target.triple)
 685             .join(builder.cargo_dir())
 686             .join("deps")
 687             .join("save-analysis");
 688
 689         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 690         tarball.include_target_in_component_name(true);
 691         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 692         Some(tarball.generate())
 693     }
 694 }
 695
 696 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 697 /// `dst_dir`.
 698 fn copy_src_dirs(
 699     builder: &Builder<'_>,
 700     base: &Path,
 701     src_dirs: &[&str],
 702     exclude_dirs: &[&str],
 703     dst_dir: &Path,
 704 ) {
 705     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 706         let spath = match path.to_str() {
 707             Some(path) => path,
 708             None => return false,
 709         };
 710         if spath.ends_with('~') || spath.ends_with(".pyc") {
 711             return false;
 712         }
 713
 714         const LLVM_PROJECTS: &[&str] = &[
 715             "llvm-project/clang",
 716             "llvm-project\\clang",
 717             "llvm-project/libunwind",
 718             "llvm-project\\libunwind",
 719             "llvm-project/lld",
 720             "llvm-project\\lld",
 721             "llvm-project/lldb",
 722             "llvm-project\\lldb",
 723             "llvm-project/llvm",
 724             "llvm-project\\llvm",
 725             "llvm-project/compiler-rt",
 726             "llvm-project\\compiler-rt",
 727             "llvm-project/cmake",
 728             "llvm-project\\cmake",
 729         ];
 730         if spath.contains("llvm-project")
 731             && !spath.ends_with("llvm-project")
 732             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 733         {
 734             return false;
 735         }
 736
 737         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 738         if LLVM_TEST.iter().any(|path| spath.contains(path))
 739             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 740         {
 741             return false;
 742         }
 743
 744         let full_path = Path::new(dir).join(path);
 745         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 746             return false;
 747         }
 748
 749         let excludes = [
 750             "CVS",
 751             "RCS",
 752             "SCCS",
 753             ".git",
 754             ".gitignore",
 755             ".gitmodules",
 756             ".gitattributes",
 757             ".cvsignore",
 758             ".svn",
 759             ".arch-ids",
 760             "{arch}",
 761             "=RELEASE-ID",
 762             "=meta-update",
 763             "=update",
 764             ".bzr",
 765             ".bzrignore",
 766             ".bzrtags",
 767             ".hg",
 768             ".hgignore",
 769             ".hgrags",
 770             "_darcs",
 771         ];
 772         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 773     }
 774
 775     // Copy the directories using our filter
 776     for item in src_dirs {
 777         let dst = &dst_dir.join(item);
 778         t!(fs::create_dir_all(dst));
 779         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 780     }
 781 }
 782
 783 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 784 pub struct Src;
 785
 786 impl Step for Src {
 787     /// The output path of the src installer tarball
 788     type Output = GeneratedTarball;
 789     const DEFAULT: bool = true;
 790     const ONLY_HOSTS: bool = true;
 791
 792     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 793         run.path("rust-src")
 794     }
 795
 796     fn make_run(run: RunConfig<'_>) {
 797         run.builder.ensure(Src);
 798     }
 799
 800     /// Creates the `rust-src` installer component
 801     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 802         let tarball = Tarball::new_targetless(builder, "rust-src");
 803
 804         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 805         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 806         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 807         // and fix them...
 808         //
 809         // NOTE: if you update the paths here, you also should update the "virtual" path
 810         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 811         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 812
 813         let src_files = ["Cargo.lock"];
 814         // This is the reduced set of paths which will become the rust-src component
 815         // (essentially libstd and all of its path dependencies).
 816         copy_src_dirs(
 817             builder,
 818             &builder.src,
 819             &["library", "src/llvm-project/libunwind"],
 820             &[
 821                 // not needed and contains symlinks which rustup currently
 822                 // chokes on when unpacking.
 823                 "library/backtrace/crates",
 824             ],
 825             &dst_src,
 826         );
 827         for file in src_files.iter() {
 828             builder.copy(&builder.src.join(file), &dst_src.join(file));
 829         }
 830
 831         tarball.generate()
 832     }
 833 }
 834
 835 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 836 pub struct PlainSourceTarball;
 837
 838 impl Step for PlainSourceTarball {
 839     /// Produces the location of the tarball generated
 840     type Output = GeneratedTarball;
 841     const DEFAULT: bool = true;
 842     const ONLY_HOSTS: bool = true;
 843
 844     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 845         let builder = run.builder;
 846         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 847     }
 848
 849     fn make_run(run: RunConfig<'_>) {
 850         run.builder.ensure(PlainSourceTarball);
 851     }
 852
 853     /// Creates the plain source tarball
 854     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 855         let tarball = Tarball::new(builder, "rustc", "src");
 856         let plain_dst_src = tarball.image_dir();
 857
 858         // This is the set of root paths which will become part of the source package
 859         let src_files = [
 860             "COPYRIGHT",
 861             "LICENSE-APACHE",
 862             "LICENSE-MIT",
 863             "CONTRIBUTING.md",
 864             "README.md",
 865             "RELEASES.md",
 866             "configure",
 867             "x.py",
 868             "config.toml.example",
 869             "Cargo.toml",
 870             "Cargo.lock",
 871         ];
 872         let src_dirs = ["src", "compiler", "library"];
 873
 874         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 875
 876         // Copy the files normally
 877         for item in &src_files {
 878             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 879         }
 880
 881         // Create the version file
 882         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 883         if let Some(sha) = builder.rust_sha() {
 884             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 885         }
 886
 887         // If we're building from git sources, we need to vendor a complete distribution.
 888         if builder.rust_info.is_git() {
 889             // Vendor all Cargo dependencies
 890             let mut cmd = Command::new(&builder.initial_cargo);
 891             cmd.arg("vendor")
 892                 .arg("--sync")
 893                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 894                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 895                 .current_dir(&plain_dst_src);
 896             builder.run(&mut cmd);
 897         }
 898
 899         tarball.bare()
 900     }
 901 }
 902
 903 // We have to run a few shell scripts, which choke quite a bit on both `\`
 904 // characters and on `C:\` paths, so normalize both of them away.
 905 pub fn sanitize_sh(path: &Path) -> String {
 906     let path = path.to_str().unwrap().replace("\\", "/");
 907     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 908
 909     fn unc_to_lfs(s: &str) -> &str {
 910         s.strip_prefix("//?/").unwrap_or(s)
 911     }
 912
 913     fn change_drive(s: &str) -> Option<String> {
 914         let mut ch = s.chars();
 915         let drive = ch.next().unwrap_or('C');
 916         if ch.next() != Some(':') {
 917             return None;
 918         }
 919         if ch.next() != Some('/') {
 920             return None;
 921         }
 922         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 923     }
 924 }
 925
 926 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 927 pub struct Cargo {
 928     pub compiler: Compiler,
 929     pub target: TargetSelection,
 930 }
 931
 932 impl Step for Cargo {
 933     type Output = Option<GeneratedTarball>;
 934     const DEFAULT: bool = true;
 935     const ONLY_HOSTS: bool = true;
 936
 937     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 938         let default = should_build_extended_tool(&run.builder, "cargo");
 939         run.path("cargo").default_condition(default)
 940     }
 941
 942     fn make_run(run: RunConfig<'_>) {
 943         run.builder.ensure(Cargo {
 944             compiler: run.builder.compiler_for(
 945                 run.builder.top_stage,
 946                 run.builder.config.build,
 947                 run.target,
 948             ),
 949             target: run.target,
 950         });
 951     }
 952
 953     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 954         let compiler = self.compiler;
 955         let target = self.target;
 956
 957         let cargo = builder.ensure(tool::Cargo { compiler, target });
 958         let src = builder.src.join("src/tools/cargo");
 959         let etc = src.join("src/etc");
 960
 961         // Prepare the image directory
 962         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 963         tarball.set_overlay(OverlayKind::Cargo);
 964
 965         tarball.add_file(&cargo, "bin", 0o755);
 966         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 967         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 968         tarball.add_dir(etc.join("man"), "share/man/man1");
 969         tarball.add_legal_and_readme_to("share/doc/cargo");
 970
 971         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 972             let dirent = dirent.expect("read dir entry");
 973             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 974                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 975             }
 976         }
 977
 978         Some(tarball.generate())
 979     }
 980 }
 981
 982 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 983 pub struct Rls {
 984     pub compiler: Compiler,
 985     pub target: TargetSelection,
 986 }
 987
 988 impl Step for Rls {
 989     type Output = Option<GeneratedTarball>;
 990     const ONLY_HOSTS: bool = true;
 991     const DEFAULT: bool = true;
 992
 993     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 994         let default = should_build_extended_tool(&run.builder, "rls");
 995         run.path("rls").default_condition(default)
 996     }
 997
 998     fn make_run(run: RunConfig<'_>) {
 999         run.builder.ensure(Rls {
1000             compiler: run.builder.compiler_for(
1001                 run.builder.top_stage,
1002                 run.builder.config.build,
1003                 run.target,
1004             ),
1005             target: run.target,
1006         });
1007     }
1008
1009     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1010         let compiler = self.compiler;
1011         let target = self.target;
1012
1013         let rls = builder
1014             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1015             .or_else(|| {
1016                 missing_tool("RLS", builder.build.config.missing_tools);
1017                 None
1018             })?;
1019
1020         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1021         tarball.set_overlay(OverlayKind::RLS);
1022         tarball.is_preview(true);
1023         tarball.add_file(rls, "bin", 0o755);
1024         tarball.add_legal_and_readme_to("share/doc/rls");
1025         Some(tarball.generate())
1026     }
1027 }
1028
1029 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1030 pub struct RustAnalyzer {
1031     pub compiler: Compiler,
1032     pub target: TargetSelection,
1033 }
1034
1035 impl Step for RustAnalyzer {
1036     type Output = Option<GeneratedTarball>;
1037     const DEFAULT: bool = true;
1038     const ONLY_HOSTS: bool = true;
1039
1040     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1041         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1042         run.path("rust-analyzer").default_condition(default)
1043     }
1044
1045     fn make_run(run: RunConfig<'_>) {
1046         run.builder.ensure(RustAnalyzer {
1047             compiler: run.builder.compiler_for(
1048                 run.builder.top_stage,
1049                 run.builder.config.build,
1050                 run.target,
1051             ),
1052             target: run.target,
1053         });
1054     }
1055
1056     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1057         // This prevents rust-analyzer from being built for "dist" or "install"
1058         // on the stable/beta channels. It is a nightly-only tool and should
1059         // not be included.
1060         if !builder.build.unstable_features() {
1061             return None;
1062         }
1063         let compiler = self.compiler;
1064         let target = self.target;
1065
1066         if target.contains("riscv64") {
1067             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1068             // to build. See #74813 for details.
1069             return None;
1070         }
1071
1072         let rust_analyzer = builder
1073             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1074             .expect("rust-analyzer always builds");
1075
1076         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1077         tarball.set_overlay(OverlayKind::RustAnalyzer);
1078         tarball.is_preview(true);
1079         tarball.add_file(rust_analyzer, "bin", 0o755);
1080         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1081         Some(tarball.generate())
1082     }
1083 }
1084
1085 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1086 pub struct Clippy {
1087     pub compiler: Compiler,
1088     pub target: TargetSelection,
1089 }
1090
1091 impl Step for Clippy {
1092     type Output = Option<GeneratedTarball>;
1093     const DEFAULT: bool = true;
1094     const ONLY_HOSTS: bool = true;
1095
1096     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1097         let default = should_build_extended_tool(&run.builder, "clippy");
1098         run.path("clippy").default_condition(default)
1099     }
1100
1101     fn make_run(run: RunConfig<'_>) {
1102         run.builder.ensure(Clippy {
1103             compiler: run.builder.compiler_for(
1104                 run.builder.top_stage,
1105                 run.builder.config.build,
1106                 run.target,
1107             ),
1108             target: run.target,
1109         });
1110     }
1111
1112     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1113         let compiler = self.compiler;
1114         let target = self.target;
1115
1116         // Prepare the image directory
1117         // We expect clippy to build, because we've exited this step above if tool
1118         // state for clippy isn't testing.
1119         let clippy = builder
1120             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1121             .expect("clippy expected to build - essential tool");
1122         let cargoclippy = builder
1123             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1124             .expect("clippy expected to build - essential tool");
1125
1126         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1127         tarball.set_overlay(OverlayKind::Clippy);
1128         tarball.is_preview(true);
1129         tarball.add_file(clippy, "bin", 0o755);
1130         tarball.add_file(cargoclippy, "bin", 0o755);
1131         tarball.add_legal_and_readme_to("share/doc/clippy");
1132         Some(tarball.generate())
1133     }
1134 }
1135
1136 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1137 pub struct Miri {
1138     pub compiler: Compiler,
1139     pub target: TargetSelection,
1140 }
1141
1142 impl Step for Miri {
1143     type Output = Option<GeneratedTarball>;
1144     const DEFAULT: bool = true;
1145     const ONLY_HOSTS: bool = true;
1146
1147     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1148         let default = should_build_extended_tool(&run.builder, "miri");
1149         run.path("miri").default_condition(default)
1150     }
1151
1152     fn make_run(run: RunConfig<'_>) {
1153         run.builder.ensure(Miri {
1154             compiler: run.builder.compiler_for(
1155                 run.builder.top_stage,
1156                 run.builder.config.build,
1157                 run.target,
1158             ),
1159             target: run.target,
1160         });
1161     }
1162
1163     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1164         // This prevents miri from being built for "dist" or "install"
1165         // on the stable/beta channels. It is a nightly-only tool and should
1166         // not be included.
1167         if !builder.build.unstable_features() {
1168             return None;
1169         }
1170         let compiler = self.compiler;
1171         let target = self.target;
1172
1173         let miri = builder
1174             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1175             .or_else(|| {
1176                 missing_tool("miri", builder.build.config.missing_tools);
1177                 None
1178             })?;
1179         let cargomiri = builder
1180             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1181             .or_else(|| {
1182                 missing_tool("cargo miri", builder.build.config.missing_tools);
1183                 None
1184             })?;
1185
1186         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1187         tarball.set_overlay(OverlayKind::Miri);
1188         tarball.is_preview(true);
1189         tarball.add_file(miri, "bin", 0o755);
1190         tarball.add_file(cargomiri, "bin", 0o755);
1191         tarball.add_legal_and_readme_to("share/doc/miri");
1192         Some(tarball.generate())
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Rustfmt {
1198     pub compiler: Compiler,
1199     pub target: TargetSelection,
1200 }
1201
1202 impl Step for Rustfmt {
1203     type Output = Option<GeneratedTarball>;
1204     const DEFAULT: bool = true;
1205     const ONLY_HOSTS: bool = true;
1206
1207     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1208         let default = should_build_extended_tool(&run.builder, "rustfmt");
1209         run.path("rustfmt").default_condition(default)
1210     }
1211
1212     fn make_run(run: RunConfig<'_>) {
1213         run.builder.ensure(Rustfmt {
1214             compiler: run.builder.compiler_for(
1215                 run.builder.top_stage,
1216                 run.builder.config.build,
1217                 run.target,
1218             ),
1219             target: run.target,
1220         });
1221     }
1222
1223     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1224         let compiler = self.compiler;
1225         let target = self.target;
1226
1227         let rustfmt = builder
1228             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1229             .or_else(|| {
1230                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1231                 None
1232             })?;
1233         let cargofmt = builder
1234             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1235             .or_else(|| {
1236                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1237                 None
1238             })?;
1239
1240         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1241         tarball.set_overlay(OverlayKind::Rustfmt);
1242         tarball.is_preview(true);
1243         tarball.add_file(rustfmt, "bin", 0o755);
1244         tarball.add_file(cargofmt, "bin", 0o755);
1245         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1246         Some(tarball.generate())
1247     }
1248 }
1249
1250 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1251 pub struct RustDemangler {
1252     pub compiler: Compiler,
1253     pub target: TargetSelection,
1254 }
1255
1256 impl Step for RustDemangler {
1257     type Output = Option<GeneratedTarball>;
1258     const DEFAULT: bool = true;
1259     const ONLY_HOSTS: bool = true;
1260
1261     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1262         // While other tools use `should_build_extended_tool` to decide whether to be run by
1263         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1264         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1265         // we run the step by default when only `extended = true`, and decide whether to actually
1266         // run it or not later.
1267         let default = run.builder.config.extended;
1268         run.path("rust-demangler").default_condition(default)
1269     }
1270
1271     fn make_run(run: RunConfig<'_>) {
1272         run.builder.ensure(RustDemangler {
1273             compiler: run.builder.compiler_for(
1274                 run.builder.top_stage,
1275                 run.builder.config.build,
1276                 run.target,
1277             ),
1278             target: run.target,
1279         });
1280     }
1281
1282     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1283         let compiler = self.compiler;
1284         let target = self.target;
1285
1286         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1287         let condition = should_build_extended_tool(builder, "rust-demangler")
1288             || builder.config.profiler_enabled(target);
1289         if builder.config.extended && !condition {
1290             return None;
1291         }
1292
1293         let rust_demangler = builder
1294             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1295             .expect("rust-demangler expected to build - in-tree tool");
1296
1297         // Prepare the image directory
1298         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1299         tarball.set_overlay(OverlayKind::RustDemangler);
1300         tarball.is_preview(true);
1301         tarball.add_file(&rust_demangler, "bin", 0o755);
1302         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1303         Some(tarball.generate())
1304     }
1305 }
1306
1307 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1308 pub struct Extended {
1309     stage: u32,
1310     host: TargetSelection,
1311     target: TargetSelection,
1312 }
1313
1314 impl Step for Extended {
1315     type Output = ();
1316     const DEFAULT: bool = true;
1317     const ONLY_HOSTS: bool = true;
1318
1319     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1320         let builder = run.builder;
1321         run.path("extended").default_condition(builder.config.extended)
1322     }
1323
1324     fn make_run(run: RunConfig<'_>) {
1325         run.builder.ensure(Extended {
1326             stage: run.builder.top_stage,
1327             host: run.builder.config.build,
1328             target: run.target,
1329         });
1330     }
1331
1332     /// Creates a combined installer for the specified target in the provided stage.
1333     fn run(self, builder: &Builder<'_>) {
1334         let target = self.target;
1335         let stage = self.stage;
1336         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1337
1338         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1339
1340         let mut tarballs = Vec::new();
1341         let mut built_tools = HashSet::new();
1342         macro_rules! add_component {
1343             ($name:expr => $step:expr) => {
1344                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1345                     tarballs.push(tarball);
1346                     built_tools.insert($name);
1347                 }
1348             };
1349         }
1350
1351         // When rust-std package split from rustc, we needed to ensure that during
1352         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1353         // the std files during uninstall. To do this ensure that rustc comes
1354         // before rust-std in the list below.
1355         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1356         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1357
1358         if target.contains("windows-gnu") {
1359             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1360         }
1361
1362         add_component!("rust-docs" => Docs { host: target });
1363         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1364         add_component!("cargo" => Cargo { compiler, target });
1365         add_component!("rustfmt" => Rustfmt { compiler, target });
1366         add_component!("rls" => Rls { compiler, target });
1367         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1368         add_component!("llvm-components" => LlvmTools { target });
1369         add_component!("clippy" => Clippy { compiler, target });
1370         add_component!("miri" => Miri { compiler, target });
1371         add_component!("analysis" => Analysis { compiler, target });
1372
1373         let etc = builder.src.join("src/etc/installer");
1374
1375         // Avoid producing tarballs during a dry run.
1376         if builder.config.dry_run {
1377             return;
1378         }
1379
1380         let tarball = Tarball::new(builder, "rust", &target.triple);
1381         let generated = tarball.combine(&tarballs);
1382
1383         let tmp = tmpdir(builder).join("combined-tarball");
1384         let work = generated.work_dir();
1385
1386         let mut license = String::new();
1387         license += &builder.read(&builder.src.join("COPYRIGHT"));
1388         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1389         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1390         license.push('\n');
1391         license.push('\n');
1392
1393         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1394         let mut rtf = rtf.to_string();
1395         rtf.push('\n');
1396         for line in license.lines() {
1397             rtf.push_str(line);
1398             rtf.push_str("\\line ");
1399         }
1400         rtf.push('}');
1401
1402         fn filter(contents: &str, marker: &str) -> String {
1403             let start = format!("tool-{}-start", marker);
1404             let end = format!("tool-{}-end", marker);
1405             let mut lines = Vec::new();
1406             let mut omitted = false;
1407             for line in contents.lines() {
1408                 if line.contains(&start) {
1409                     omitted = true;
1410                 } else if line.contains(&end) {
1411                     omitted = false;
1412                 } else if !omitted {
1413                     lines.push(line);
1414                 }
1415             }
1416
1417             lines.join("\n")
1418         }
1419
1420         let xform = |p: &Path| {
1421             let mut contents = t!(fs::read_to_string(p));
1422             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1423                 if !built_tools.contains(tool) {
1424                     contents = filter(&contents, tool);
1425                 }
1426             }
1427             let ret = tmp.join(p.file_name().unwrap());
1428             t!(fs::write(&ret, &contents));
1429             ret
1430         };
1431
1432         if target.contains("apple-darwin") {
1433             builder.info("building pkg installer");
1434             let pkg = tmp.join("pkg");
1435             let _ = fs::remove_dir_all(&pkg);
1436
1437             let pkgbuild = |component: &str| {
1438                 let mut cmd = Command::new("pkgbuild");
1439                 cmd.arg("--identifier")
1440                     .arg(format!("org.rust-lang.{}", component))
1441                     .arg("--scripts")
1442                     .arg(pkg.join(component))
1443                     .arg("--nopayload")
1444                     .arg(pkg.join(component).with_extension("pkg"));
1445                 builder.run(&mut cmd);
1446             };
1447
1448             let prepare = |name: &str| {
1449                 builder.create_dir(&pkg.join(name));
1450                 builder.cp_r(
1451                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1452                     &pkg.join(name),
1453                 );
1454                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1455                 pkgbuild(name);
1456             };
1457             prepare("rustc");
1458             prepare("cargo");
1459             prepare("rust-std");
1460             prepare("rust-analysis");
1461             prepare("clippy");
1462             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1463                 if built_tools.contains(tool) {
1464                     prepare(tool);
1465                 }
1466             }
1467             // create an 'uninstall' package
1468             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1469             pkgbuild("uninstall");
1470
1471             builder.create_dir(&pkg.join("res"));
1472             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1473             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1474             let mut cmd = Command::new("productbuild");
1475             cmd.arg("--distribution")
1476                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1477                 .arg("--resources")
1478                 .arg(pkg.join("res"))
1479                 .arg(distdir(builder).join(format!(
1480                     "{}-{}.pkg",
1481                     pkgname(builder, "rust"),
1482                     target.triple
1483                 )))
1484                 .arg("--package-path")
1485                 .arg(&pkg);
1486             let _time = timeit(builder);
1487             builder.run(&mut cmd);
1488         }
1489
1490         if target.contains("windows") {
1491             let exe = tmp.join("exe");
1492             let _ = fs::remove_dir_all(&exe);
1493
1494             let prepare = |name: &str| {
1495                 builder.create_dir(&exe.join(name));
1496                 let dir = if name == "rust-std" || name == "rust-analysis" {
1497                     format!("{}-{}", name, target.triple)
1498                 } else if name == "rls" {
1499                     "rls-preview".to_string()
1500                 } else if name == "rust-analyzer" {
1501                     "rust-analyzer-preview".to_string()
1502                 } else if name == "clippy" {
1503                     "clippy-preview".to_string()
1504                 } else if name == "rust-demangler" {
1505                     "rust-demangler-preview".to_string()
1506                 } else if name == "miri" {
1507                     "miri-preview".to_string()
1508                 } else {
1509                     name.to_string()
1510                 };
1511                 builder.cp_r(
1512                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1513                     &exe.join(name),
1514                 );
1515                 builder.remove(&exe.join(name).join("manifest.in"));
1516             };
1517             prepare("rustc");
1518             prepare("cargo");
1519             prepare("rust-analysis");
1520             prepare("rust-docs");
1521             prepare("rust-std");
1522             prepare("clippy");
1523             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1524                 if built_tools.contains(tool) {
1525                     prepare(tool);
1526                 }
1527             }
1528             if target.contains("windows-gnu") {
1529                 prepare("rust-mingw");
1530             }
1531
1532             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1533
1534             // Generate msi installer
1535             let wix_path = env::var_os("WIX")
1536                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1537             let wix = PathBuf::from(wix_path);
1538             let heat = wix.join("bin/heat.exe");
1539             let candle = wix.join("bin/candle.exe");
1540             let light = wix.join("bin/light.exe");
1541
1542             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1543             builder.run(
1544                 Command::new(&heat)
1545                     .current_dir(&exe)
1546                     .arg("dir")
1547                     .arg("rustc")
1548                     .args(&heat_flags)
1549                     .arg("-cg")
1550                     .arg("RustcGroup")
1551                     .arg("-dr")
1552                     .arg("Rustc")
1553                     .arg("-var")
1554                     .arg("var.RustcDir")
1555                     .arg("-out")
1556                     .arg(exe.join("RustcGroup.wxs")),
1557             );
1558             builder.run(
1559                 Command::new(&heat)
1560                     .current_dir(&exe)
1561                     .arg("dir")
1562                     .arg("rust-docs")
1563                     .args(&heat_flags)
1564                     .arg("-cg")
1565                     .arg("DocsGroup")
1566                     .arg("-dr")
1567                     .arg("Docs")
1568                     .arg("-var")
1569                     .arg("var.DocsDir")
1570                     .arg("-out")
1571                     .arg(exe.join("DocsGroup.wxs"))
1572                     .arg("-t")
1573                     .arg(etc.join("msi/squash-components.xsl")),
1574             );
1575             builder.run(
1576                 Command::new(&heat)
1577                     .current_dir(&exe)
1578                     .arg("dir")
1579                     .arg("cargo")
1580                     .args(&heat_flags)
1581                     .arg("-cg")
1582                     .arg("CargoGroup")
1583                     .arg("-dr")
1584                     .arg("Cargo")
1585                     .arg("-var")
1586                     .arg("var.CargoDir")
1587                     .arg("-out")
1588                     .arg(exe.join("CargoGroup.wxs"))
1589                     .arg("-t")
1590                     .arg(etc.join("msi/remove-duplicates.xsl")),
1591             );
1592             builder.run(
1593                 Command::new(&heat)
1594                     .current_dir(&exe)
1595                     .arg("dir")
1596                     .arg("rust-std")
1597                     .args(&heat_flags)
1598                     .arg("-cg")
1599                     .arg("StdGroup")
1600                     .arg("-dr")
1601                     .arg("Std")
1602                     .arg("-var")
1603                     .arg("var.StdDir")
1604                     .arg("-out")
1605                     .arg(exe.join("StdGroup.wxs")),
1606             );
1607             if built_tools.contains("rls") {
1608                 builder.run(
1609                     Command::new(&heat)
1610                         .current_dir(&exe)
1611                         .arg("dir")
1612                         .arg("rls")
1613                         .args(&heat_flags)
1614                         .arg("-cg")
1615                         .arg("RlsGroup")
1616                         .arg("-dr")
1617                         .arg("Rls")
1618                         .arg("-var")
1619                         .arg("var.RlsDir")
1620                         .arg("-out")
1621                         .arg(exe.join("RlsGroup.wxs"))
1622                         .arg("-t")
1623                         .arg(etc.join("msi/remove-duplicates.xsl")),
1624                 );
1625             }
1626             if built_tools.contains("rust-analyzer") {
1627                 builder.run(
1628                     Command::new(&heat)
1629                         .current_dir(&exe)
1630                         .arg("dir")
1631                         .arg("rust-analyzer")
1632                         .args(&heat_flags)
1633                         .arg("-cg")
1634                         .arg("RustAnalyzerGroup")
1635                         .arg("-dr")
1636                         .arg("RustAnalyzer")
1637                         .arg("-var")
1638                         .arg("var.RustAnalyzerDir")
1639                         .arg("-out")
1640                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1641                         .arg("-t")
1642                         .arg(etc.join("msi/remove-duplicates.xsl")),
1643                 );
1644             }
1645             builder.run(
1646                 Command::new(&heat)
1647                     .current_dir(&exe)
1648                     .arg("dir")
1649                     .arg("clippy")
1650                     .args(&heat_flags)
1651                     .arg("-cg")
1652                     .arg("ClippyGroup")
1653                     .arg("-dr")
1654                     .arg("Clippy")
1655                     .arg("-var")
1656                     .arg("var.ClippyDir")
1657                     .arg("-out")
1658                     .arg(exe.join("ClippyGroup.wxs"))
1659                     .arg("-t")
1660                     .arg(etc.join("msi/remove-duplicates.xsl")),
1661             );
1662             if built_tools.contains("rust-demangler") {
1663                 builder.run(
1664                     Command::new(&heat)
1665                         .current_dir(&exe)
1666                         .arg("dir")
1667                         .arg("rust-demangler")
1668                         .args(&heat_flags)
1669                         .arg("-cg")
1670                         .arg("RustDemanglerGroup")
1671                         .arg("-dr")
1672                         .arg("RustDemangler")
1673                         .arg("-var")
1674                         .arg("var.RustDemanglerDir")
1675                         .arg("-out")
1676                         .arg(exe.join("RustDemanglerGroup.wxs"))
1677                         .arg("-t")
1678                         .arg(etc.join("msi/remove-duplicates.xsl")),
1679                 );
1680             }
1681             if built_tools.contains("miri") {
1682                 builder.run(
1683                     Command::new(&heat)
1684                         .current_dir(&exe)
1685                         .arg("dir")
1686                         .arg("miri")
1687                         .args(&heat_flags)
1688                         .arg("-cg")
1689                         .arg("MiriGroup")
1690                         .arg("-dr")
1691                         .arg("Miri")
1692                         .arg("-var")
1693                         .arg("var.MiriDir")
1694                         .arg("-out")
1695                         .arg(exe.join("MiriGroup.wxs"))
1696                         .arg("-t")
1697                         .arg(etc.join("msi/remove-duplicates.xsl")),
1698                 );
1699             }
1700             builder.run(
1701                 Command::new(&heat)
1702                     .current_dir(&exe)
1703                     .arg("dir")
1704                     .arg("rust-analysis")
1705                     .args(&heat_flags)
1706                     .arg("-cg")
1707                     .arg("AnalysisGroup")
1708                     .arg("-dr")
1709                     .arg("Analysis")
1710                     .arg("-var")
1711                     .arg("var.AnalysisDir")
1712                     .arg("-out")
1713                     .arg(exe.join("AnalysisGroup.wxs"))
1714                     .arg("-t")
1715                     .arg(etc.join("msi/remove-duplicates.xsl")),
1716             );
1717             if target.contains("windows-gnu") {
1718                 builder.run(
1719                     Command::new(&heat)
1720                         .current_dir(&exe)
1721                         .arg("dir")
1722                         .arg("rust-mingw")
1723                         .args(&heat_flags)
1724                         .arg("-cg")
1725                         .arg("GccGroup")
1726                         .arg("-dr")
1727                         .arg("Gcc")
1728                         .arg("-var")
1729                         .arg("var.GccDir")
1730                         .arg("-out")
1731                         .arg(exe.join("GccGroup.wxs")),
1732                 );
1733             }
1734
1735             let candle = |input: &Path| {
1736                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1737                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1738                 let mut cmd = Command::new(&candle);
1739                 cmd.current_dir(&exe)
1740                     .arg("-nologo")
1741                     .arg("-dRustcDir=rustc")
1742                     .arg("-dDocsDir=rust-docs")
1743                     .arg("-dCargoDir=cargo")
1744                     .arg("-dStdDir=rust-std")
1745                     .arg("-dAnalysisDir=rust-analysis")
1746                     .arg("-dClippyDir=clippy")
1747                     .arg("-arch")
1748                     .arg(&arch)
1749                     .arg("-out")
1750                     .arg(&output)
1751                     .arg(&input);
1752                 add_env(builder, &mut cmd, target);
1753
1754                 if built_tools.contains("rust-demangler") {
1755                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1756                 }
1757                 if built_tools.contains("rls") {
1758                     cmd.arg("-dRlsDir=rls");
1759                 }
1760                 if built_tools.contains("rust-analyzer") {
1761                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1762                 }
1763                 if built_tools.contains("miri") {
1764                     cmd.arg("-dMiriDir=miri");
1765                 }
1766                 if target.contains("windows-gnu") {
1767                     cmd.arg("-dGccDir=rust-mingw");
1768                 }
1769                 builder.run(&mut cmd);
1770             };
1771             candle(&xform(&etc.join("msi/rust.wxs")));
1772             candle(&etc.join("msi/ui.wxs"));
1773             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1774             candle("RustcGroup.wxs".as_ref());
1775             candle("DocsGroup.wxs".as_ref());
1776             candle("CargoGroup.wxs".as_ref());
1777             candle("StdGroup.wxs".as_ref());
1778             candle("ClippyGroup.wxs".as_ref());
1779             if built_tools.contains("rust-demangler") {
1780                 candle("RustDemanglerGroup.wxs".as_ref());
1781             }
1782             if built_tools.contains("rls") {
1783                 candle("RlsGroup.wxs".as_ref());
1784             }
1785             if built_tools.contains("rust-analyzer") {
1786                 candle("RustAnalyzerGroup.wxs".as_ref());
1787             }
1788             if built_tools.contains("miri") {
1789                 candle("MiriGroup.wxs".as_ref());
1790             }
1791             candle("AnalysisGroup.wxs".as_ref());
1792
1793             if target.contains("windows-gnu") {
1794                 candle("GccGroup.wxs".as_ref());
1795             }
1796
1797             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1798             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1799             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1800
1801             builder.info(&format!("building `msi` installer with {:?}", light));
1802             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1803             let mut cmd = Command::new(&light);
1804             cmd.arg("-nologo")
1805                 .arg("-ext")
1806                 .arg("WixUIExtension")
1807                 .arg("-ext")
1808                 .arg("WixUtilExtension")
1809                 .arg("-out")
1810                 .arg(exe.join(&filename))
1811                 .arg("rust.wixobj")
1812                 .arg("ui.wixobj")
1813                 .arg("rustwelcomedlg.wixobj")
1814                 .arg("RustcGroup.wixobj")
1815                 .arg("DocsGroup.wixobj")
1816                 .arg("CargoGroup.wixobj")
1817                 .arg("StdGroup.wixobj")
1818                 .arg("AnalysisGroup.wixobj")
1819                 .arg("ClippyGroup.wixobj")
1820                 .current_dir(&exe);
1821
1822             if built_tools.contains("rls") {
1823                 cmd.arg("RlsGroup.wixobj");
1824             }
1825             if built_tools.contains("rust-analyzer") {
1826                 cmd.arg("RustAnalyzerGroup.wixobj");
1827             }
1828             if built_tools.contains("rust-demangler") {
1829                 cmd.arg("RustDemanglerGroup.wixobj");
1830             }
1831             if built_tools.contains("miri") {
1832                 cmd.arg("MiriGroup.wixobj");
1833             }
1834
1835             if target.contains("windows-gnu") {
1836                 cmd.arg("GccGroup.wixobj");
1837             }
1838             // ICE57 wrongly complains about the shortcuts
1839             cmd.arg("-sice:ICE57");
1840
1841             let _time = timeit(builder);
1842             builder.run(&mut cmd);
1843
1844             if !builder.config.dry_run {
1845                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1846             }
1847         }
1848     }
1849 }
1850
1851 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1852     let mut parts = builder.version.split('.');
1853     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1854         .env("CFG_RELEASE_NUM", &builder.version)
1855         .env("CFG_RELEASE", builder.rust_release())
1856         .env("CFG_VER_MAJOR", parts.next().unwrap())
1857         .env("CFG_VER_MINOR", parts.next().unwrap())
1858         .env("CFG_VER_PATCH", parts.next().unwrap())
1859         .env("CFG_VER_BUILD", "0") // just needed to build
1860         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1861         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1862         .env("CFG_BUILD", target.triple)
1863         .env("CFG_CHANNEL", &builder.config.channel);
1864
1865     if target.contains("windows-gnu") {
1866         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1867     } else {
1868         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1869     }
1870 }
1871
1872 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1873 ///
1874
1875 /// Returns whether the files were actually copied.
1876 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1877     if let Some(config) = builder.config.target_config.get(&target) {
1878         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1879             // If the LLVM was externally provided, then we don't currently copy
1880             // artifacts into the sysroot. This is not necessarily the right
1881             // choice (in particular, it will require the LLVM dylib to be in
1882             // the linker's load path at runtime), but the common use case for
1883             // external LLVMs is distribution provided LLVMs, and in that case
1884             // they're usually in the standard search path (e.g., /usr/lib) and
1885             // copying them here is going to cause problems as we may end up
1886             // with the wrong files and isn't what distributions want.
1887             //
1888             // This behavior may be revisited in the future though.
1889             //
1890             // If the LLVM is coming from ourselves (just from CI) though, we
1891             // still want to install it, as it otherwise won't be available.
1892             return false;
1893         }
1894     }
1895
1896     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1897     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1898     // clear why this is the case, though. llvm-config will emit the versioned
1899     // paths and we don't want those in the sysroot (as we're expecting
1900     // unversioned paths).
1901     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1902         let src_libdir = builder.llvm_out(target).join("lib");
1903         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1904         if llvm_dylib_path.exists() {
1905             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1906         }
1907         !builder.config.dry_run
1908     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1909         let mut cmd = Command::new(llvm_config);
1910         cmd.arg("--libfiles");
1911         builder.verbose(&format!("running {:?}", cmd));
1912         let files = output(&mut cmd);
1913         let build_llvm_out = &builder.llvm_out(builder.config.build);
1914         let target_llvm_out = &builder.llvm_out(target);
1915         for file in files.trim_end().split(' ') {
1916             // If we're not using a custom LLVM, make sure we package for the target.
1917             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1918                 target_llvm_out.join(relative_path)
1919             } else {
1920                 PathBuf::from(file)
1921             };
1922             builder.install(&file, dst_libdir, 0o644);
1923         }
1924         !builder.config.dry_run
1925     } else {
1926         false
1927     }
1928 }
1929
1930 /// Maybe add libLLVM.so to the target lib-dir for linking.
1931 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1932     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1933     // We do not need to copy LLVM files into the sysroot if it is not
1934     // dynamically linked; it is already included into librustc_llvm
1935     // statically.
1936     if builder.config.llvm_link_shared {
1937         maybe_install_llvm(builder, target, &dst_libdir);
1938     }
1939 }
1940
1941 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1942 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1943     let dst_libdir =
1944         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1945     // We do not need to copy LLVM files into the sysroot if it is not
1946     // dynamically linked; it is already included into librustc_llvm
1947     // statically.
1948     if builder.config.llvm_link_shared {
1949         maybe_install_llvm(builder, target, &dst_libdir);
1950     }
1951 }
1952
1953 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1954 pub struct LlvmTools {
1955     pub target: TargetSelection,
1956 }
1957
1958 impl Step for LlvmTools {
1959     type Output = Option<GeneratedTarball>;
1960     const ONLY_HOSTS: bool = true;
1961     const DEFAULT: bool = true;
1962
1963     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1964         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1965         run.path("llvm-tools").default_condition(default)
1966     }
1967
1968     fn make_run(run: RunConfig<'_>) {
1969         run.builder.ensure(LlvmTools { target: run.target });
1970     }
1971
1972     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1973         let target = self.target;
1974
1975         /* run only if llvm-config isn't used */
1976         if let Some(config) = builder.config.target_config.get(&target) {
1977             if let Some(ref _s) = config.llvm_config {
1978                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1979                 return None;
1980             }
1981         }
1982
1983         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1984         tarball.set_overlay(OverlayKind::LLVM);
1985         tarball.is_preview(true);
1986
1987         // Prepare the image directory
1988         let src_bindir = builder.llvm_out(target).join("bin");
1989         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1990         for tool in LLVM_TOOLS {
1991             let exe = src_bindir.join(exe(tool, target));
1992             tarball.add_file(&exe, &dst_bindir, 0o755);
1993         }
1994
1995         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1996         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1997         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1998         // compiler libraries.
1999         maybe_install_llvm_target(builder, target, tarball.image_dir());
2000
2001         Some(tarball.generate())
2002     }
2003 }
2004
2005 // Tarball intended for internal consumption to ease rustc/std development.
2006 //
2007 // Should not be considered stable by end users.
2008 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2009 pub struct RustDev {
2010     pub target: TargetSelection,
2011 }
2012
2013 impl Step for RustDev {
2014     type Output = Option<GeneratedTarball>;
2015     const DEFAULT: bool = true;
2016     const ONLY_HOSTS: bool = true;
2017
2018     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2019         run.path("rust-dev")
2020     }
2021
2022     fn make_run(run: RunConfig<'_>) {
2023         run.builder.ensure(RustDev { target: run.target });
2024     }
2025
2026     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2027         let target = self.target;
2028
2029         /* run only if llvm-config isn't used */
2030         if let Some(config) = builder.config.target_config.get(&target) {
2031             if let Some(ref _s) = config.llvm_config {
2032                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2033                 return None;
2034             }
2035         }
2036
2037         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2038         tarball.set_overlay(OverlayKind::LLVM);
2039
2040         let src_bindir = builder.llvm_out(target).join("bin");
2041         for bin in &[
2042             "llvm-config",
2043             "llvm-ar",
2044             "llvm-objdump",
2045             "llvm-profdata",
2046             "llvm-bcanalyzer",
2047             "llvm-cov",
2048             "llvm-dwp",
2049             "llvm-nm",
2050         ] {
2051             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2052         }
2053
2054         // We don't build LLD on some platforms, so only add it if it exists
2055         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2056         if lld_path.exists() {
2057             tarball.add_file(lld_path, "bin", 0o755);
2058         }
2059
2060         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2061
2062         // Copy the include directory as well; needed mostly to build
2063         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2064         // just broadly useful to be able to link against the bundled LLVM.
2065         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2066
2067         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2068         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2069         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2070         // compiler libraries.
2071         let dst_libdir = tarball.image_dir().join("lib");
2072         maybe_install_llvm(builder, target, &dst_libdir);
2073         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2074         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2075
2076         Some(tarball.generate())
2077     }
2078 }
2079
2080 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2081 /// release process to avoid cloning the monorepo and building stuff.
2082 ///
2083 /// Should not be considered stable by end users.
2084 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2085 pub struct BuildManifest {
2086     pub target: TargetSelection,
2087 }
2088
2089 impl Step for BuildManifest {
2090     type Output = GeneratedTarball;
2091     const DEFAULT: bool = false;
2092     const ONLY_HOSTS: bool = true;
2093
2094     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2095         run.path("build-manifest")
2096     }
2097
2098     fn make_run(run: RunConfig<'_>) {
2099         run.builder.ensure(BuildManifest { target: run.target });
2100     }
2101
2102     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2103         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2104
2105         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2106         tarball.add_file(&build_manifest, "bin", 0o755);
2107         tarball.generate()
2108     }
2109 }
2110
2111 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2112 ///
2113 /// Currently this is the PGO profile data.
2114 ///
2115 /// Should not be considered stable by end users.
2116 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2117 pub struct ReproducibleArtifacts {
2118     pub target: TargetSelection,
2119 }
2120
2121 impl Step for ReproducibleArtifacts {
2122     type Output = Option<GeneratedTarball>;
2123     const DEFAULT: bool = true;
2124     const ONLY_HOSTS: bool = true;
2125
2126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2127         run.path("reproducible-artifacts")
2128     }
2129
2130     fn make_run(run: RunConfig<'_>) {
2131         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2132     }
2133
2134     fn run(self, builder: &Builder<'_>) -> Self::Output {
2135         let mut added_anything = false;
2136         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2137         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2138             tarball.add_file(path, ".", 0o644);
2139             added_anything = true;
2140         }
2141         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2142             tarball.add_file(path, ".", 0o644);
2143             added_anything = true;
2144         }
2145         if added_anything { Some(tarball.generate()) } else { None }
2146     }
2147 }