src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = GeneratedTarball;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("src/doc").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         tarball.generate()
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         run.path("src/librustc")
 100     }
 101
 102     fn make_run(run: RunConfig<'_>) {
 103         run.builder.ensure(RustcDocs { host: run.target });
 104     }
 105
 106     /// Builds the `rustc-docs` installer component.
 107     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 108         let host = self.host;
 109         if !builder.config.compiler_docs {
 110             return None;
 111         }
 112         builder.default_doc(&[]);
 113
 114         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 115         tarball.set_product_name("Rustc Documentation");
 116         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 117         Some(tarball.generate())
 118     }
 119 }
 120
 121 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 122     let mut found = Vec::with_capacity(files.len());
 123
 124     for file in files {
 125         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 126
 127         if let Some(file_path) = file_path {
 128             found.push(file_path);
 129         } else {
 130             panic!("Could not find '{}' in {:?}", file, path);
 131         }
 132     }
 133
 134     found
 135 }
 136
 137 fn make_win_dist(
 138     rust_root: &Path,
 139     plat_root: &Path,
 140     target: TargetSelection,
 141     builder: &Builder<'_>,
 142 ) {
 143     //Ask gcc where it keeps its stuff
 144     let mut cmd = Command::new(builder.cc(target));
 145     cmd.arg("-print-search-dirs");
 146     let gcc_out = output(&mut cmd);
 147
 148     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 149     let mut lib_path = Vec::new();
 150
 151     for line in gcc_out.lines() {
 152         let idx = line.find(':').unwrap();
 153         let key = &line[..idx];
 154         let trim_chars: &[_] = &[' ', '='];
 155         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 156
 157         if key == "programs" {
 158             bin_path.extend(value);
 159         } else if key == "libraries" {
 160             lib_path.extend(value);
 161         }
 162     }
 163
 164     let compiler = if target == "i686-pc-windows-gnu" {
 165         "i686-w64-mingw32-gcc.exe"
 166     } else if target == "x86_64-pc-windows-gnu" {
 167         "x86_64-w64-mingw32-gcc.exe"
 168     } else {
 169         "gcc.exe"
 170     };
 171     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 172     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 173     if target.starts_with("i686-") {
 174         rustc_dlls.push("libgcc_s_dw2-1.dll");
 175     } else {
 176         rustc_dlls.push("libgcc_s_seh-1.dll");
 177     }
 178
 179     let target_libs = [
 180         //MinGW libs
 181         "libgcc.a",
 182         "libgcc_eh.a",
 183         "libgcc_s.a",
 184         "libm.a",
 185         "libmingw32.a",
 186         "libmingwex.a",
 187         "libstdc++.a",
 188         "libiconv.a",
 189         "libmoldname.a",
 190         "libpthread.a",
 191         //Windows import libs
 192         "libadvapi32.a",
 193         "libbcrypt.a",
 194         "libcomctl32.a",
 195         "libcomdlg32.a",
 196         "libcredui.a",
 197         "libcrypt32.a",
 198         "libdbghelp.a",
 199         "libgdi32.a",
 200         "libimagehlp.a",
 201         "libiphlpapi.a",
 202         "libkernel32.a",
 203         "libmsimg32.a",
 204         "libmsvcrt.a",
 205         "libodbc32.a",
 206         "libole32.a",
 207         "liboleaut32.a",
 208         "libopengl32.a",
 209         "libpsapi.a",
 210         "librpcrt4.a",
 211         "libsecur32.a",
 212         "libsetupapi.a",
 213         "libshell32.a",
 214         "libsynchronization.a",
 215         "libuser32.a",
 216         "libuserenv.a",
 217         "libuuid.a",
 218         "libwinhttp.a",
 219         "libwinmm.a",
 220         "libwinspool.a",
 221         "libws2_32.a",
 222         "libwsock32.a",
 223     ];
 224
 225     //Find mingw artifacts we want to bundle
 226     let target_tools = find_files(&target_tools, &bin_path);
 227     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 228     let target_libs = find_files(&target_libs, &lib_path);
 229
 230     // Copy runtime dlls next to rustc.exe
 231     let dist_bin_dir = rust_root.join("bin/");
 232     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 233     for src in rustc_dlls {
 234         builder.copy_to_folder(&src, &dist_bin_dir);
 235     }
 236
 237     //Copy platform tools to platform-specific bin directory
 238     let target_bin_dir = plat_root
 239         .join("lib")
 240         .join("rustlib")
 241         .join(target.triple)
 242         .join("bin")
 243         .join("self-contained");
 244     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 245     for src in target_tools {
 246         builder.copy_to_folder(&src, &target_bin_dir);
 247     }
 248
 249     // Warn windows-gnu users that the bundled GCC cannot compile C files
 250     builder.create(
 251         &target_bin_dir.join("GCC-WARNING.txt"),
 252         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 253          used as a linker. In order to be able to compile projects containing C code use \
 254          the GCC provided by MinGW or Cygwin.",
 255     );
 256
 257     //Copy platform libs to platform-specific lib directory
 258     let target_lib_dir = plat_root
 259         .join("lib")
 260         .join("rustlib")
 261         .join(target.triple)
 262         .join("lib")
 263         .join("self-contained");
 264     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 265     for src in target_libs {
 266         builder.copy_to_folder(&src, &target_lib_dir);
 267     }
 268 }
 269
 270 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 271 pub struct Mingw {
 272     pub host: TargetSelection,
 273 }
 274
 275 impl Step for Mingw {
 276     type Output = Option<GeneratedTarball>;
 277     const DEFAULT: bool = true;
 278
 279     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 280         run.never()
 281     }
 282
 283     fn make_run(run: RunConfig<'_>) {
 284         run.builder.ensure(Mingw { host: run.target });
 285     }
 286
 287     /// Builds the `rust-mingw` installer component.
 288     ///
 289     /// This contains all the bits and pieces to run the MinGW Windows targets
 290     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 291     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 292         let host = self.host;
 293         if !host.contains("pc-windows-gnu") {
 294             return None;
 295         }
 296
 297         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 298         tarball.set_product_name("Rust MinGW");
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 305
 306         Some(tarball.generate())
 307     }
 308 }
 309
 310 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 311 pub struct Rustc {
 312     pub compiler: Compiler,
 313 }
 314
 315 impl Step for Rustc {
 316     type Output = GeneratedTarball;
 317     const DEFAULT: bool = true;
 318     const ONLY_HOSTS: bool = true;
 319
 320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 321         run.path("src/librustc")
 322     }
 323
 324     fn make_run(run: RunConfig<'_>) {
 325         run.builder
 326             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 327     }
 328
 329     /// Creates the `rustc` installer component.
 330     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 331         let compiler = self.compiler;
 332         let host = self.compiler.host;
 333
 334         let tarball = Tarball::new(builder, "rustc", &host.triple);
 335
 336         // Prepare the rustc "image", what will actually end up getting installed
 337         prepare_image(builder, compiler, tarball.image_dir());
 338
 339         // On MinGW we've got a few runtime DLL dependencies that we need to
 340         // include. The first argument to this script is where to put these DLLs
 341         // (the image we're creating), and the second argument is a junk directory
 342         // to ignore all other MinGW stuff the script creates.
 343         //
 344         // On 32-bit MinGW we're always including a DLL which needs some extra
 345         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 346         // anything requiring us to distribute a license, but it's likely the
 347         // install will *also* include the rust-mingw package, which also needs
 348         // licenses, so to be safe we just include it here in all MinGW packages.
 349         if host.contains("pc-windows-gnu") {
 350             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 351             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 352         }
 353
 354         return tarball.generate();
 355
 356         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 357             let host = compiler.host;
 358             let src = builder.sysroot(compiler);
 359
 360             // Copy rustc/rustdoc binaries
 361             t!(fs::create_dir_all(image.join("bin")));
 362             builder.cp_r(&src.join("bin"), &image.join("bin"));
 363
 364             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 365
 366             let libdir_relative = builder.libdir_relative(compiler);
 367
 368             // Copy runtime DLLs needed by the compiler
 369             if libdir_relative.to_str() != Some("bin") {
 370                 let libdir = builder.rustc_libdir(compiler);
 371                 for entry in builder.read_dir(&libdir) {
 372                     let name = entry.file_name();
 373                     if let Some(s) = name.to_str() {
 374                         if is_dylib(s) {
 375                             // Don't use custom libdir here because ^lib/ will be resolved again
 376                             // with installer
 377                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 378                         }
 379                     }
 380                 }
 381             }
 382
 383             // Copy over the codegen backends
 384             let backends_src = builder.sysroot_codegen_backends(compiler);
 385             let backends_rel = backends_src
 386                 .strip_prefix(&src)
 387                 .unwrap()
 388                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 389                 .unwrap();
 390             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 391             let backends_dst = image.join("lib").join(&backends_rel);
 392
 393             t!(fs::create_dir_all(&backends_dst));
 394             builder.cp_r(&backends_src, &backends_dst);
 395
 396             // Copy libLLVM.so to the lib dir as well, if needed. While not
 397             // technically needed by rustc itself it's needed by lots of other
 398             // components like the llvm tools and LLD. LLD is included below and
 399             // tools/LLDB come later, so let's just throw it in the rustc
 400             // component for now.
 401             maybe_install_llvm_runtime(builder, host, image);
 402
 403             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 405             t!(fs::create_dir_all(&dst_dir));
 406
 407             // Copy over lld if it's there
 408             if builder.config.lld_enabled {
 409                 let rust_lld = exe("rust-lld", compiler.host);
 410                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 411                 // for `-Z gcc-ld=lld`
 412                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 413                 t!(fs::create_dir(&gcc_lld_dir));
 414                 builder.copy(&src_dir.join(&rust_lld), &gcc_lld_dir.join(exe("ld", compiler.host)));
 415             }
 416
 417             // Copy over llvm-dwp if it's there
 418             let exe = exe("rust-llvm-dwp", compiler.host);
 419             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 420
 421             // Man pages
 422             t!(fs::create_dir_all(image.join("share/man/man1")));
 423             let man_src = builder.src.join("src/doc/man");
 424             let man_dst = image.join("share/man/man1");
 425
 426             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 427             let time = env::var("SOURCE_DATE_EPOCH")
 428                 .map(|timestamp| {
 429                     let epoch = timestamp
 430                         .parse()
 431                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 432                         .unwrap();
 433
 434                     time::at(Timespec::new(epoch, 0))
 435                 })
 436                 .unwrap_or_else(|_| time::now());
 437
 438             let month_year = t!(time::strftime("%B %Y", &time));
 439             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 440             // to hardlink, and we don't want to edit the source templates
 441             for file_entry in builder.read_dir(&man_src) {
 442                 let page_src = file_entry.path();
 443                 let page_dst = man_dst.join(file_entry.file_name());
 444                 t!(fs::copy(&page_src, &page_dst));
 445                 // template in month/year and version number
 446                 builder.replace_in_file(
 447                     &page_dst,
 448                     &[
 449                         ("<INSERT DATE HERE>", &month_year),
 450                         ("<INSERT VERSION HERE>", &builder.version),
 451                     ],
 452                 );
 453             }
 454
 455             // Debugger scripts
 456             builder
 457                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 458
 459             // Misc license info
 460             let cp = |file: &str| {
 461                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 462             };
 463             cp("COPYRIGHT");
 464             cp("LICENSE-APACHE");
 465             cp("LICENSE-MIT");
 466             cp("README.md");
 467         }
 468     }
 469 }
 470
 471 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 472 pub struct DebuggerScripts {
 473     pub sysroot: Interned<PathBuf>,
 474     pub host: TargetSelection,
 475 }
 476
 477 impl Step for DebuggerScripts {
 478     type Output = ();
 479
 480     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 481         run.path("src/lldb_batchmode.py")
 482     }
 483
 484     fn make_run(run: RunConfig<'_>) {
 485         run.builder.ensure(DebuggerScripts {
 486             sysroot: run
 487                 .builder
 488                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 489             host: run.target,
 490         });
 491     }
 492
 493     /// Copies debugger scripts for `target` into the `sysroot` specified.
 494     fn run(self, builder: &Builder<'_>) {
 495         let host = self.host;
 496         let sysroot = self.sysroot;
 497         let dst = sysroot.join("lib/rustlib/etc");
 498         t!(fs::create_dir_all(&dst));
 499         let cp_debugger_script = |file: &str| {
 500             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 501         };
 502         if host.contains("windows-msvc") {
 503             // windbg debugger scripts
 504             builder.install(
 505                 &builder.src.join("src/etc/rust-windbg.cmd"),
 506                 &sysroot.join("bin"),
 507                 0o755,
 508             );
 509
 510             cp_debugger_script("natvis/intrinsic.natvis");
 511             cp_debugger_script("natvis/liballoc.natvis");
 512             cp_debugger_script("natvis/libcore.natvis");
 513             cp_debugger_script("natvis/libstd.natvis");
 514         } else {
 515             cp_debugger_script("rust_types.py");
 516
 517             // gdb debugger scripts
 518             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 519             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 520
 521             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 522             cp_debugger_script("gdb_lookup.py");
 523             cp_debugger_script("gdb_providers.py");
 524
 525             // lldb debugger scripts
 526             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 527
 528             cp_debugger_script("lldb_lookup.py");
 529             cp_debugger_script("lldb_providers.py");
 530             cp_debugger_script("lldb_commands")
 531         }
 532     }
 533 }
 534
 535 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 536     // The only true set of target libraries came from the build triple, so
 537     // let's reduce redundant work by only producing archives from that host.
 538     if compiler.host != builder.config.build {
 539         builder.info("\tskipping, not a build host");
 540         true
 541     } else {
 542         false
 543     }
 544 }
 545
 546 /// Copy stamped files into an image's `target/lib` directory.
 547 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 548     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 549     let self_contained_dst = dst.join("self-contained");
 550     t!(fs::create_dir_all(&dst));
 551     t!(fs::create_dir_all(&self_contained_dst));
 552     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 553         if dependency_type == DependencyType::TargetSelfContained {
 554             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 555         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 556             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 557         }
 558     }
 559 }
 560
 561 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 562 pub struct Std {
 563     pub compiler: Compiler,
 564     pub target: TargetSelection,
 565 }
 566
 567 impl Step for Std {
 568     type Output = Option<GeneratedTarball>;
 569     const DEFAULT: bool = true;
 570
 571     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 572         run.path("library/std")
 573     }
 574
 575     fn make_run(run: RunConfig<'_>) {
 576         run.builder.ensure(Std {
 577             compiler: run.builder.compiler_for(
 578                 run.builder.top_stage,
 579                 run.builder.config.build,
 580                 run.target,
 581             ),
 582             target: run.target,
 583         });
 584     }
 585
 586     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 587         let compiler = self.compiler;
 588         let target = self.target;
 589
 590         if skip_host_target_lib(builder, compiler) {
 591             return None;
 592         }
 593
 594         builder.ensure(compile::Std { compiler, target });
 595
 596         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 597         tarball.include_target_in_component_name(true);
 598
 599         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 600         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 601         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 602
 603         Some(tarball.generate())
 604     }
 605 }
 606
 607 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 608 pub struct RustcDev {
 609     pub compiler: Compiler,
 610     pub target: TargetSelection,
 611 }
 612
 613 impl Step for RustcDev {
 614     type Output = Option<GeneratedTarball>;
 615     const DEFAULT: bool = true;
 616     const ONLY_HOSTS: bool = true;
 617
 618     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 619         run.path("rustc-dev")
 620     }
 621
 622     fn make_run(run: RunConfig<'_>) {
 623         run.builder.ensure(RustcDev {
 624             compiler: run.builder.compiler_for(
 625                 run.builder.top_stage,
 626                 run.builder.config.build,
 627                 run.target,
 628             ),
 629             target: run.target,
 630         });
 631     }
 632
 633     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 634         let compiler = self.compiler;
 635         let target = self.target;
 636         if skip_host_target_lib(builder, compiler) {
 637             return None;
 638         }
 639
 640         builder.ensure(compile::Rustc { compiler, target });
 641
 642         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 643
 644         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 645         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 646         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 647
 648         let src_files = &["Cargo.lock"];
 649         // This is the reduced set of paths which will become the rustc-dev component
 650         // (essentially the compiler crates and all of their path dependencies).
 651         copy_src_dirs(
 652             builder,
 653             &builder.src,
 654             &["compiler"],
 655             &[],
 656             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 657         );
 658         // This particular crate is used as a build dependency of the above.
 659         copy_src_dirs(
 660             builder,
 661             &builder.src,
 662             &["src/build_helper"],
 663             &[],
 664             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 665         );
 666         for file in src_files {
 667             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 668         }
 669
 670         Some(tarball.generate())
 671     }
 672 }
 673
 674 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 675 pub struct Analysis {
 676     pub compiler: Compiler,
 677     pub target: TargetSelection,
 678 }
 679
 680 impl Step for Analysis {
 681     type Output = Option<GeneratedTarball>;
 682
 683     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 684         let default = should_build_extended_tool(&run.builder, "analysis");
 685         run.path("analysis").default_condition(default)
 686     }
 687
 688     fn make_run(run: RunConfig<'_>) {
 689         run.builder.ensure(Analysis {
 690             // Find the actual compiler (handling the full bootstrap option) which
 691             // produced the save-analysis data because that data isn't copied
 692             // through the sysroot uplifting.
 693             compiler: run.builder.compiler_for(
 694                 run.builder.top_stage,
 695                 run.builder.config.build,
 696                 run.target,
 697             ),
 698             target: run.target,
 699         });
 700     }
 701
 702     /// Creates a tarball of save-analysis metadata, if available.
 703     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 704         let compiler = self.compiler;
 705         let target = self.target;
 706         if compiler.host != builder.config.build {
 707             return None;
 708         }
 709
 710         builder.ensure(compile::Std { compiler, target });
 711         let src = builder
 712             .stage_out(compiler, Mode::Std)
 713             .join(target.triple)
 714             .join(builder.cargo_dir())
 715             .join("deps")
 716             .join("save-analysis");
 717
 718         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 719         tarball.include_target_in_component_name(true);
 720         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 721         Some(tarball.generate())
 722     }
 723 }
 724
 725 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 726 /// `dst_dir`.
 727 fn copy_src_dirs(
 728     builder: &Builder<'_>,
 729     base: &Path,
 730     src_dirs: &[&str],
 731     exclude_dirs: &[&str],
 732     dst_dir: &Path,
 733 ) {
 734     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 735         let spath = match path.to_str() {
 736             Some(path) => path,
 737             None => return false,
 738         };
 739         if spath.ends_with('~') || spath.ends_with(".pyc") {
 740             return false;
 741         }
 742
 743         const LLVM_PROJECTS: &[&str] = &[
 744             "llvm-project/clang",
 745             "llvm-project\\clang",
 746             "llvm-project/libunwind",
 747             "llvm-project\\libunwind",
 748             "llvm-project/lld",
 749             "llvm-project\\lld",
 750             "llvm-project/lldb",
 751             "llvm-project\\lldb",
 752             "llvm-project/llvm",
 753             "llvm-project\\llvm",
 754             "llvm-project/compiler-rt",
 755             "llvm-project\\compiler-rt",
 756         ];
 757         if spath.contains("llvm-project")
 758             && !spath.ends_with("llvm-project")
 759             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 760         {
 761             return false;
 762         }
 763
 764         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 765         if LLVM_TEST.iter().any(|path| spath.contains(path))
 766             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 767         {
 768             return false;
 769         }
 770
 771         let full_path = Path::new(dir).join(path);
 772         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 773             return false;
 774         }
 775
 776         let excludes = [
 777             "CVS",
 778             "RCS",
 779             "SCCS",
 780             ".git",
 781             ".gitignore",
 782             ".gitmodules",
 783             ".gitattributes",
 784             ".cvsignore",
 785             ".svn",
 786             ".arch-ids",
 787             "{arch}",
 788             "=RELEASE-ID",
 789             "=meta-update",
 790             "=update",
 791             ".bzr",
 792             ".bzrignore",
 793             ".bzrtags",
 794             ".hg",
 795             ".hgignore",
 796             ".hgrags",
 797             "_darcs",
 798         ];
 799         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 800     }
 801
 802     // Copy the directories using our filter
 803     for item in src_dirs {
 804         let dst = &dst_dir.join(item);
 805         t!(fs::create_dir_all(dst));
 806         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 807     }
 808 }
 809
 810 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 811 pub struct Src;
 812
 813 impl Step for Src {
 814     /// The output path of the src installer tarball
 815     type Output = GeneratedTarball;
 816     const DEFAULT: bool = true;
 817     const ONLY_HOSTS: bool = true;
 818
 819     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 820         run.path("src")
 821     }
 822
 823     fn make_run(run: RunConfig<'_>) {
 824         run.builder.ensure(Src);
 825     }
 826
 827     /// Creates the `rust-src` installer component
 828     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 829         let tarball = Tarball::new_targetless(builder, "rust-src");
 830
 831         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 832         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 833         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 834         // and fix them...
 835         //
 836         // NOTE: if you update the paths here, you also should update the "virtual" path
 837         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 838         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 839
 840         let src_files = ["Cargo.lock"];
 841         // This is the reduced set of paths which will become the rust-src component
 842         // (essentially libstd and all of its path dependencies).
 843         copy_src_dirs(
 844             builder,
 845             &builder.src,
 846             &["library", "src/llvm-project/libunwind"],
 847             &[
 848                 // not needed and contains symlinks which rustup currently
 849                 // chokes on when unpacking.
 850                 "library/backtrace/crates",
 851             ],
 852             &dst_src,
 853         );
 854         for file in src_files.iter() {
 855             builder.copy(&builder.src.join(file), &dst_src.join(file));
 856         }
 857
 858         tarball.generate()
 859     }
 860 }
 861
 862 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 863 pub struct PlainSourceTarball;
 864
 865 impl Step for PlainSourceTarball {
 866     /// Produces the location of the tarball generated
 867     type Output = GeneratedTarball;
 868     const DEFAULT: bool = true;
 869     const ONLY_HOSTS: bool = true;
 870
 871     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 872         let builder = run.builder;
 873         run.path("src").default_condition(builder.config.rust_dist_src)
 874     }
 875
 876     fn make_run(run: RunConfig<'_>) {
 877         run.builder.ensure(PlainSourceTarball);
 878     }
 879
 880     /// Creates the plain source tarball
 881     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 882         let tarball = Tarball::new(builder, "rustc", "src");
 883         let plain_dst_src = tarball.image_dir();
 884
 885         // This is the set of root paths which will become part of the source package
 886         let src_files = [
 887             "COPYRIGHT",
 888             "LICENSE-APACHE",
 889             "LICENSE-MIT",
 890             "CONTRIBUTING.md",
 891             "README.md",
 892             "RELEASES.md",
 893             "configure",
 894             "x.py",
 895             "config.toml.example",
 896             "Cargo.toml",
 897             "Cargo.lock",
 898         ];
 899         let src_dirs = ["src", "compiler", "library"];
 900
 901         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 902
 903         // Copy the files normally
 904         for item in &src_files {
 905             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 906         }
 907
 908         // Create the version file
 909         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 910         if let Some(sha) = builder.rust_sha() {
 911             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 912         }
 913
 914         // If we're building from git sources, we need to vendor a complete distribution.
 915         if builder.rust_info.is_git() {
 916             // Vendor all Cargo dependencies
 917             let mut cmd = Command::new(&builder.initial_cargo);
 918             cmd.arg("vendor")
 919                 .arg("--sync")
 920                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 921                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 922                 .current_dir(&plain_dst_src);
 923             builder.run(&mut cmd);
 924         }
 925
 926         tarball.bare()
 927     }
 928 }
 929
 930 // We have to run a few shell scripts, which choke quite a bit on both `\`
 931 // characters and on `C:\` paths, so normalize both of them away.
 932 pub fn sanitize_sh(path: &Path) -> String {
 933     let path = path.to_str().unwrap().replace("\\", "/");
 934     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 935
 936     fn unc_to_lfs(s: &str) -> &str {
 937         s.strip_prefix("//?/").unwrap_or(s)
 938     }
 939
 940     fn change_drive(s: &str) -> Option<String> {
 941         let mut ch = s.chars();
 942         let drive = ch.next().unwrap_or('C');
 943         if ch.next() != Some(':') {
 944             return None;
 945         }
 946         if ch.next() != Some('/') {
 947             return None;
 948         }
 949         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 950     }
 951 }
 952
 953 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 954 pub struct Cargo {
 955     pub compiler: Compiler,
 956     pub target: TargetSelection,
 957 }
 958
 959 impl Step for Cargo {
 960     type Output = Option<GeneratedTarball>;
 961     const ONLY_HOSTS: bool = true;
 962
 963     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 964         let default = should_build_extended_tool(&run.builder, "cargo");
 965         run.path("cargo").default_condition(default)
 966     }
 967
 968     fn make_run(run: RunConfig<'_>) {
 969         run.builder.ensure(Cargo {
 970             compiler: run.builder.compiler_for(
 971                 run.builder.top_stage,
 972                 run.builder.config.build,
 973                 run.target,
 974             ),
 975             target: run.target,
 976         });
 977     }
 978
 979     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 980         let compiler = self.compiler;
 981         let target = self.target;
 982
 983         let cargo = builder.ensure(tool::Cargo { compiler, target });
 984         let src = builder.src.join("src/tools/cargo");
 985         let etc = src.join("src/etc");
 986
 987         // Prepare the image directory
 988         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 989         tarball.set_overlay(OverlayKind::Cargo);
 990
 991         tarball.add_file(&cargo, "bin", 0o755);
 992         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 993         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 994         tarball.add_dir(etc.join("man"), "share/man/man1");
 995         tarball.add_legal_and_readme_to("share/doc/cargo");
 996
 997         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 998             let dirent = dirent.expect("read dir entry");
 999             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1000                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1001             }
1002         }
1003
1004         Some(tarball.generate())
1005     }
1006 }
1007
1008 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1009 pub struct Rls {
1010     pub compiler: Compiler,
1011     pub target: TargetSelection,
1012 }
1013
1014 impl Step for Rls {
1015     type Output = Option<GeneratedTarball>;
1016     const ONLY_HOSTS: bool = true;
1017
1018     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1019         let default = should_build_extended_tool(&run.builder, "rls");
1020         run.path("rls").default_condition(default)
1021     }
1022
1023     fn make_run(run: RunConfig<'_>) {
1024         run.builder.ensure(Rls {
1025             compiler: run.builder.compiler_for(
1026                 run.builder.top_stage,
1027                 run.builder.config.build,
1028                 run.target,
1029             ),
1030             target: run.target,
1031         });
1032     }
1033
1034     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1035         let compiler = self.compiler;
1036         let target = self.target;
1037
1038         let rls = builder
1039             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1040             .or_else(|| {
1041                 missing_tool("RLS", builder.build.config.missing_tools);
1042                 None
1043             })?;
1044
1045         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1046         tarball.set_overlay(OverlayKind::RLS);
1047         tarball.is_preview(true);
1048         tarball.add_file(rls, "bin", 0o755);
1049         tarball.add_legal_and_readme_to("share/doc/rls");
1050         Some(tarball.generate())
1051     }
1052 }
1053
1054 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1055 pub struct RustAnalyzer {
1056     pub compiler: Compiler,
1057     pub target: TargetSelection,
1058 }
1059
1060 impl Step for RustAnalyzer {
1061     type Output = Option<GeneratedTarball>;
1062     const ONLY_HOSTS: bool = true;
1063
1064     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1065         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1066         run.path("rust-analyzer").default_condition(default)
1067     }
1068
1069     fn make_run(run: RunConfig<'_>) {
1070         run.builder.ensure(RustAnalyzer {
1071             compiler: run.builder.compiler_for(
1072                 run.builder.top_stage,
1073                 run.builder.config.build,
1074                 run.target,
1075             ),
1076             target: run.target,
1077         });
1078     }
1079
1080     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1081         // This prevents rust-analyzer from being built for "dist" or "install"
1082         // on the stable/beta channels. It is a nightly-only tool and should
1083         // not be included.
1084         if !builder.build.unstable_features() {
1085             return None;
1086         }
1087         let compiler = self.compiler;
1088         let target = self.target;
1089
1090         if target.contains("riscv64") {
1091             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1092             // to build. See #74813 for details.
1093             return None;
1094         }
1095
1096         let rust_analyzer = builder
1097             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1098             .expect("rust-analyzer always builds");
1099
1100         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1101         tarball.set_overlay(OverlayKind::RustAnalyzer);
1102         tarball.is_preview(true);
1103         tarball.add_file(rust_analyzer, "bin", 0o755);
1104         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1105         Some(tarball.generate())
1106     }
1107 }
1108
1109 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1110 pub struct Clippy {
1111     pub compiler: Compiler,
1112     pub target: TargetSelection,
1113 }
1114
1115 impl Step for Clippy {
1116     type Output = Option<GeneratedTarball>;
1117     const ONLY_HOSTS: bool = true;
1118
1119     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1120         let default = should_build_extended_tool(&run.builder, "clippy");
1121         run.path("clippy").default_condition(default)
1122     }
1123
1124     fn make_run(run: RunConfig<'_>) {
1125         run.builder.ensure(Clippy {
1126             compiler: run.builder.compiler_for(
1127                 run.builder.top_stage,
1128                 run.builder.config.build,
1129                 run.target,
1130             ),
1131             target: run.target,
1132         });
1133     }
1134
1135     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1136         let compiler = self.compiler;
1137         let target = self.target;
1138
1139         // Prepare the image directory
1140         // We expect clippy to build, because we've exited this step above if tool
1141         // state for clippy isn't testing.
1142         let clippy = builder
1143             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1144             .expect("clippy expected to build - essential tool");
1145         let cargoclippy = builder
1146             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1147             .expect("clippy expected to build - essential tool");
1148
1149         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1150         tarball.set_overlay(OverlayKind::Clippy);
1151         tarball.is_preview(true);
1152         tarball.add_file(clippy, "bin", 0o755);
1153         tarball.add_file(cargoclippy, "bin", 0o755);
1154         tarball.add_legal_and_readme_to("share/doc/clippy");
1155         Some(tarball.generate())
1156     }
1157 }
1158
1159 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1160 pub struct Miri {
1161     pub compiler: Compiler,
1162     pub target: TargetSelection,
1163 }
1164
1165 impl Step for Miri {
1166     type Output = Option<GeneratedTarball>;
1167     const ONLY_HOSTS: bool = true;
1168
1169     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1170         let default = should_build_extended_tool(&run.builder, "miri");
1171         run.path("miri").default_condition(default)
1172     }
1173
1174     fn make_run(run: RunConfig<'_>) {
1175         run.builder.ensure(Miri {
1176             compiler: run.builder.compiler_for(
1177                 run.builder.top_stage,
1178                 run.builder.config.build,
1179                 run.target,
1180             ),
1181             target: run.target,
1182         });
1183     }
1184
1185     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1186         // This prevents miri from being built for "dist" or "install"
1187         // on the stable/beta channels. It is a nightly-only tool and should
1188         // not be included.
1189         if !builder.build.unstable_features() {
1190             return None;
1191         }
1192         let compiler = self.compiler;
1193         let target = self.target;
1194
1195         let miri = builder
1196             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1197             .or_else(|| {
1198                 missing_tool("miri", builder.build.config.missing_tools);
1199                 None
1200             })?;
1201         let cargomiri = builder
1202             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1203             .or_else(|| {
1204                 missing_tool("cargo miri", builder.build.config.missing_tools);
1205                 None
1206             })?;
1207
1208         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1209         tarball.set_overlay(OverlayKind::Miri);
1210         tarball.is_preview(true);
1211         tarball.add_file(miri, "bin", 0o755);
1212         tarball.add_file(cargomiri, "bin", 0o755);
1213         tarball.add_legal_and_readme_to("share/doc/miri");
1214         Some(tarball.generate())
1215     }
1216 }
1217
1218 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1219 pub struct Rustfmt {
1220     pub compiler: Compiler,
1221     pub target: TargetSelection,
1222 }
1223
1224 impl Step for Rustfmt {
1225     type Output = Option<GeneratedTarball>;
1226     const ONLY_HOSTS: bool = true;
1227
1228     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1229         let default = should_build_extended_tool(&run.builder, "rustfmt");
1230         run.path("rustfmt").default_condition(default)
1231     }
1232
1233     fn make_run(run: RunConfig<'_>) {
1234         run.builder.ensure(Rustfmt {
1235             compiler: run.builder.compiler_for(
1236                 run.builder.top_stage,
1237                 run.builder.config.build,
1238                 run.target,
1239             ),
1240             target: run.target,
1241         });
1242     }
1243
1244     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1245         let compiler = self.compiler;
1246         let target = self.target;
1247
1248         let rustfmt = builder
1249             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1250             .or_else(|| {
1251                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1252                 None
1253             })?;
1254         let cargofmt = builder
1255             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1256             .or_else(|| {
1257                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1258                 None
1259             })?;
1260
1261         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1262         tarball.set_overlay(OverlayKind::Rustfmt);
1263         tarball.is_preview(true);
1264         tarball.add_file(rustfmt, "bin", 0o755);
1265         tarball.add_file(cargofmt, "bin", 0o755);
1266         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1267         Some(tarball.generate())
1268     }
1269 }
1270
1271 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1272 pub struct RustDemangler {
1273     pub compiler: Compiler,
1274     pub target: TargetSelection,
1275 }
1276
1277 impl Step for RustDemangler {
1278     type Output = Option<GeneratedTarball>;
1279     const ONLY_HOSTS: bool = true;
1280
1281     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1282         // While other tools use `should_build_extended_tool` to decide whether to be run by
1283         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1284         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1285         // we run the step by default when only `extended = true`, and decide whether to actually
1286         // run it or not later.
1287         let default = run.builder.config.extended;
1288         run.path("rust-demangler").default_condition(default)
1289     }
1290
1291     fn make_run(run: RunConfig<'_>) {
1292         run.builder.ensure(RustDemangler {
1293             compiler: run.builder.compiler_for(
1294                 run.builder.top_stage,
1295                 run.builder.config.build,
1296                 run.target,
1297             ),
1298             target: run.target,
1299         });
1300     }
1301
1302     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1303         let compiler = self.compiler;
1304         let target = self.target;
1305
1306         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1307         let condition = should_build_extended_tool(builder, "rust-demangler")
1308             || builder.config.profiler_enabled(target);
1309         if builder.config.extended && !condition {
1310             return None;
1311         }
1312
1313         let rust_demangler = builder
1314             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1315             .expect("rust-demangler expected to build - in-tree tool");
1316
1317         // Prepare the image directory
1318         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1319         tarball.set_overlay(OverlayKind::RustDemangler);
1320         tarball.is_preview(true);
1321         tarball.add_file(&rust_demangler, "bin", 0o755);
1322         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1323         Some(tarball.generate())
1324     }
1325 }
1326
1327 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1328 pub struct Extended {
1329     stage: u32,
1330     host: TargetSelection,
1331     target: TargetSelection,
1332 }
1333
1334 impl Step for Extended {
1335     type Output = ();
1336     const DEFAULT: bool = true;
1337     const ONLY_HOSTS: bool = true;
1338
1339     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1340         let builder = run.builder;
1341         run.path("extended").default_condition(builder.config.extended)
1342     }
1343
1344     fn make_run(run: RunConfig<'_>) {
1345         run.builder.ensure(Extended {
1346             stage: run.builder.top_stage,
1347             host: run.builder.config.build,
1348             target: run.target,
1349         });
1350     }
1351
1352     /// Creates a combined installer for the specified target in the provided stage.
1353     fn run(self, builder: &Builder<'_>) {
1354         let target = self.target;
1355         let stage = self.stage;
1356         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1357
1358         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1359
1360         let mut tarballs = Vec::new();
1361         let mut built_tools = HashSet::new();
1362         macro_rules! add_tool {
1363             ($name:expr => $step:expr) => {
1364                 if should_build_extended_tool(builder, $name) {
1365                     if let Some(tarball) = builder.ensure($step) {
1366                         tarballs.push(tarball);
1367                         built_tools.insert($name);
1368                     }
1369                 }
1370             };
1371         }
1372
1373         // When rust-std package split from rustc, we needed to ensure that during
1374         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1375         // the std files during uninstall. To do this ensure that rustc comes
1376         // before rust-std in the list below.
1377         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1378         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1379
1380         if builder.config.docs {
1381             tarballs.push(builder.ensure(Docs { host: target }));
1382         }
1383
1384         if target.contains("windows-gnu") {
1385             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1386         }
1387
1388         if builder.config.profiler_enabled(target)
1389             || should_build_extended_tool(builder, "rust-demangler")
1390         {
1391             if let Some(tarball) = builder.ensure(RustDemangler { compiler, target }) {
1392                 tarballs.push(tarball);
1393                 built_tools.insert("rust-demangler");
1394             }
1395         }
1396
1397         add_tool!("cargo" => Cargo { compiler, target });
1398         add_tool!("rustfmt" => Rustfmt { compiler, target });
1399         add_tool!("rls" => Rls { compiler, target });
1400         add_tool!("rust-analyzer" => RustAnalyzer { compiler, target });
1401         add_tool!("llvm-tools" => LlvmTools { target });
1402         add_tool!("clippy" => Clippy { compiler, target });
1403         add_tool!("miri" => Miri { compiler, target });
1404         add_tool!("analysis" => Analysis { compiler, target });
1405
1406         let etc = builder.src.join("src/etc/installer");
1407
1408         // Avoid producing tarballs during a dry run.
1409         if builder.config.dry_run {
1410             return;
1411         }
1412
1413         let tarball = Tarball::new(builder, "rust", &target.triple);
1414         let generated = tarball.combine(&tarballs);
1415
1416         let tmp = tmpdir(builder).join("combined-tarball");
1417         let work = generated.work_dir();
1418
1419         let mut license = String::new();
1420         license += &builder.read(&builder.src.join("COPYRIGHT"));
1421         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1422         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1423         license.push('\n');
1424         license.push('\n');
1425
1426         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1427         let mut rtf = rtf.to_string();
1428         rtf.push('\n');
1429         for line in license.lines() {
1430             rtf.push_str(line);
1431             rtf.push_str("\\line ");
1432         }
1433         rtf.push('}');
1434
1435         fn filter(contents: &str, marker: &str) -> String {
1436             let start = format!("tool-{}-start", marker);
1437             let end = format!("tool-{}-end", marker);
1438             let mut lines = Vec::new();
1439             let mut omitted = false;
1440             for line in contents.lines() {
1441                 if line.contains(&start) {
1442                     omitted = true;
1443                 } else if line.contains(&end) {
1444                     omitted = false;
1445                 } else if !omitted {
1446                     lines.push(line);
1447                 }
1448             }
1449
1450             lines.join("\n")
1451         }
1452
1453         let xform = |p: &Path| {
1454             let mut contents = t!(fs::read_to_string(p));
1455             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1456                 if !built_tools.contains(tool) {
1457                     contents = filter(&contents, tool);
1458                 }
1459             }
1460             let ret = tmp.join(p.file_name().unwrap());
1461             t!(fs::write(&ret, &contents));
1462             ret
1463         };
1464
1465         if target.contains("apple-darwin") {
1466             builder.info("building pkg installer");
1467             let pkg = tmp.join("pkg");
1468             let _ = fs::remove_dir_all(&pkg);
1469
1470             let pkgbuild = |component: &str| {
1471                 let mut cmd = Command::new("pkgbuild");
1472                 cmd.arg("--identifier")
1473                     .arg(format!("org.rust-lang.{}", component))
1474                     .arg("--scripts")
1475                     .arg(pkg.join(component))
1476                     .arg("--nopayload")
1477                     .arg(pkg.join(component).with_extension("pkg"));
1478                 builder.run(&mut cmd);
1479             };
1480
1481             let prepare = |name: &str| {
1482                 builder.create_dir(&pkg.join(name));
1483                 builder.cp_r(
1484                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1485                     &pkg.join(name),
1486                 );
1487                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1488                 pkgbuild(name);
1489             };
1490             prepare("rustc");
1491             prepare("cargo");
1492             prepare("rust-docs");
1493             prepare("rust-std");
1494             prepare("rust-analysis");
1495             prepare("clippy");
1496             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1497                 if built_tools.contains(tool) {
1498                     prepare(tool);
1499                 }
1500             }
1501             // create an 'uninstall' package
1502             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1503             pkgbuild("uninstall");
1504
1505             builder.create_dir(&pkg.join("res"));
1506             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1507             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1508             let mut cmd = Command::new("productbuild");
1509             cmd.arg("--distribution")
1510                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1511                 .arg("--resources")
1512                 .arg(pkg.join("res"))
1513                 .arg(distdir(builder).join(format!(
1514                     "{}-{}.pkg",
1515                     pkgname(builder, "rust"),
1516                     target.triple
1517                 )))
1518                 .arg("--package-path")
1519                 .arg(&pkg);
1520             let _time = timeit(builder);
1521             builder.run(&mut cmd);
1522         }
1523
1524         if target.contains("windows") {
1525             let exe = tmp.join("exe");
1526             let _ = fs::remove_dir_all(&exe);
1527
1528             let prepare = |name: &str| {
1529                 builder.create_dir(&exe.join(name));
1530                 let dir = if name == "rust-std" || name == "rust-analysis" {
1531                     format!("{}-{}", name, target.triple)
1532                 } else if name == "rls" {
1533                     "rls-preview".to_string()
1534                 } else if name == "rust-analyzer" {
1535                     "rust-analyzer-preview".to_string()
1536                 } else if name == "clippy" {
1537                     "clippy-preview".to_string()
1538                 } else if name == "rust-demangler" {
1539                     "rust-demangler-preview".to_string()
1540                 } else if name == "miri" {
1541                     "miri-preview".to_string()
1542                 } else {
1543                     name.to_string()
1544                 };
1545                 builder.cp_r(
1546                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1547                     &exe.join(name),
1548                 );
1549                 builder.remove(&exe.join(name).join("manifest.in"));
1550             };
1551             prepare("rustc");
1552             prepare("cargo");
1553             prepare("rust-analysis");
1554             prepare("rust-docs");
1555             prepare("rust-std");
1556             prepare("clippy");
1557             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1558                 if built_tools.contains(tool) {
1559                     prepare(tool);
1560                 }
1561             }
1562             if target.contains("windows-gnu") {
1563                 prepare("rust-mingw");
1564             }
1565
1566             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1567
1568             // Generate msi installer
1569             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1570             let heat = wix.join("bin/heat.exe");
1571             let candle = wix.join("bin/candle.exe");
1572             let light = wix.join("bin/light.exe");
1573
1574             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1575             builder.run(
1576                 Command::new(&heat)
1577                     .current_dir(&exe)
1578                     .arg("dir")
1579                     .arg("rustc")
1580                     .args(&heat_flags)
1581                     .arg("-cg")
1582                     .arg("RustcGroup")
1583                     .arg("-dr")
1584                     .arg("Rustc")
1585                     .arg("-var")
1586                     .arg("var.RustcDir")
1587                     .arg("-out")
1588                     .arg(exe.join("RustcGroup.wxs")),
1589             );
1590             builder.run(
1591                 Command::new(&heat)
1592                     .current_dir(&exe)
1593                     .arg("dir")
1594                     .arg("rust-docs")
1595                     .args(&heat_flags)
1596                     .arg("-cg")
1597                     .arg("DocsGroup")
1598                     .arg("-dr")
1599                     .arg("Docs")
1600                     .arg("-var")
1601                     .arg("var.DocsDir")
1602                     .arg("-out")
1603                     .arg(exe.join("DocsGroup.wxs"))
1604                     .arg("-t")
1605                     .arg(etc.join("msi/squash-components.xsl")),
1606             );
1607             builder.run(
1608                 Command::new(&heat)
1609                     .current_dir(&exe)
1610                     .arg("dir")
1611                     .arg("cargo")
1612                     .args(&heat_flags)
1613                     .arg("-cg")
1614                     .arg("CargoGroup")
1615                     .arg("-dr")
1616                     .arg("Cargo")
1617                     .arg("-var")
1618                     .arg("var.CargoDir")
1619                     .arg("-out")
1620                     .arg(exe.join("CargoGroup.wxs"))
1621                     .arg("-t")
1622                     .arg(etc.join("msi/remove-duplicates.xsl")),
1623             );
1624             builder.run(
1625                 Command::new(&heat)
1626                     .current_dir(&exe)
1627                     .arg("dir")
1628                     .arg("rust-std")
1629                     .args(&heat_flags)
1630                     .arg("-cg")
1631                     .arg("StdGroup")
1632                     .arg("-dr")
1633                     .arg("Std")
1634                     .arg("-var")
1635                     .arg("var.StdDir")
1636                     .arg("-out")
1637                     .arg(exe.join("StdGroup.wxs")),
1638             );
1639             if built_tools.contains("rls") {
1640                 builder.run(
1641                     Command::new(&heat)
1642                         .current_dir(&exe)
1643                         .arg("dir")
1644                         .arg("rls")
1645                         .args(&heat_flags)
1646                         .arg("-cg")
1647                         .arg("RlsGroup")
1648                         .arg("-dr")
1649                         .arg("Rls")
1650                         .arg("-var")
1651                         .arg("var.RlsDir")
1652                         .arg("-out")
1653                         .arg(exe.join("RlsGroup.wxs"))
1654                         .arg("-t")
1655                         .arg(etc.join("msi/remove-duplicates.xsl")),
1656                 );
1657             }
1658             if built_tools.contains("rust-analyzer") {
1659                 builder.run(
1660                     Command::new(&heat)
1661                         .current_dir(&exe)
1662                         .arg("dir")
1663                         .arg("rust-analyzer")
1664                         .args(&heat_flags)
1665                         .arg("-cg")
1666                         .arg("RustAnalyzerGroup")
1667                         .arg("-dr")
1668                         .arg("RustAnalyzer")
1669                         .arg("-var")
1670                         .arg("var.RustAnalyzerDir")
1671                         .arg("-out")
1672                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1673                         .arg("-t")
1674                         .arg(etc.join("msi/remove-duplicates.xsl")),
1675                 );
1676             }
1677             builder.run(
1678                 Command::new(&heat)
1679                     .current_dir(&exe)
1680                     .arg("dir")
1681                     .arg("clippy")
1682                     .args(&heat_flags)
1683                     .arg("-cg")
1684                     .arg("ClippyGroup")
1685                     .arg("-dr")
1686                     .arg("Clippy")
1687                     .arg("-var")
1688                     .arg("var.ClippyDir")
1689                     .arg("-out")
1690                     .arg(exe.join("ClippyGroup.wxs"))
1691                     .arg("-t")
1692                     .arg(etc.join("msi/remove-duplicates.xsl")),
1693             );
1694             if built_tools.contains("rust-demangler") {
1695                 builder.run(
1696                     Command::new(&heat)
1697                         .current_dir(&exe)
1698                         .arg("dir")
1699                         .arg("rust-demangler")
1700                         .args(&heat_flags)
1701                         .arg("-cg")
1702                         .arg("RustDemanglerGroup")
1703                         .arg("-dr")
1704                         .arg("RustDemangler")
1705                         .arg("-var")
1706                         .arg("var.RustDemanglerDir")
1707                         .arg("-out")
1708                         .arg(exe.join("RustDemanglerGroup.wxs"))
1709                         .arg("-t")
1710                         .arg(etc.join("msi/remove-duplicates.xsl")),
1711                 );
1712             }
1713             if built_tools.contains("miri") {
1714                 builder.run(
1715                     Command::new(&heat)
1716                         .current_dir(&exe)
1717                         .arg("dir")
1718                         .arg("miri")
1719                         .args(&heat_flags)
1720                         .arg("-cg")
1721                         .arg("MiriGroup")
1722                         .arg("-dr")
1723                         .arg("Miri")
1724                         .arg("-var")
1725                         .arg("var.MiriDir")
1726                         .arg("-out")
1727                         .arg(exe.join("MiriGroup.wxs"))
1728                         .arg("-t")
1729                         .arg(etc.join("msi/remove-duplicates.xsl")),
1730                 );
1731             }
1732             builder.run(
1733                 Command::new(&heat)
1734                     .current_dir(&exe)
1735                     .arg("dir")
1736                     .arg("rust-analysis")
1737                     .args(&heat_flags)
1738                     .arg("-cg")
1739                     .arg("AnalysisGroup")
1740                     .arg("-dr")
1741                     .arg("Analysis")
1742                     .arg("-var")
1743                     .arg("var.AnalysisDir")
1744                     .arg("-out")
1745                     .arg(exe.join("AnalysisGroup.wxs"))
1746                     .arg("-t")
1747                     .arg(etc.join("msi/remove-duplicates.xsl")),
1748             );
1749             if target.contains("windows-gnu") {
1750                 builder.run(
1751                     Command::new(&heat)
1752                         .current_dir(&exe)
1753                         .arg("dir")
1754                         .arg("rust-mingw")
1755                         .args(&heat_flags)
1756                         .arg("-cg")
1757                         .arg("GccGroup")
1758                         .arg("-dr")
1759                         .arg("Gcc")
1760                         .arg("-var")
1761                         .arg("var.GccDir")
1762                         .arg("-out")
1763                         .arg(exe.join("GccGroup.wxs")),
1764                 );
1765             }
1766
1767             let candle = |input: &Path| {
1768                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1769                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1770                 let mut cmd = Command::new(&candle);
1771                 cmd.current_dir(&exe)
1772                     .arg("-nologo")
1773                     .arg("-dRustcDir=rustc")
1774                     .arg("-dDocsDir=rust-docs")
1775                     .arg("-dCargoDir=cargo")
1776                     .arg("-dStdDir=rust-std")
1777                     .arg("-dAnalysisDir=rust-analysis")
1778                     .arg("-dClippyDir=clippy")
1779                     .arg("-arch")
1780                     .arg(&arch)
1781                     .arg("-out")
1782                     .arg(&output)
1783                     .arg(&input);
1784                 add_env(builder, &mut cmd, target);
1785
1786                 if built_tools.contains("rust-demangler") {
1787                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1788                 }
1789                 if built_tools.contains("rls") {
1790                     cmd.arg("-dRlsDir=rls");
1791                 }
1792                 if built_tools.contains("rust-analyzer") {
1793                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1794                 }
1795                 if built_tools.contains("miri") {
1796                     cmd.arg("-dMiriDir=miri");
1797                 }
1798                 if target.contains("windows-gnu") {
1799                     cmd.arg("-dGccDir=rust-mingw");
1800                 }
1801                 builder.run(&mut cmd);
1802             };
1803             candle(&xform(&etc.join("msi/rust.wxs")));
1804             candle(&etc.join("msi/ui.wxs"));
1805             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1806             candle("RustcGroup.wxs".as_ref());
1807             candle("DocsGroup.wxs".as_ref());
1808             candle("CargoGroup.wxs".as_ref());
1809             candle("StdGroup.wxs".as_ref());
1810             candle("ClippyGroup.wxs".as_ref());
1811             if built_tools.contains("rust-demangler") {
1812                 candle("RustDemanglerGroup.wxs".as_ref());
1813             }
1814             if built_tools.contains("rls") {
1815                 candle("RlsGroup.wxs".as_ref());
1816             }
1817             if built_tools.contains("rust-analyzer") {
1818                 candle("RustAnalyzerGroup.wxs".as_ref());
1819             }
1820             if built_tools.contains("miri") {
1821                 candle("MiriGroup.wxs".as_ref());
1822             }
1823             candle("AnalysisGroup.wxs".as_ref());
1824
1825             if target.contains("windows-gnu") {
1826                 candle("GccGroup.wxs".as_ref());
1827             }
1828
1829             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1830             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1831             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1832
1833             builder.info(&format!("building `msi` installer with {:?}", light));
1834             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1835             let mut cmd = Command::new(&light);
1836             cmd.arg("-nologo")
1837                 .arg("-ext")
1838                 .arg("WixUIExtension")
1839                 .arg("-ext")
1840                 .arg("WixUtilExtension")
1841                 .arg("-out")
1842                 .arg(exe.join(&filename))
1843                 .arg("rust.wixobj")
1844                 .arg("ui.wixobj")
1845                 .arg("rustwelcomedlg.wixobj")
1846                 .arg("RustcGroup.wixobj")
1847                 .arg("DocsGroup.wixobj")
1848                 .arg("CargoGroup.wixobj")
1849                 .arg("StdGroup.wixobj")
1850                 .arg("AnalysisGroup.wixobj")
1851                 .arg("ClippyGroup.wixobj")
1852                 .current_dir(&exe);
1853
1854             if built_tools.contains("rls") {
1855                 cmd.arg("RlsGroup.wixobj");
1856             }
1857             if built_tools.contains("rust-analyzer") {
1858                 cmd.arg("RustAnalyzerGroup.wixobj");
1859             }
1860             if built_tools.contains("rust-demangler") {
1861                 cmd.arg("RustDemanglerGroup.wixobj");
1862             }
1863             if built_tools.contains("miri") {
1864                 cmd.arg("MiriGroup.wixobj");
1865             }
1866
1867             if target.contains("windows-gnu") {
1868                 cmd.arg("GccGroup.wixobj");
1869             }
1870             // ICE57 wrongly complains about the shortcuts
1871             cmd.arg("-sice:ICE57");
1872
1873             let _time = timeit(builder);
1874             builder.run(&mut cmd);
1875
1876             if !builder.config.dry_run {
1877                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1878             }
1879         }
1880     }
1881 }
1882
1883 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1884     let mut parts = builder.version.split('.');
1885     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1886         .env("CFG_RELEASE_NUM", &builder.version)
1887         .env("CFG_RELEASE", builder.rust_release())
1888         .env("CFG_VER_MAJOR", parts.next().unwrap())
1889         .env("CFG_VER_MINOR", parts.next().unwrap())
1890         .env("CFG_VER_PATCH", parts.next().unwrap())
1891         .env("CFG_VER_BUILD", "0") // just needed to build
1892         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1893         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1894         .env("CFG_BUILD", target.triple)
1895         .env("CFG_CHANNEL", &builder.config.channel);
1896
1897     if target.contains("windows-gnu") {
1898         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1899     } else {
1900         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1901     }
1902
1903     if target.contains("x86_64") {
1904         cmd.env("CFG_PLATFORM", "x64");
1905     } else {
1906         cmd.env("CFG_PLATFORM", "x86");
1907     }
1908 }
1909
1910 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1911 ///
1912
1913 /// Returns whether the files were actually copied.
1914 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1915     if let Some(config) = builder.config.target_config.get(&target) {
1916         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1917             // If the LLVM was externally provided, then we don't currently copy
1918             // artifacts into the sysroot. This is not necessarily the right
1919             // choice (in particular, it will require the LLVM dylib to be in
1920             // the linker's load path at runtime), but the common use case for
1921             // external LLVMs is distribution provided LLVMs, and in that case
1922             // they're usually in the standard search path (e.g., /usr/lib) and
1923             // copying them here is going to cause problems as we may end up
1924             // with the wrong files and isn't what distributions want.
1925             //
1926             // This behavior may be revisited in the future though.
1927             //
1928             // If the LLVM is coming from ourselves (just from CI) though, we
1929             // still want to install it, as it otherwise won't be available.
1930             return false;
1931         }
1932     }
1933
1934     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1935     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1936     // clear why this is the case, though. llvm-config will emit the versioned
1937     // paths and we don't want those in the sysroot (as we're expecting
1938     // unversioned paths).
1939     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1940         let src_libdir = builder.llvm_out(target).join("lib");
1941         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1942         if llvm_dylib_path.exists() {
1943             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1944         }
1945         !builder.config.dry_run
1946     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1947         let mut cmd = Command::new(llvm_config);
1948         cmd.arg("--libfiles");
1949         builder.verbose(&format!("running {:?}", cmd));
1950         let files = output(&mut cmd);
1951         for file in files.trim_end().split(' ') {
1952             builder.install(Path::new(file), dst_libdir, 0o644);
1953         }
1954         !builder.config.dry_run
1955     } else {
1956         false
1957     }
1958 }
1959
1960 /// Maybe add libLLVM.so to the target lib-dir for linking.
1961 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1962     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1963     // We do not need to copy LLVM files into the sysroot if it is not
1964     // dynamically linked; it is already included into librustc_llvm
1965     // statically.
1966     if builder.config.llvm_link_shared {
1967         maybe_install_llvm(builder, target, &dst_libdir);
1968     }
1969 }
1970
1971 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1972 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1973     let dst_libdir =
1974         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1975     // We do not need to copy LLVM files into the sysroot if it is not
1976     // dynamically linked; it is already included into librustc_llvm
1977     // statically.
1978     if builder.config.llvm_link_shared {
1979         maybe_install_llvm(builder, target, &dst_libdir);
1980     }
1981 }
1982
1983 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1984 pub struct LlvmTools {
1985     pub target: TargetSelection,
1986 }
1987
1988 impl Step for LlvmTools {
1989     type Output = Option<GeneratedTarball>;
1990     const ONLY_HOSTS: bool = true;
1991
1992     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1993         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1994         run.path("llvm-tools").default_condition(default)
1995     }
1996
1997     fn make_run(run: RunConfig<'_>) {
1998         run.builder.ensure(LlvmTools { target: run.target });
1999     }
2000
2001     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2002         let target = self.target;
2003
2004         /* run only if llvm-config isn't used */
2005         if let Some(config) = builder.config.target_config.get(&target) {
2006             if let Some(ref _s) = config.llvm_config {
2007                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2008                 return None;
2009             }
2010         }
2011
2012         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2013         tarball.set_overlay(OverlayKind::LLVM);
2014         tarball.is_preview(true);
2015
2016         // Prepare the image directory
2017         let src_bindir = builder.llvm_out(target).join("bin");
2018         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2019         for tool in LLVM_TOOLS {
2020             let exe = src_bindir.join(exe(tool, target));
2021             tarball.add_file(&exe, &dst_bindir, 0o755);
2022         }
2023
2024         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2025         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2026         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2027         // compiler libraries.
2028         maybe_install_llvm_target(builder, target, tarball.image_dir());
2029
2030         Some(tarball.generate())
2031     }
2032 }
2033
2034 // Tarball intended for internal consumption to ease rustc/std development.
2035 //
2036 // Should not be considered stable by end users.
2037 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2038 pub struct RustDev {
2039     pub target: TargetSelection,
2040 }
2041
2042 impl Step for RustDev {
2043     type Output = Option<GeneratedTarball>;
2044     const DEFAULT: bool = true;
2045     const ONLY_HOSTS: bool = true;
2046
2047     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2048         run.path("rust-dev")
2049     }
2050
2051     fn make_run(run: RunConfig<'_>) {
2052         run.builder.ensure(RustDev { target: run.target });
2053     }
2054
2055     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2056         let target = self.target;
2057
2058         /* run only if llvm-config isn't used */
2059         if let Some(config) = builder.config.target_config.get(&target) {
2060             if let Some(ref _s) = config.llvm_config {
2061                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2062                 return None;
2063             }
2064         }
2065
2066         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2067         tarball.set_overlay(OverlayKind::LLVM);
2068
2069         let src_bindir = builder.llvm_out(target).join("bin");
2070         for bin in &[
2071             "llvm-config",
2072             "llvm-ar",
2073             "llvm-objdump",
2074             "llvm-profdata",
2075             "llvm-bcanalyzer",
2076             "llvm-cov",
2077             "llvm-dwp",
2078         ] {
2079             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2080         }
2081         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2082
2083         // Copy the include directory as well; needed mostly to build
2084         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2085         // just broadly useful to be able to link against the bundled LLVM.
2086         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2087
2088         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2089         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2090         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2091         // compiler libraries.
2092         let dst_libdir = tarball.image_dir().join("lib");
2093         maybe_install_llvm(builder, target, &dst_libdir);
2094         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2095         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2096
2097         Some(tarball.generate())
2098     }
2099 }
2100
2101 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2102 /// release process to avoid cloning the monorepo and building stuff.
2103 ///
2104 /// Should not be considered stable by end users.
2105 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2106 pub struct BuildManifest {
2107     pub target: TargetSelection,
2108 }
2109
2110 impl Step for BuildManifest {
2111     type Output = GeneratedTarball;
2112     const DEFAULT: bool = false;
2113     const ONLY_HOSTS: bool = true;
2114
2115     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2116         run.path("src/tools/build-manifest")
2117     }
2118
2119     fn make_run(run: RunConfig<'_>) {
2120         run.builder.ensure(BuildManifest { target: run.target });
2121     }
2122
2123     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2124         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2125
2126         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2127         tarball.add_file(&build_manifest, "bin", 0o755);
2128         tarball.generate()
2129     }
2130 }
2131
2132 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2133 ///
2134 /// Currently this is the PGO profile data.
2135 ///
2136 /// Should not be considered stable by end users.
2137 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2138 pub struct ReproducibleArtifacts {
2139     pub target: TargetSelection,
2140 }
2141
2142 impl Step for ReproducibleArtifacts {
2143     type Output = Option<GeneratedTarball>;
2144     const DEFAULT: bool = true;
2145     const ONLY_HOSTS: bool = true;
2146
2147     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2148         run.path("reproducible")
2149     }
2150
2151     fn make_run(run: RunConfig<'_>) {
2152         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2153     }
2154
2155     fn run(self, builder: &Builder<'_>) -> Self::Output {
2156         let path = builder.config.rust_profile_use.as_ref()?;
2157
2158         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2159         tarball.add_file(path, ".", 0o644);
2160         Some(tarball.generate())
2161     }
2162 }