src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::channel;
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, output, t, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26
  27 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  28     format!("{}-{}", component, builder.rust_package_vers())
  29 }
  30
  31 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  32     builder.out.join("dist")
  33 }
  34
  35 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  36     builder.out.join("tmp/dist")
  37 }
  38
  39 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  40     if !builder.config.extended {
  41         return false;
  42     }
  43     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  44 }
  45
  46 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  47 pub struct Docs {
  48     pub host: TargetSelection,
  49 }
  50
  51 impl Step for Docs {
  52     type Output = Option<GeneratedTarball>;
  53     const DEFAULT: bool = true;
  54
  55     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  56         let default = run.builder.config.docs;
  57         run.alias("rust-docs").default_condition(default)
  58     }
  59
  60     fn make_run(run: RunConfig<'_>) {
  61         run.builder.ensure(Docs { host: run.target });
  62     }
  63
  64     /// Builds the `rust-docs` installer component.
  65     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  66         let host = self.host;
  67         builder.default_doc(&[]);
  68
  69         let dest = "share/doc/rust/html";
  70
  71         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  72         tarball.set_product_name("Rust Documentation");
  73         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  74         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  75         Some(tarball.generate())
  76     }
  77 }
  78
  79 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  80 pub struct JsonDocs {
  81     pub host: TargetSelection,
  82 }
  83
  84 impl Step for JsonDocs {
  85     type Output = Option<GeneratedTarball>;
  86     const DEFAULT: bool = true;
  87
  88     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  89         let default = run.builder.config.docs;
  90         run.alias("rust-docs-json").default_condition(default)
  91     }
  92
  93     fn make_run(run: RunConfig<'_>) {
  94         run.builder.ensure(JsonDocs { host: run.target });
  95     }
  96
  97     /// Builds the `rust-docs-json` installer component.
  98     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  99         let host = self.host;
 100         builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
 101
 102         let dest = "share/doc/rust/json";
 103
 104         let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple);
 105         tarball.set_product_name("Rust Documentation In JSON Format");
 106         tarball.is_preview(true);
 107         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 108         Some(tarball.generate())
 109     }
 110 }
 111
 112 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 113 pub struct RustcDocs {
 114     pub host: TargetSelection,
 115 }
 116
 117 impl Step for RustcDocs {
 118     type Output = Option<GeneratedTarball>;
 119     const DEFAULT: bool = true;
 120
 121     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 122         let builder = run.builder;
 123         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 124     }
 125
 126     fn make_run(run: RunConfig<'_>) {
 127         run.builder.ensure(RustcDocs { host: run.target });
 128     }
 129
 130     /// Builds the `rustc-docs` installer component.
 131     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 132         let host = self.host;
 133         builder.default_doc(&[]);
 134
 135         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 136         tarball.set_product_name("Rustc Documentation");
 137         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 138         Some(tarball.generate())
 139     }
 140 }
 141
 142 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 143     let mut found = Vec::with_capacity(files.len());
 144
 145     for file in files {
 146         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 147
 148         if let Some(file_path) = file_path {
 149             found.push(file_path);
 150         } else {
 151             panic!("Could not find '{}' in {:?}", file, path);
 152         }
 153     }
 154
 155     found
 156 }
 157
 158 fn make_win_dist(
 159     rust_root: &Path,
 160     plat_root: &Path,
 161     target: TargetSelection,
 162     builder: &Builder<'_>,
 163 ) {
 164     //Ask gcc where it keeps its stuff
 165     let mut cmd = Command::new(builder.cc(target));
 166     cmd.arg("-print-search-dirs");
 167     let gcc_out = output(&mut cmd);
 168
 169     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 170     let mut lib_path = Vec::new();
 171
 172     for line in gcc_out.lines() {
 173         let idx = line.find(':').unwrap();
 174         let key = &line[..idx];
 175         let trim_chars: &[_] = &[' ', '='];
 176         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 177
 178         if key == "programs" {
 179             bin_path.extend(value);
 180         } else if key == "libraries" {
 181             lib_path.extend(value);
 182         }
 183     }
 184
 185     let compiler = if target == "i686-pc-windows-gnu" {
 186         "i686-w64-mingw32-gcc.exe"
 187     } else if target == "x86_64-pc-windows-gnu" {
 188         "x86_64-w64-mingw32-gcc.exe"
 189     } else {
 190         "gcc.exe"
 191     };
 192     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 193     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 194     if target.starts_with("i686-") {
 195         rustc_dlls.push("libgcc_s_dw2-1.dll");
 196     } else {
 197         rustc_dlls.push("libgcc_s_seh-1.dll");
 198     }
 199
 200     let target_libs = [
 201         //MinGW libs
 202         "libgcc.a",
 203         "libgcc_eh.a",
 204         "libgcc_s.a",
 205         "libm.a",
 206         "libmingw32.a",
 207         "libmingwex.a",
 208         "libstdc++.a",
 209         "libiconv.a",
 210         "libmoldname.a",
 211         "libpthread.a",
 212         //Windows import libs
 213         "libadvapi32.a",
 214         "libbcrypt.a",
 215         "libcomctl32.a",
 216         "libcomdlg32.a",
 217         "libcredui.a",
 218         "libcrypt32.a",
 219         "libdbghelp.a",
 220         "libgdi32.a",
 221         "libimagehlp.a",
 222         "libiphlpapi.a",
 223         "libkernel32.a",
 224         "libmsimg32.a",
 225         "libmsvcrt.a",
 226         "libodbc32.a",
 227         "libole32.a",
 228         "liboleaut32.a",
 229         "libopengl32.a",
 230         "libpsapi.a",
 231         "librpcrt4.a",
 232         "libsecur32.a",
 233         "libsetupapi.a",
 234         "libshell32.a",
 235         "libsynchronization.a",
 236         "libuser32.a",
 237         "libuserenv.a",
 238         "libuuid.a",
 239         "libwinhttp.a",
 240         "libwinmm.a",
 241         "libwinspool.a",
 242         "libws2_32.a",
 243         "libwsock32.a",
 244     ];
 245
 246     //Find mingw artifacts we want to bundle
 247     let target_tools = find_files(&target_tools, &bin_path);
 248     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 249     let target_libs = find_files(&target_libs, &lib_path);
 250
 251     // Copy runtime dlls next to rustc.exe
 252     let dist_bin_dir = rust_root.join("bin/");
 253     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 254     for src in rustc_dlls {
 255         builder.copy_to_folder(&src, &dist_bin_dir);
 256     }
 257
 258     //Copy platform tools to platform-specific bin directory
 259     let target_bin_dir = plat_root
 260         .join("lib")
 261         .join("rustlib")
 262         .join(target.triple)
 263         .join("bin")
 264         .join("self-contained");
 265     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 266     for src in target_tools {
 267         builder.copy_to_folder(&src, &target_bin_dir);
 268     }
 269
 270     // Warn windows-gnu users that the bundled GCC cannot compile C files
 271     builder.create(
 272         &target_bin_dir.join("GCC-WARNING.txt"),
 273         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 274          used as a linker. In order to be able to compile projects containing C code use \
 275          the GCC provided by MinGW or Cygwin.",
 276     );
 277
 278     //Copy platform libs to platform-specific lib directory
 279     let target_lib_dir = plat_root
 280         .join("lib")
 281         .join("rustlib")
 282         .join(target.triple)
 283         .join("lib")
 284         .join("self-contained");
 285     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 286     for src in target_libs {
 287         builder.copy_to_folder(&src, &target_lib_dir);
 288     }
 289 }
 290
 291 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 292 pub struct Mingw {
 293     pub host: TargetSelection,
 294 }
 295
 296 impl Step for Mingw {
 297     type Output = Option<GeneratedTarball>;
 298     const DEFAULT: bool = true;
 299
 300     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 301         run.alias("rust-mingw")
 302     }
 303
 304     fn make_run(run: RunConfig<'_>) {
 305         run.builder.ensure(Mingw { host: run.target });
 306     }
 307
 308     /// Builds the `rust-mingw` installer component.
 309     ///
 310     /// This contains all the bits and pieces to run the MinGW Windows targets
 311     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 312     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 313         let host = self.host;
 314         if !host.ends_with("pc-windows-gnu") {
 315             return None;
 316         }
 317
 318         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 319         tarball.set_product_name("Rust MinGW");
 320
 321         // The first argument is a "temporary directory" which is just
 322         // thrown away (this contains the runtime DLLs included in the rustc package
 323         // above) and the second argument is where to place all the MinGW components
 324         // (which is what we want).
 325         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 326
 327         Some(tarball.generate())
 328     }
 329 }
 330
 331 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 332 pub struct Rustc {
 333     pub compiler: Compiler,
 334 }
 335
 336 impl Step for Rustc {
 337     type Output = GeneratedTarball;
 338     const DEFAULT: bool = true;
 339     const ONLY_HOSTS: bool = true;
 340
 341     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 342         run.alias("rustc")
 343     }
 344
 345     fn make_run(run: RunConfig<'_>) {
 346         run.builder
 347             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 348     }
 349
 350     /// Creates the `rustc` installer component.
 351     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 352         let compiler = self.compiler;
 353         let host = self.compiler.host;
 354
 355         let tarball = Tarball::new(builder, "rustc", &host.triple);
 356
 357         // Prepare the rustc "image", what will actually end up getting installed
 358         prepare_image(builder, compiler, tarball.image_dir());
 359
 360         // On MinGW we've got a few runtime DLL dependencies that we need to
 361         // include. The first argument to this script is where to put these DLLs
 362         // (the image we're creating), and the second argument is a junk directory
 363         // to ignore all other MinGW stuff the script creates.
 364         //
 365         // On 32-bit MinGW we're always including a DLL which needs some extra
 366         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 367         // anything requiring us to distribute a license, but it's likely the
 368         // install will *also* include the rust-mingw package, which also needs
 369         // licenses, so to be safe we just include it here in all MinGW packages.
 370         if host.ends_with("pc-windows-gnu") {
 371             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 372             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 373         }
 374
 375         return tarball.generate();
 376
 377         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 378             let host = compiler.host;
 379             let src = builder.sysroot(compiler);
 380
 381             // Copy rustc/rustdoc binaries
 382             t!(fs::create_dir_all(image.join("bin")));
 383             builder.cp_r(&src.join("bin"), &image.join("bin"));
 384
 385             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 386
 387             let ra_proc_macro_srv = builder
 388                 .ensure(tool::RustAnalyzerProcMacroSrv {
 389                     compiler: builder.compiler_for(
 390                         compiler.stage,
 391                         builder.config.build,
 392                         compiler.host,
 393                     ),
 394                     target: compiler.host,
 395                 })
 396                 .expect("rust-analyzer-proc-macro-server always builds");
 397             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 398
 399             let libdir_relative = builder.libdir_relative(compiler);
 400
 401             // Copy runtime DLLs needed by the compiler
 402             if libdir_relative.to_str() != Some("bin") {
 403                 let libdir = builder.rustc_libdir(compiler);
 404                 for entry in builder.read_dir(&libdir) {
 405                     let name = entry.file_name();
 406                     if let Some(s) = name.to_str() {
 407                         if is_dylib(s) {
 408                             // Don't use custom libdir here because ^lib/ will be resolved again
 409                             // with installer
 410                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 411                         }
 412                     }
 413                 }
 414             }
 415
 416             // Copy over the codegen backends
 417             let backends_src = builder.sysroot_codegen_backends(compiler);
 418             let backends_rel = backends_src
 419                 .strip_prefix(&src)
 420                 .unwrap()
 421                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 422                 .unwrap();
 423             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 424             let backends_dst = image.join("lib").join(&backends_rel);
 425
 426             t!(fs::create_dir_all(&backends_dst));
 427             builder.cp_r(&backends_src, &backends_dst);
 428
 429             // Copy libLLVM.so to the lib dir as well, if needed. While not
 430             // technically needed by rustc itself it's needed by lots of other
 431             // components like the llvm tools and LLD. LLD is included below and
 432             // tools/LLDB come later, so let's just throw it in the rustc
 433             // component for now.
 434             maybe_install_llvm_runtime(builder, host, image);
 435
 436             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 437             t!(fs::create_dir_all(&dst_dir));
 438
 439             // Copy over lld if it's there
 440             if builder.config.lld_enabled {
 441                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 442                 let rust_lld = exe("rust-lld", compiler.host);
 443                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 444                 // for `-Z gcc-ld=lld`
 445                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 446                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 447                 t!(fs::create_dir(&gcc_lld_dst_dir));
 448                 for name in crate::LLD_FILE_NAMES {
 449                     let exe_name = exe(name, compiler.host);
 450                     builder
 451                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 452                 }
 453             }
 454
 455             // Man pages
 456             t!(fs::create_dir_all(image.join("share/man/man1")));
 457             let man_src = builder.src.join("src/doc/man");
 458             let man_dst = image.join("share/man/man1");
 459
 460             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 461             // to hardlink, and we don't want to edit the source templates
 462             for file_entry in builder.read_dir(&man_src) {
 463                 let page_src = file_entry.path();
 464                 let page_dst = man_dst.join(file_entry.file_name());
 465                 let src_text = t!(std::fs::read_to_string(&page_src));
 466                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 467                 t!(std::fs::write(&page_dst, &new_text));
 468                 t!(fs::copy(&page_src, &page_dst));
 469             }
 470
 471             // Debugger scripts
 472             builder
 473                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 474
 475             // Misc license info
 476             let cp = |file: &str| {
 477                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 478             };
 479             cp("COPYRIGHT");
 480             cp("LICENSE-APACHE");
 481             cp("LICENSE-MIT");
 482             cp("README.md");
 483         }
 484     }
 485 }
 486
 487 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 488 pub struct DebuggerScripts {
 489     pub sysroot: Interned<PathBuf>,
 490     pub host: TargetSelection,
 491 }
 492
 493 impl Step for DebuggerScripts {
 494     type Output = ();
 495
 496     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 497         run.never()
 498     }
 499
 500     /// Copies debugger scripts for `target` into the `sysroot` specified.
 501     fn run(self, builder: &Builder<'_>) {
 502         let host = self.host;
 503         let sysroot = self.sysroot;
 504         let dst = sysroot.join("lib/rustlib/etc");
 505         t!(fs::create_dir_all(&dst));
 506         let cp_debugger_script = |file: &str| {
 507             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 508         };
 509         if host.contains("windows-msvc") {
 510             // windbg debugger scripts
 511             builder.install(
 512                 &builder.src.join("src/etc/rust-windbg.cmd"),
 513                 &sysroot.join("bin"),
 514                 0o755,
 515             );
 516
 517             cp_debugger_script("natvis/intrinsic.natvis");
 518             cp_debugger_script("natvis/liballoc.natvis");
 519             cp_debugger_script("natvis/libcore.natvis");
 520             cp_debugger_script("natvis/libstd.natvis");
 521         } else {
 522             cp_debugger_script("rust_types.py");
 523
 524             // gdb debugger scripts
 525             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 526             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 527
 528             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 529             cp_debugger_script("gdb_lookup.py");
 530             cp_debugger_script("gdb_providers.py");
 531
 532             // lldb debugger scripts
 533             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 534
 535             cp_debugger_script("lldb_lookup.py");
 536             cp_debugger_script("lldb_providers.py");
 537             cp_debugger_script("lldb_commands")
 538         }
 539     }
 540 }
 541
 542 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 543     // The only true set of target libraries came from the build triple, so
 544     // let's reduce redundant work by only producing archives from that host.
 545     if compiler.host != builder.config.build {
 546         builder.info("\tskipping, not a build host");
 547         true
 548     } else {
 549         false
 550     }
 551 }
 552
 553 /// Copy stamped files into an image's `target/lib` directory.
 554 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 555     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 556     let self_contained_dst = dst.join("self-contained");
 557     t!(fs::create_dir_all(&dst));
 558     t!(fs::create_dir_all(&self_contained_dst));
 559     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 560         if dependency_type == DependencyType::TargetSelfContained {
 561             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 562         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 563             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 564         }
 565     }
 566 }
 567
 568 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 569 pub struct Std {
 570     pub compiler: Compiler,
 571     pub target: TargetSelection,
 572 }
 573
 574 impl Step for Std {
 575     type Output = Option<GeneratedTarball>;
 576     const DEFAULT: bool = true;
 577
 578     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 579         run.alias("rust-std")
 580     }
 581
 582     fn make_run(run: RunConfig<'_>) {
 583         run.builder.ensure(Std {
 584             compiler: run.builder.compiler_for(
 585                 run.builder.top_stage,
 586                 run.builder.config.build,
 587                 run.target,
 588             ),
 589             target: run.target,
 590         });
 591     }
 592
 593     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 594         let compiler = self.compiler;
 595         let target = self.target;
 596
 597         if skip_host_target_lib(builder, compiler) {
 598             return None;
 599         }
 600
 601         builder.ensure(compile::Std::new(compiler, target));
 602
 603         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 604         tarball.include_target_in_component_name(true);
 605
 606         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 607         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 608         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 609
 610         Some(tarball.generate())
 611     }
 612 }
 613
 614 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 615 pub struct RustcDev {
 616     pub compiler: Compiler,
 617     pub target: TargetSelection,
 618 }
 619
 620 impl Step for RustcDev {
 621     type Output = Option<GeneratedTarball>;
 622     const DEFAULT: bool = true;
 623     const ONLY_HOSTS: bool = true;
 624
 625     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 626         run.alias("rustc-dev")
 627     }
 628
 629     fn make_run(run: RunConfig<'_>) {
 630         run.builder.ensure(RustcDev {
 631             compiler: run.builder.compiler_for(
 632                 run.builder.top_stage,
 633                 run.builder.config.build,
 634                 run.target,
 635             ),
 636             target: run.target,
 637         });
 638     }
 639
 640     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 641         let compiler = self.compiler;
 642         let target = self.target;
 643         if skip_host_target_lib(builder, compiler) {
 644             return None;
 645         }
 646
 647         builder.ensure(compile::Rustc::new(compiler, target));
 648
 649         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 650
 651         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 652         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 653         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 654
 655         let src_files = &["Cargo.lock"];
 656         // This is the reduced set of paths which will become the rustc-dev component
 657         // (essentially the compiler crates and all of their path dependencies).
 658         copy_src_dirs(
 659             builder,
 660             &builder.src,
 661             &["compiler"],
 662             &[],
 663             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 664         );
 665         for file in src_files {
 666             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 667         }
 668
 669         Some(tarball.generate())
 670     }
 671 }
 672
 673 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 674 pub struct Analysis {
 675     pub compiler: Compiler,
 676     pub target: TargetSelection,
 677 }
 678
 679 impl Step for Analysis {
 680     type Output = Option<GeneratedTarball>;
 681     const DEFAULT: bool = true;
 682
 683     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 684         let default = should_build_extended_tool(&run.builder, "analysis");
 685         run.alias("rust-analysis").default_condition(default)
 686     }
 687
 688     fn make_run(run: RunConfig<'_>) {
 689         run.builder.ensure(Analysis {
 690             // Find the actual compiler (handling the full bootstrap option) which
 691             // produced the save-analysis data because that data isn't copied
 692             // through the sysroot uplifting.
 693             compiler: run.builder.compiler_for(
 694                 run.builder.top_stage,
 695                 run.builder.config.build,
 696                 run.target,
 697             ),
 698             target: run.target,
 699         });
 700     }
 701
 702     /// Creates a tarball of save-analysis metadata, if available.
 703     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 704         let compiler = self.compiler;
 705         let target = self.target;
 706         if compiler.host != builder.config.build {
 707             return None;
 708         }
 709
 710         builder.ensure(compile::Std::new(compiler, target));
 711         let src = builder
 712             .stage_out(compiler, Mode::Std)
 713             .join(target.triple)
 714             .join(builder.cargo_dir())
 715             .join("deps")
 716             .join("save-analysis");
 717
 718         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 719         tarball.include_target_in_component_name(true);
 720         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 721         Some(tarball.generate())
 722     }
 723 }
 724
 725 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 726 /// `dst_dir`.
 727 fn copy_src_dirs(
 728     builder: &Builder<'_>,
 729     base: &Path,
 730     src_dirs: &[&str],
 731     exclude_dirs: &[&str],
 732     dst_dir: &Path,
 733 ) {
 734     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 735         let spath = match path.to_str() {
 736             Some(path) => path,
 737             None => return false,
 738         };
 739         if spath.ends_with('~') || spath.ends_with(".pyc") {
 740             return false;
 741         }
 742
 743         const LLVM_PROJECTS: &[&str] = &[
 744             "llvm-project/clang",
 745             "llvm-project\\clang",
 746             "llvm-project/libunwind",
 747             "llvm-project\\libunwind",
 748             "llvm-project/lld",
 749             "llvm-project\\lld",
 750             "llvm-project/lldb",
 751             "llvm-project\\lldb",
 752             "llvm-project/llvm",
 753             "llvm-project\\llvm",
 754             "llvm-project/compiler-rt",
 755             "llvm-project\\compiler-rt",
 756             "llvm-project/cmake",
 757             "llvm-project\\cmake",
 758         ];
 759         if spath.contains("llvm-project")
 760             && !spath.ends_with("llvm-project")
 761             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 762         {
 763             return false;
 764         }
 765
 766         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 767         if LLVM_TEST.iter().any(|path| spath.contains(path))
 768             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 769         {
 770             return false;
 771         }
 772
 773         let full_path = Path::new(dir).join(path);
 774         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 775             return false;
 776         }
 777
 778         let excludes = [
 779             "CVS",
 780             "RCS",
 781             "SCCS",
 782             ".git",
 783             ".gitignore",
 784             ".gitmodules",
 785             ".gitattributes",
 786             ".cvsignore",
 787             ".svn",
 788             ".arch-ids",
 789             "{arch}",
 790             "=RELEASE-ID",
 791             "=meta-update",
 792             "=update",
 793             ".bzr",
 794             ".bzrignore",
 795             ".bzrtags",
 796             ".hg",
 797             ".hgignore",
 798             ".hgrags",
 799             "_darcs",
 800         ];
 801         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 802     }
 803
 804     // Copy the directories using our filter
 805     for item in src_dirs {
 806         let dst = &dst_dir.join(item);
 807         t!(fs::create_dir_all(dst));
 808         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 809     }
 810 }
 811
 812 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 813 pub struct Src;
 814
 815 impl Step for Src {
 816     /// The output path of the src installer tarball
 817     type Output = GeneratedTarball;
 818     const DEFAULT: bool = true;
 819     const ONLY_HOSTS: bool = true;
 820
 821     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 822         run.alias("rust-src")
 823     }
 824
 825     fn make_run(run: RunConfig<'_>) {
 826         run.builder.ensure(Src);
 827     }
 828
 829     /// Creates the `rust-src` installer component
 830     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 831         let tarball = Tarball::new_targetless(builder, "rust-src");
 832
 833         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 834         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 835         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 836         // and fix them...
 837         //
 838         // NOTE: if you update the paths here, you also should update the "virtual" path
 839         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 840         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 841
 842         let src_files = ["Cargo.lock"];
 843         // This is the reduced set of paths which will become the rust-src component
 844         // (essentially libstd and all of its path dependencies).
 845         copy_src_dirs(
 846             builder,
 847             &builder.src,
 848             &["library", "src/llvm-project/libunwind"],
 849             &[
 850                 // not needed and contains symlinks which rustup currently
 851                 // chokes on when unpacking.
 852                 "library/backtrace/crates",
 853                 // these are 30MB combined and aren't necessary for building
 854                 // the standard library.
 855                 "library/stdarch/Cargo.toml",
 856                 "library/stdarch/crates/stdarch-verify",
 857                 "library/stdarch/crates/intrinsic-test",
 858             ],
 859             &dst_src,
 860         );
 861         for file in src_files.iter() {
 862             builder.copy(&builder.src.join(file), &dst_src.join(file));
 863         }
 864
 865         tarball.generate()
 866     }
 867 }
 868
 869 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 870 pub struct PlainSourceTarball;
 871
 872 impl Step for PlainSourceTarball {
 873     /// Produces the location of the tarball generated
 874     type Output = GeneratedTarball;
 875     const DEFAULT: bool = true;
 876     const ONLY_HOSTS: bool = true;
 877
 878     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 879         let builder = run.builder;
 880         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 881     }
 882
 883     fn make_run(run: RunConfig<'_>) {
 884         run.builder.ensure(PlainSourceTarball);
 885     }
 886
 887     /// Creates the plain source tarball
 888     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 889         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 890         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 891         // It also contains symbolic links, unlike other any other dist tarball.
 892         // It's used for distros building rustc from source in a pre-vendored environment.
 893         let mut tarball = Tarball::new(builder, "rustc", "src");
 894         tarball.permit_symlinks(true);
 895         let plain_dst_src = tarball.image_dir();
 896
 897         // This is the set of root paths which will become part of the source package
 898         let src_files = [
 899             "COPYRIGHT",
 900             "LICENSE-APACHE",
 901             "LICENSE-MIT",
 902             "CONTRIBUTING.md",
 903             "README.md",
 904             "RELEASES.md",
 905             "configure",
 906             "x.py",
 907             "config.toml.example",
 908             "Cargo.toml",
 909             "Cargo.lock",
 910         ];
 911         let src_dirs = ["src", "compiler", "library"];
 912
 913         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 914
 915         // Copy the files normally
 916         for item in &src_files {
 917             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 918         }
 919
 920         // Create the version file
 921         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 922         if let Some(info) = builder.rust_info.info() {
 923             channel::write_commit_hash_file(&plain_dst_src, &info.sha);
 924             channel::write_commit_info_file(&plain_dst_src, info);
 925         }
 926
 927         // If we're building from git sources, we need to vendor a complete distribution.
 928         if builder.rust_info.is_managed_git_subrepository() {
 929             // Ensure we have the submodules checked out.
 930             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 931
 932             // Vendor all Cargo dependencies
 933             let mut cmd = Command::new(&builder.initial_cargo);
 934             cmd.arg("vendor")
 935                 .arg("--sync")
 936                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 937                 .arg("--sync")
 938                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 939                 .arg("--sync")
 940                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 941                 .current_dir(&plain_dst_src);
 942
 943             let config = if !builder.config.dry_run {
 944                 t!(String::from_utf8(t!(cmd.output()).stdout))
 945             } else {
 946                 String::new()
 947             };
 948
 949             let cargo_config_dir = plain_dst_src.join(".cargo");
 950             builder.create_dir(&cargo_config_dir);
 951             builder.create(&cargo_config_dir.join("config.toml"), &config);
 952         }
 953
 954         tarball.bare()
 955     }
 956 }
 957
 958 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 959 pub struct Cargo {
 960     pub compiler: Compiler,
 961     pub target: TargetSelection,
 962 }
 963
 964 impl Step for Cargo {
 965     type Output = Option<GeneratedTarball>;
 966     const DEFAULT: bool = true;
 967     const ONLY_HOSTS: bool = true;
 968
 969     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 970         let default = should_build_extended_tool(&run.builder, "cargo");
 971         run.alias("cargo").default_condition(default)
 972     }
 973
 974     fn make_run(run: RunConfig<'_>) {
 975         run.builder.ensure(Cargo {
 976             compiler: run.builder.compiler_for(
 977                 run.builder.top_stage,
 978                 run.builder.config.build,
 979                 run.target,
 980             ),
 981             target: run.target,
 982         });
 983     }
 984
 985     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 986         let compiler = self.compiler;
 987         let target = self.target;
 988
 989         let cargo = builder.ensure(tool::Cargo { compiler, target });
 990         let src = builder.src.join("src/tools/cargo");
 991         let etc = src.join("src/etc");
 992
 993         // Prepare the image directory
 994         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 995         tarball.set_overlay(OverlayKind::Cargo);
 996
 997         tarball.add_file(&cargo, "bin", 0o755);
 998         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 999         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1000         tarball.add_dir(etc.join("man"), "share/man/man1");
1001         tarball.add_legal_and_readme_to("share/doc/cargo");
1002
1003         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1004             let dirent = dirent.expect("read dir entry");
1005             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1006                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1007             }
1008         }
1009
1010         Some(tarball.generate())
1011     }
1012 }
1013
1014 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1015 pub struct Rls {
1016     pub compiler: Compiler,
1017     pub target: TargetSelection,
1018 }
1019
1020 impl Step for Rls {
1021     type Output = Option<GeneratedTarball>;
1022     const ONLY_HOSTS: bool = true;
1023     const DEFAULT: bool = true;
1024
1025     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1026         let default = should_build_extended_tool(&run.builder, "rls");
1027         run.alias("rls").default_condition(default)
1028     }
1029
1030     fn make_run(run: RunConfig<'_>) {
1031         run.builder.ensure(Rls {
1032             compiler: run.builder.compiler_for(
1033                 run.builder.top_stage,
1034                 run.builder.config.build,
1035                 run.target,
1036             ),
1037             target: run.target,
1038         });
1039     }
1040
1041     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1042         let compiler = self.compiler;
1043         let target = self.target;
1044
1045         let rls = builder
1046             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1047             .expect("rls expected to build");
1048
1049         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1050         tarball.set_overlay(OverlayKind::RLS);
1051         tarball.is_preview(true);
1052         tarball.add_file(rls, "bin", 0o755);
1053         tarball.add_legal_and_readme_to("share/doc/rls");
1054         Some(tarball.generate())
1055     }
1056 }
1057
1058 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1059 pub struct RustAnalyzer {
1060     pub compiler: Compiler,
1061     pub target: TargetSelection,
1062 }
1063
1064 impl Step for RustAnalyzer {
1065     type Output = Option<GeneratedTarball>;
1066     const DEFAULT: bool = true;
1067     const ONLY_HOSTS: bool = true;
1068
1069     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1070         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1071         run.alias("rust-analyzer").default_condition(default)
1072     }
1073
1074     fn make_run(run: RunConfig<'_>) {
1075         run.builder.ensure(RustAnalyzer {
1076             compiler: run.builder.compiler_for(
1077                 run.builder.top_stage,
1078                 run.builder.config.build,
1079                 run.target,
1080             ),
1081             target: run.target,
1082         });
1083     }
1084
1085     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1086         let compiler = self.compiler;
1087         let target = self.target;
1088
1089         if target.contains("riscv64") {
1090             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1091             // to build. See #74813 for details.
1092             return None;
1093         }
1094
1095         let rust_analyzer = builder
1096             .ensure(tool::RustAnalyzer { compiler, target })
1097             .expect("rust-analyzer always builds");
1098
1099         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1100         tarball.set_overlay(OverlayKind::RustAnalyzer);
1101         tarball.is_preview(true);
1102         tarball.add_file(rust_analyzer, "bin", 0o755);
1103         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1104         Some(tarball.generate())
1105     }
1106 }
1107
1108 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1109 pub struct Clippy {
1110     pub compiler: Compiler,
1111     pub target: TargetSelection,
1112 }
1113
1114 impl Step for Clippy {
1115     type Output = Option<GeneratedTarball>;
1116     const DEFAULT: bool = true;
1117     const ONLY_HOSTS: bool = true;
1118
1119     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1120         let default = should_build_extended_tool(&run.builder, "clippy");
1121         run.alias("clippy").default_condition(default)
1122     }
1123
1124     fn make_run(run: RunConfig<'_>) {
1125         run.builder.ensure(Clippy {
1126             compiler: run.builder.compiler_for(
1127                 run.builder.top_stage,
1128                 run.builder.config.build,
1129                 run.target,
1130             ),
1131             target: run.target,
1132         });
1133     }
1134
1135     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1136         let compiler = self.compiler;
1137         let target = self.target;
1138
1139         // Prepare the image directory
1140         // We expect clippy to build, because we've exited this step above if tool
1141         // state for clippy isn't testing.
1142         let clippy = builder
1143             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1144             .expect("clippy expected to build - essential tool");
1145         let cargoclippy = builder
1146             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1147             .expect("clippy expected to build - essential tool");
1148
1149         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1150         tarball.set_overlay(OverlayKind::Clippy);
1151         tarball.is_preview(true);
1152         tarball.add_file(clippy, "bin", 0o755);
1153         tarball.add_file(cargoclippy, "bin", 0o755);
1154         tarball.add_legal_and_readme_to("share/doc/clippy");
1155         Some(tarball.generate())
1156     }
1157 }
1158
1159 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1160 pub struct Miri {
1161     pub compiler: Compiler,
1162     pub target: TargetSelection,
1163 }
1164
1165 impl Step for Miri {
1166     type Output = Option<GeneratedTarball>;
1167     const DEFAULT: bool = true;
1168     const ONLY_HOSTS: bool = true;
1169
1170     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1171         let default = should_build_extended_tool(&run.builder, "miri");
1172         run.alias("miri").default_condition(default)
1173     }
1174
1175     fn make_run(run: RunConfig<'_>) {
1176         run.builder.ensure(Miri {
1177             compiler: run.builder.compiler_for(
1178                 run.builder.top_stage,
1179                 run.builder.config.build,
1180                 run.target,
1181             ),
1182             target: run.target,
1183         });
1184     }
1185
1186     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1187         // This prevents miri from being built for "dist" or "install"
1188         // on the stable/beta channels. It is a nightly-only tool and should
1189         // not be included.
1190         if !builder.build.unstable_features() {
1191             return None;
1192         }
1193         let compiler = self.compiler;
1194         let target = self.target;
1195
1196         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1197         let cargomiri =
1198             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1199
1200         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1201         tarball.set_overlay(OverlayKind::Miri);
1202         tarball.is_preview(true);
1203         tarball.add_file(miri, "bin", 0o755);
1204         tarball.add_file(cargomiri, "bin", 0o755);
1205         tarball.add_legal_and_readme_to("share/doc/miri");
1206         Some(tarball.generate())
1207     }
1208 }
1209
1210 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1211 pub struct Rustfmt {
1212     pub compiler: Compiler,
1213     pub target: TargetSelection,
1214 }
1215
1216 impl Step for Rustfmt {
1217     type Output = Option<GeneratedTarball>;
1218     const DEFAULT: bool = true;
1219     const ONLY_HOSTS: bool = true;
1220
1221     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1222         let default = should_build_extended_tool(&run.builder, "rustfmt");
1223         run.alias("rustfmt").default_condition(default)
1224     }
1225
1226     fn make_run(run: RunConfig<'_>) {
1227         run.builder.ensure(Rustfmt {
1228             compiler: run.builder.compiler_for(
1229                 run.builder.top_stage,
1230                 run.builder.config.build,
1231                 run.target,
1232             ),
1233             target: run.target,
1234         });
1235     }
1236
1237     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1238         let compiler = self.compiler;
1239         let target = self.target;
1240
1241         let rustfmt = builder
1242             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1243             .expect("rustfmt expected to build - essential tool");
1244         let cargofmt = builder
1245             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1246             .expect("cargo fmt expected to build - essential tool");
1247         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1248         tarball.set_overlay(OverlayKind::Rustfmt);
1249         tarball.is_preview(true);
1250         tarball.add_file(rustfmt, "bin", 0o755);
1251         tarball.add_file(cargofmt, "bin", 0o755);
1252         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1253         Some(tarball.generate())
1254     }
1255 }
1256
1257 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1258 pub struct RustDemangler {
1259     pub compiler: Compiler,
1260     pub target: TargetSelection,
1261 }
1262
1263 impl Step for RustDemangler {
1264     type Output = Option<GeneratedTarball>;
1265     const DEFAULT: bool = true;
1266     const ONLY_HOSTS: bool = true;
1267
1268     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1269         // While other tools use `should_build_extended_tool` to decide whether to be run by
1270         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1271         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1272         // we run the step by default when only `extended = true`, and decide whether to actually
1273         // run it or not later.
1274         let default = run.builder.config.extended;
1275         run.alias("rust-demangler").default_condition(default)
1276     }
1277
1278     fn make_run(run: RunConfig<'_>) {
1279         run.builder.ensure(RustDemangler {
1280             compiler: run.builder.compiler_for(
1281                 run.builder.top_stage,
1282                 run.builder.config.build,
1283                 run.target,
1284             ),
1285             target: run.target,
1286         });
1287     }
1288
1289     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1290         let compiler = self.compiler;
1291         let target = self.target;
1292
1293         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1294         let condition = should_build_extended_tool(builder, "rust-demangler")
1295             || builder.config.profiler_enabled(target);
1296         if builder.config.extended && !condition {
1297             return None;
1298         }
1299
1300         let rust_demangler = builder
1301             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1302             .expect("rust-demangler expected to build - in-tree tool");
1303
1304         // Prepare the image directory
1305         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1306         tarball.set_overlay(OverlayKind::RustDemangler);
1307         tarball.is_preview(true);
1308         tarball.add_file(&rust_demangler, "bin", 0o755);
1309         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1310         Some(tarball.generate())
1311     }
1312 }
1313
1314 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1315 pub struct Extended {
1316     stage: u32,
1317     host: TargetSelection,
1318     target: TargetSelection,
1319 }
1320
1321 impl Step for Extended {
1322     type Output = ();
1323     const DEFAULT: bool = true;
1324     const ONLY_HOSTS: bool = true;
1325
1326     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1327         let builder = run.builder;
1328         run.alias("extended").default_condition(builder.config.extended)
1329     }
1330
1331     fn make_run(run: RunConfig<'_>) {
1332         run.builder.ensure(Extended {
1333             stage: run.builder.top_stage,
1334             host: run.builder.config.build,
1335             target: run.target,
1336         });
1337     }
1338
1339     /// Creates a combined installer for the specified target in the provided stage.
1340     fn run(self, builder: &Builder<'_>) {
1341         let target = self.target;
1342         let stage = self.stage;
1343         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1344
1345         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1346
1347         let mut tarballs = Vec::new();
1348         let mut built_tools = HashSet::new();
1349         macro_rules! add_component {
1350             ($name:expr => $step:expr) => {
1351                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1352                     tarballs.push(tarball);
1353                     built_tools.insert($name);
1354                 }
1355             };
1356         }
1357
1358         // When rust-std package split from rustc, we needed to ensure that during
1359         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1360         // the std files during uninstall. To do this ensure that rustc comes
1361         // before rust-std in the list below.
1362         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1363         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1364
1365         if target.ends_with("windows-gnu") {
1366             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1367         }
1368
1369         add_component!("rust-docs" => Docs { host: target });
1370         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1371         add_component!("cargo" => Cargo { compiler, target });
1372         add_component!("rustfmt" => Rustfmt { compiler, target });
1373         add_component!("rls" => Rls { compiler, target });
1374         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1375         add_component!("llvm-components" => LlvmTools { target });
1376         add_component!("clippy" => Clippy { compiler, target });
1377         add_component!("miri" => Miri { compiler, target });
1378         add_component!("analysis" => Analysis { compiler, target });
1379
1380         let etc = builder.src.join("src/etc/installer");
1381
1382         // Avoid producing tarballs during a dry run.
1383         if builder.config.dry_run {
1384             return;
1385         }
1386
1387         let tarball = Tarball::new(builder, "rust", &target.triple);
1388         let generated = tarball.combine(&tarballs);
1389
1390         let tmp = tmpdir(builder).join("combined-tarball");
1391         let work = generated.work_dir();
1392
1393         let mut license = String::new();
1394         license += &builder.read(&builder.src.join("COPYRIGHT"));
1395         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1396         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1397         license.push('\n');
1398         license.push('\n');
1399
1400         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1401         let mut rtf = rtf.to_string();
1402         rtf.push('\n');
1403         for line in license.lines() {
1404             rtf.push_str(line);
1405             rtf.push_str("\\line ");
1406         }
1407         rtf.push('}');
1408
1409         fn filter(contents: &str, marker: &str) -> String {
1410             let start = format!("tool-{}-start", marker);
1411             let end = format!("tool-{}-end", marker);
1412             let mut lines = Vec::new();
1413             let mut omitted = false;
1414             for line in contents.lines() {
1415                 if line.contains(&start) {
1416                     omitted = true;
1417                 } else if line.contains(&end) {
1418                     omitted = false;
1419                 } else if !omitted {
1420                     lines.push(line);
1421                 }
1422             }
1423
1424             lines.join("\n")
1425         }
1426
1427         let xform = |p: &Path| {
1428             let mut contents = t!(fs::read_to_string(p));
1429             for tool in &["rust-demangler"] {
1430                 if !built_tools.contains(tool) {
1431                     contents = filter(&contents, tool);
1432                 }
1433             }
1434             let ret = tmp.join(p.file_name().unwrap());
1435             t!(fs::write(&ret, &contents));
1436             ret
1437         };
1438
1439         if target.contains("apple-darwin") {
1440             builder.info("building pkg installer");
1441             let pkg = tmp.join("pkg");
1442             let _ = fs::remove_dir_all(&pkg);
1443
1444             let pkgbuild = |component: &str| {
1445                 let mut cmd = Command::new("pkgbuild");
1446                 cmd.arg("--identifier")
1447                     .arg(format!("org.rust-lang.{}", component))
1448                     .arg("--scripts")
1449                     .arg(pkg.join(component))
1450                     .arg("--nopayload")
1451                     .arg(pkg.join(component).with_extension("pkg"));
1452                 builder.run(&mut cmd);
1453             };
1454
1455             let prepare = |name: &str| {
1456                 builder.create_dir(&pkg.join(name));
1457                 builder.cp_r(
1458                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1459                     &pkg.join(name),
1460                 );
1461                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1462                 pkgbuild(name);
1463             };
1464             prepare("rustc");
1465             prepare("cargo");
1466             prepare("rust-std");
1467             prepare("rust-analysis");
1468             prepare("clippy");
1469             prepare("miri");
1470             prepare("rust-analyzer");
1471             for tool in &["rust-docs", "rust-demangler"] {
1472                 if built_tools.contains(tool) {
1473                     prepare(tool);
1474                 }
1475             }
1476             // create an 'uninstall' package
1477             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1478             pkgbuild("uninstall");
1479
1480             builder.create_dir(&pkg.join("res"));
1481             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1482             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1483             let mut cmd = Command::new("productbuild");
1484             cmd.arg("--distribution")
1485                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1486                 .arg("--resources")
1487                 .arg(pkg.join("res"))
1488                 .arg(distdir(builder).join(format!(
1489                     "{}-{}.pkg",
1490                     pkgname(builder, "rust"),
1491                     target.triple
1492                 )))
1493                 .arg("--package-path")
1494                 .arg(&pkg);
1495             let _time = timeit(builder);
1496             builder.run(&mut cmd);
1497         }
1498
1499         if target.contains("windows") {
1500             let exe = tmp.join("exe");
1501             let _ = fs::remove_dir_all(&exe);
1502
1503             let prepare = |name: &str| {
1504                 builder.create_dir(&exe.join(name));
1505                 let dir = if name == "rust-std" || name == "rust-analysis" {
1506                     format!("{}-{}", name, target.triple)
1507                 } else if name == "rust-analyzer" {
1508                     "rust-analyzer-preview".to_string()
1509                 } else if name == "clippy" {
1510                     "clippy-preview".to_string()
1511                 } else if name == "rust-demangler" {
1512                     "rust-demangler-preview".to_string()
1513                 } else if name == "miri" {
1514                     "miri-preview".to_string()
1515                 } else {
1516                     name.to_string()
1517                 };
1518                 builder.cp_r(
1519                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1520                     &exe.join(name),
1521                 );
1522                 builder.remove(&exe.join(name).join("manifest.in"));
1523             };
1524             prepare("rustc");
1525             prepare("cargo");
1526             prepare("rust-analysis");
1527             prepare("rust-docs");
1528             prepare("rust-std");
1529             prepare("clippy");
1530             prepare("miri");
1531             prepare("rust-analyzer");
1532             for tool in &["rust-demangler"] {
1533                 if built_tools.contains(tool) {
1534                     prepare(tool);
1535                 }
1536             }
1537             if target.ends_with("windows-gnu") {
1538                 prepare("rust-mingw");
1539             }
1540
1541             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1542
1543             // Generate msi installer
1544             let wix_path = env::var_os("WIX")
1545                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1546             let wix = PathBuf::from(wix_path);
1547             let heat = wix.join("bin/heat.exe");
1548             let candle = wix.join("bin/candle.exe");
1549             let light = wix.join("bin/light.exe");
1550
1551             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1552             builder.run(
1553                 Command::new(&heat)
1554                     .current_dir(&exe)
1555                     .arg("dir")
1556                     .arg("rustc")
1557                     .args(&heat_flags)
1558                     .arg("-cg")
1559                     .arg("RustcGroup")
1560                     .arg("-dr")
1561                     .arg("Rustc")
1562                     .arg("-var")
1563                     .arg("var.RustcDir")
1564                     .arg("-out")
1565                     .arg(exe.join("RustcGroup.wxs")),
1566             );
1567             builder.run(
1568                 Command::new(&heat)
1569                     .current_dir(&exe)
1570                     .arg("dir")
1571                     .arg("rust-docs")
1572                     .args(&heat_flags)
1573                     .arg("-cg")
1574                     .arg("DocsGroup")
1575                     .arg("-dr")
1576                     .arg("Docs")
1577                     .arg("-var")
1578                     .arg("var.DocsDir")
1579                     .arg("-out")
1580                     .arg(exe.join("DocsGroup.wxs"))
1581                     .arg("-t")
1582                     .arg(etc.join("msi/squash-components.xsl")),
1583             );
1584             builder.run(
1585                 Command::new(&heat)
1586                     .current_dir(&exe)
1587                     .arg("dir")
1588                     .arg("cargo")
1589                     .args(&heat_flags)
1590                     .arg("-cg")
1591                     .arg("CargoGroup")
1592                     .arg("-dr")
1593                     .arg("Cargo")
1594                     .arg("-var")
1595                     .arg("var.CargoDir")
1596                     .arg("-out")
1597                     .arg(exe.join("CargoGroup.wxs"))
1598                     .arg("-t")
1599                     .arg(etc.join("msi/remove-duplicates.xsl")),
1600             );
1601             builder.run(
1602                 Command::new(&heat)
1603                     .current_dir(&exe)
1604                     .arg("dir")
1605                     .arg("rust-std")
1606                     .args(&heat_flags)
1607                     .arg("-cg")
1608                     .arg("StdGroup")
1609                     .arg("-dr")
1610                     .arg("Std")
1611                     .arg("-var")
1612                     .arg("var.StdDir")
1613                     .arg("-out")
1614                     .arg(exe.join("StdGroup.wxs")),
1615             );
1616             builder.run(
1617                 Command::new(&heat)
1618                     .current_dir(&exe)
1619                     .arg("dir")
1620                     .arg("rust-analyzer")
1621                     .args(&heat_flags)
1622                     .arg("-cg")
1623                     .arg("RustAnalyzerGroup")
1624                     .arg("-dr")
1625                     .arg("RustAnalyzer")
1626                     .arg("-var")
1627                     .arg("var.RustAnalyzerDir")
1628                     .arg("-out")
1629                     .arg(exe.join("RustAnalyzerGroup.wxs"))
1630                     .arg("-t")
1631                     .arg(etc.join("msi/remove-duplicates.xsl")),
1632             );
1633             builder.run(
1634                 Command::new(&heat)
1635                     .current_dir(&exe)
1636                     .arg("dir")
1637                     .arg("clippy")
1638                     .args(&heat_flags)
1639                     .arg("-cg")
1640                     .arg("ClippyGroup")
1641                     .arg("-dr")
1642                     .arg("Clippy")
1643                     .arg("-var")
1644                     .arg("var.ClippyDir")
1645                     .arg("-out")
1646                     .arg(exe.join("ClippyGroup.wxs"))
1647                     .arg("-t")
1648                     .arg(etc.join("msi/remove-duplicates.xsl")),
1649             );
1650             if built_tools.contains("rust-demangler") {
1651                 builder.run(
1652                     Command::new(&heat)
1653                         .current_dir(&exe)
1654                         .arg("dir")
1655                         .arg("rust-demangler")
1656                         .args(&heat_flags)
1657                         .arg("-cg")
1658                         .arg("RustDemanglerGroup")
1659                         .arg("-dr")
1660                         .arg("RustDemangler")
1661                         .arg("-var")
1662                         .arg("var.RustDemanglerDir")
1663                         .arg("-out")
1664                         .arg(exe.join("RustDemanglerGroup.wxs"))
1665                         .arg("-t")
1666                         .arg(etc.join("msi/remove-duplicates.xsl")),
1667                 );
1668             }
1669             builder.run(
1670                 Command::new(&heat)
1671                     .current_dir(&exe)
1672                     .arg("dir")
1673                     .arg("miri")
1674                     .args(&heat_flags)
1675                     .arg("-cg")
1676                     .arg("MiriGroup")
1677                     .arg("-dr")
1678                     .arg("Miri")
1679                     .arg("-var")
1680                     .arg("var.MiriDir")
1681                     .arg("-out")
1682                     .arg(exe.join("MiriGroup.wxs"))
1683                     .arg("-t")
1684                     .arg(etc.join("msi/remove-duplicates.xsl")),
1685             );
1686             builder.run(
1687                 Command::new(&heat)
1688                     .current_dir(&exe)
1689                     .arg("dir")
1690                     .arg("rust-analysis")
1691                     .args(&heat_flags)
1692                     .arg("-cg")
1693                     .arg("AnalysisGroup")
1694                     .arg("-dr")
1695                     .arg("Analysis")
1696                     .arg("-var")
1697                     .arg("var.AnalysisDir")
1698                     .arg("-out")
1699                     .arg(exe.join("AnalysisGroup.wxs"))
1700                     .arg("-t")
1701                     .arg(etc.join("msi/remove-duplicates.xsl")),
1702             );
1703             if target.ends_with("windows-gnu") {
1704                 builder.run(
1705                     Command::new(&heat)
1706                         .current_dir(&exe)
1707                         .arg("dir")
1708                         .arg("rust-mingw")
1709                         .args(&heat_flags)
1710                         .arg("-cg")
1711                         .arg("GccGroup")
1712                         .arg("-dr")
1713                         .arg("Gcc")
1714                         .arg("-var")
1715                         .arg("var.GccDir")
1716                         .arg("-out")
1717                         .arg(exe.join("GccGroup.wxs")),
1718                 );
1719             }
1720
1721             let candle = |input: &Path| {
1722                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1723                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1724                 let mut cmd = Command::new(&candle);
1725                 cmd.current_dir(&exe)
1726                     .arg("-nologo")
1727                     .arg("-dRustcDir=rustc")
1728                     .arg("-dDocsDir=rust-docs")
1729                     .arg("-dCargoDir=cargo")
1730                     .arg("-dStdDir=rust-std")
1731                     .arg("-dAnalysisDir=rust-analysis")
1732                     .arg("-dClippyDir=clippy")
1733                     .arg("-dMiriDir=miri")
1734                     .arg("-arch")
1735                     .arg(&arch)
1736                     .arg("-out")
1737                     .arg(&output)
1738                     .arg(&input);
1739                 add_env(builder, &mut cmd, target);
1740
1741                 if built_tools.contains("rust-demangler") {
1742                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1743                 }
1744                 if built_tools.contains("rust-analyzer") {
1745                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1746                 }
1747                 if target.ends_with("windows-gnu") {
1748                     cmd.arg("-dGccDir=rust-mingw");
1749                 }
1750                 builder.run(&mut cmd);
1751             };
1752             candle(&xform(&etc.join("msi/rust.wxs")));
1753             candle(&etc.join("msi/ui.wxs"));
1754             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1755             candle("RustcGroup.wxs".as_ref());
1756             candle("DocsGroup.wxs".as_ref());
1757             candle("CargoGroup.wxs".as_ref());
1758             candle("StdGroup.wxs".as_ref());
1759             candle("ClippyGroup.wxs".as_ref());
1760             candle("MiriGroup.wxs".as_ref());
1761             if built_tools.contains("rust-demangler") {
1762                 candle("RustDemanglerGroup.wxs".as_ref());
1763             }
1764             if built_tools.contains("rust-analyzer") {
1765                 candle("RustAnalyzerGroup.wxs".as_ref());
1766             }
1767             candle("AnalysisGroup.wxs".as_ref());
1768
1769             if target.ends_with("windows-gnu") {
1770                 candle("GccGroup.wxs".as_ref());
1771             }
1772
1773             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1774             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1775             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1776
1777             builder.info(&format!("building `msi` installer with {:?}", light));
1778             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1779             let mut cmd = Command::new(&light);
1780             cmd.arg("-nologo")
1781                 .arg("-ext")
1782                 .arg("WixUIExtension")
1783                 .arg("-ext")
1784                 .arg("WixUtilExtension")
1785                 .arg("-out")
1786                 .arg(exe.join(&filename))
1787                 .arg("rust.wixobj")
1788                 .arg("ui.wixobj")
1789                 .arg("rustwelcomedlg.wixobj")
1790                 .arg("RustcGroup.wixobj")
1791                 .arg("DocsGroup.wixobj")
1792                 .arg("CargoGroup.wixobj")
1793                 .arg("StdGroup.wixobj")
1794                 .arg("AnalysisGroup.wixobj")
1795                 .arg("ClippyGroup.wixobj")
1796                 .arg("MiriGroup.wixobj")
1797                 .current_dir(&exe);
1798
1799             if built_tools.contains("rust-analyzer") {
1800                 cmd.arg("RustAnalyzerGroup.wixobj");
1801             }
1802             if built_tools.contains("rust-demangler") {
1803                 cmd.arg("RustDemanglerGroup.wixobj");
1804             }
1805
1806             if target.ends_with("windows-gnu") {
1807                 cmd.arg("GccGroup.wixobj");
1808             }
1809             // ICE57 wrongly complains about the shortcuts
1810             cmd.arg("-sice:ICE57");
1811
1812             let _time = timeit(builder);
1813             builder.run(&mut cmd);
1814
1815             if !builder.config.dry_run {
1816                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1817             }
1818         }
1819     }
1820 }
1821
1822 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1823     let mut parts = builder.version.split('.');
1824     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1825         .env("CFG_RELEASE_NUM", &builder.version)
1826         .env("CFG_RELEASE", builder.rust_release())
1827         .env("CFG_VER_MAJOR", parts.next().unwrap())
1828         .env("CFG_VER_MINOR", parts.next().unwrap())
1829         .env("CFG_VER_PATCH", parts.next().unwrap())
1830         .env("CFG_VER_BUILD", "0") // just needed to build
1831         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1832         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1833         .env("CFG_BUILD", target.triple)
1834         .env("CFG_CHANNEL", &builder.config.channel);
1835
1836     if target.contains("windows-gnullvm") {
1837         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1838     } else if target.contains("windows-gnu") {
1839         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1840     } else {
1841         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1842     }
1843 }
1844
1845 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1846 ///
1847 /// Returns whether the files were actually copied.
1848 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1849     if !builder.is_rust_llvm(target) {
1850         // If the LLVM was externally provided, then we don't currently copy
1851         // artifacts into the sysroot. This is not necessarily the right
1852         // choice (in particular, it will require the LLVM dylib to be in
1853         // the linker's load path at runtime), but the common use case for
1854         // external LLVMs is distribution provided LLVMs, and in that case
1855         // they're usually in the standard search path (e.g., /usr/lib) and
1856         // copying them here is going to cause problems as we may end up
1857         // with the wrong files and isn't what distributions want.
1858         //
1859         // This behavior may be revisited in the future though.
1860         //
1861         // If the LLVM is coming from ourselves (just from CI) though, we
1862         // still want to install it, as it otherwise won't be available.
1863         return false;
1864     }
1865
1866     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1867     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1868     // clear why this is the case, though. llvm-config will emit the versioned
1869     // paths and we don't want those in the sysroot (as we're expecting
1870     // unversioned paths).
1871     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1872         let src_libdir = builder.llvm_out(target).join("lib");
1873         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1874         if llvm_dylib_path.exists() {
1875             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1876         }
1877         !builder.config.dry_run
1878     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1879         let mut cmd = Command::new(llvm_config);
1880         cmd.arg("--libfiles");
1881         builder.verbose(&format!("running {:?}", cmd));
1882         let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) };
1883         let build_llvm_out = &builder.llvm_out(builder.config.build);
1884         let target_llvm_out = &builder.llvm_out(target);
1885         for file in files.trim_end().split(' ') {
1886             // If we're not using a custom LLVM, make sure we package for the target.
1887             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1888                 target_llvm_out.join(relative_path)
1889             } else {
1890                 PathBuf::from(file)
1891             };
1892             builder.install(&file, dst_libdir, 0o644);
1893         }
1894         !builder.config.dry_run
1895     } else {
1896         false
1897     }
1898 }
1899
1900 /// Maybe add libLLVM.so to the target lib-dir for linking.
1901 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1902     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1903     // We do not need to copy LLVM files into the sysroot if it is not
1904     // dynamically linked; it is already included into librustc_llvm
1905     // statically.
1906     if builder.llvm_link_shared() {
1907         maybe_install_llvm(builder, target, &dst_libdir);
1908     }
1909 }
1910
1911 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1912 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1913     let dst_libdir =
1914         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1915     // We do not need to copy LLVM files into the sysroot if it is not
1916     // dynamically linked; it is already included into librustc_llvm
1917     // statically.
1918     if builder.llvm_link_shared() {
1919         maybe_install_llvm(builder, target, &dst_libdir);
1920     }
1921 }
1922
1923 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1924 pub struct LlvmTools {
1925     pub target: TargetSelection,
1926 }
1927
1928 impl Step for LlvmTools {
1929     type Output = Option<GeneratedTarball>;
1930     const ONLY_HOSTS: bool = true;
1931     const DEFAULT: bool = true;
1932
1933     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1934         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1935         // FIXME: allow using the names of the tools themselves?
1936         run.alias("llvm-tools").default_condition(default)
1937     }
1938
1939     fn make_run(run: RunConfig<'_>) {
1940         run.builder.ensure(LlvmTools { target: run.target });
1941     }
1942
1943     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1944         let target = self.target;
1945
1946         /* run only if llvm-config isn't used */
1947         if let Some(config) = builder.config.target_config.get(&target) {
1948             if let Some(ref _s) = config.llvm_config {
1949                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1950                 return None;
1951             }
1952         }
1953
1954         builder.ensure(crate::native::Llvm { target });
1955
1956         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1957         tarball.set_overlay(OverlayKind::LLVM);
1958         tarball.is_preview(true);
1959
1960         // Prepare the image directory
1961         let src_bindir = builder.llvm_out(target).join("bin");
1962         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1963         for tool in LLVM_TOOLS {
1964             let exe = src_bindir.join(exe(tool, target));
1965             tarball.add_file(&exe, &dst_bindir, 0o755);
1966         }
1967
1968         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1969         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1970         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1971         // compiler libraries.
1972         maybe_install_llvm_target(builder, target, tarball.image_dir());
1973
1974         Some(tarball.generate())
1975     }
1976 }
1977
1978 // Tarball intended for internal consumption to ease rustc/std development.
1979 //
1980 // Should not be considered stable by end users.
1981 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1982 pub struct RustDev {
1983     pub target: TargetSelection,
1984 }
1985
1986 impl Step for RustDev {
1987     type Output = Option<GeneratedTarball>;
1988     const DEFAULT: bool = true;
1989     const ONLY_HOSTS: bool = true;
1990
1991     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1992         run.alias("rust-dev")
1993     }
1994
1995     fn make_run(run: RunConfig<'_>) {
1996         run.builder.ensure(RustDev { target: run.target });
1997     }
1998
1999     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2000         let target = self.target;
2001
2002         /* run only if llvm-config isn't used */
2003         if let Some(config) = builder.config.target_config.get(&target) {
2004             if let Some(ref _s) = config.llvm_config {
2005                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2006                 return None;
2007             }
2008         }
2009
2010         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2011         tarball.set_overlay(OverlayKind::LLVM);
2012
2013         builder.ensure(crate::native::Llvm { target });
2014
2015         let src_bindir = builder.llvm_out(target).join("bin");
2016         // If updating this list, you likely want to change
2017         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2018         // will not pick up the extra file until LLVM gets bumped.
2019         for bin in &[
2020             "llvm-config",
2021             "llvm-ar",
2022             "llvm-objdump",
2023             "llvm-profdata",
2024             "llvm-bcanalyzer",
2025             "llvm-cov",
2026             "llvm-dwp",
2027             "llvm-nm",
2028             "llvm-dwarfdump",
2029             "llvm-dis",
2030         ] {
2031             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2032         }
2033
2034         // We don't build LLD on some platforms, so only add it if it exists
2035         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2036         if lld_path.exists() {
2037             tarball.add_file(lld_path, "bin", 0o755);
2038         }
2039
2040         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2041
2042         // Copy the include directory as well; needed mostly to build
2043         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2044         // just broadly useful to be able to link against the bundled LLVM.
2045         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2046
2047         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2048         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2049         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2050         // compiler libraries.
2051         let dst_libdir = tarball.image_dir().join("lib");
2052         maybe_install_llvm(builder, target, &dst_libdir);
2053         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2054         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2055
2056         Some(tarball.generate())
2057     }
2058 }
2059
2060 // Tarball intended for internal consumption to ease rustc/std development.
2061 //
2062 // Should not be considered stable by end users.
2063 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2064 pub struct Bootstrap {
2065     pub target: TargetSelection,
2066 }
2067
2068 impl Step for Bootstrap {
2069     type Output = Option<GeneratedTarball>;
2070     const DEFAULT: bool = false;
2071     const ONLY_HOSTS: bool = true;
2072
2073     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2074         run.alias("bootstrap")
2075     }
2076
2077     fn make_run(run: RunConfig<'_>) {
2078         run.builder.ensure(Bootstrap { target: run.target });
2079     }
2080
2081     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2082         let target = self.target;
2083
2084         let tarball = Tarball::new(builder, "bootstrap", &target.triple);
2085
2086         let bootstrap_outdir = &builder.bootstrap_out;
2087         for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
2088             tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
2089         }
2090
2091         Some(tarball.generate())
2092     }
2093 }
2094
2095 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2096 /// release process to avoid cloning the monorepo and building stuff.
2097 ///
2098 /// Should not be considered stable by end users.
2099 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2100 pub struct BuildManifest {
2101     pub target: TargetSelection,
2102 }
2103
2104 impl Step for BuildManifest {
2105     type Output = GeneratedTarball;
2106     const DEFAULT: bool = false;
2107     const ONLY_HOSTS: bool = true;
2108
2109     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2110         run.alias("build-manifest")
2111     }
2112
2113     fn make_run(run: RunConfig<'_>) {
2114         run.builder.ensure(BuildManifest { target: run.target });
2115     }
2116
2117     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2118         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2119
2120         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2121         tarball.add_file(&build_manifest, "bin", 0o755);
2122         tarball.generate()
2123     }
2124 }
2125
2126 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2127 ///
2128 /// Currently this is the PGO profile data.
2129 ///
2130 /// Should not be considered stable by end users.
2131 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2132 pub struct ReproducibleArtifacts {
2133     pub target: TargetSelection,
2134 }
2135
2136 impl Step for ReproducibleArtifacts {
2137     type Output = Option<GeneratedTarball>;
2138     const DEFAULT: bool = true;
2139     const ONLY_HOSTS: bool = true;
2140
2141     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2142         run.alias("reproducible-artifacts")
2143     }
2144
2145     fn make_run(run: RunConfig<'_>) {
2146         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2147     }
2148
2149     fn run(self, builder: &Builder<'_>) -> Self::Output {
2150         let mut added_anything = false;
2151         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2152         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2153             tarball.add_file(path, ".", 0o644);
2154             added_anything = true;
2155         }
2156         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2157             tarball.add_file(path, ".", 0o644);
2158             added_anything = true;
2159         }
2160         if added_anything { Some(tarball.generate()) } else { None }
2161     }
2162 }