src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::channel;
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, output, t, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26
  27 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  28     format!("{}-{}", component, builder.rust_package_vers())
  29 }
  30
  31 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  32     builder.out.join("dist")
  33 }
  34
  35 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  36     builder.out.join("tmp/dist")
  37 }
  38
  39 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  40     if !builder.config.extended {
  41         return false;
  42     }
  43     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  44 }
  45
  46 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  47 pub struct Docs {
  48     pub host: TargetSelection,
  49 }
  50
  51 impl Step for Docs {
  52     type Output = Option<GeneratedTarball>;
  53     const DEFAULT: bool = true;
  54
  55     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  56         let default = run.builder.config.docs;
  57         run.alias("rust-docs").default_condition(default)
  58     }
  59
  60     fn make_run(run: RunConfig<'_>) {
  61         run.builder.ensure(Docs { host: run.target });
  62     }
  63
  64     /// Builds the `rust-docs` installer component.
  65     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  66         let host = self.host;
  67         builder.default_doc(&[]);
  68
  69         let dest = "share/doc/rust/html";
  70
  71         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  72         tarball.set_product_name("Rust Documentation");
  73         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  74         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  75         Some(tarball.generate())
  76     }
  77 }
  78
  79 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  80 pub struct JsonDocs {
  81     pub host: TargetSelection,
  82 }
  83
  84 impl Step for JsonDocs {
  85     type Output = Option<GeneratedTarball>;
  86     const DEFAULT: bool = true;
  87
  88     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  89         let default = run.builder.config.docs;
  90         run.alias("rust-docs-json").default_condition(default)
  91     }
  92
  93     fn make_run(run: RunConfig<'_>) {
  94         run.builder.ensure(JsonDocs { host: run.target });
  95     }
  96
  97     /// Builds the `rust-docs-json` installer component.
  98     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  99         let host = self.host;
 100         builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
 101
 102         let dest = "share/doc/rust/json";
 103
 104         let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple);
 105         tarball.set_product_name("Rust Documentation In JSON Format");
 106         tarball.is_preview(true);
 107         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 108         Some(tarball.generate())
 109     }
 110 }
 111
 112 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 113 pub struct RustcDocs {
 114     pub host: TargetSelection,
 115 }
 116
 117 impl Step for RustcDocs {
 118     type Output = Option<GeneratedTarball>;
 119     const DEFAULT: bool = true;
 120
 121     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 122         let builder = run.builder;
 123         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 124     }
 125
 126     fn make_run(run: RunConfig<'_>) {
 127         run.builder.ensure(RustcDocs { host: run.target });
 128     }
 129
 130     /// Builds the `rustc-docs` installer component.
 131     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 132         let host = self.host;
 133         builder.default_doc(&[]);
 134
 135         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 136         tarball.set_product_name("Rustc Documentation");
 137         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 138         Some(tarball.generate())
 139     }
 140 }
 141
 142 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 143     let mut found = Vec::with_capacity(files.len());
 144
 145     for file in files {
 146         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 147
 148         if let Some(file_path) = file_path {
 149             found.push(file_path);
 150         } else {
 151             panic!("Could not find '{}' in {:?}", file, path);
 152         }
 153     }
 154
 155     found
 156 }
 157
 158 fn make_win_dist(
 159     rust_root: &Path,
 160     plat_root: &Path,
 161     target: TargetSelection,
 162     builder: &Builder<'_>,
 163 ) {
 164     //Ask gcc where it keeps its stuff
 165     let mut cmd = Command::new(builder.cc(target));
 166     cmd.arg("-print-search-dirs");
 167     let gcc_out = output(&mut cmd);
 168
 169     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 170     let mut lib_path = Vec::new();
 171
 172     for line in gcc_out.lines() {
 173         let idx = line.find(':').unwrap();
 174         let key = &line[..idx];
 175         let trim_chars: &[_] = &[' ', '='];
 176         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 177
 178         if key == "programs" {
 179             bin_path.extend(value);
 180         } else if key == "libraries" {
 181             lib_path.extend(value);
 182         }
 183     }
 184
 185     let compiler = if target == "i686-pc-windows-gnu" {
 186         "i686-w64-mingw32-gcc.exe"
 187     } else if target == "x86_64-pc-windows-gnu" {
 188         "x86_64-w64-mingw32-gcc.exe"
 189     } else {
 190         "gcc.exe"
 191     };
 192     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 193     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 194     if target.starts_with("i686-") {
 195         rustc_dlls.push("libgcc_s_dw2-1.dll");
 196     } else {
 197         rustc_dlls.push("libgcc_s_seh-1.dll");
 198     }
 199
 200     let target_libs = [
 201         //MinGW libs
 202         "libgcc.a",
 203         "libgcc_eh.a",
 204         "libgcc_s.a",
 205         "libm.a",
 206         "libmingw32.a",
 207         "libmingwex.a",
 208         "libstdc++.a",
 209         "libiconv.a",
 210         "libmoldname.a",
 211         "libpthread.a",
 212         //Windows import libs
 213         "libadvapi32.a",
 214         "libbcrypt.a",
 215         "libcomctl32.a",
 216         "libcomdlg32.a",
 217         "libcredui.a",
 218         "libcrypt32.a",
 219         "libdbghelp.a",
 220         "libgdi32.a",
 221         "libimagehlp.a",
 222         "libiphlpapi.a",
 223         "libkernel32.a",
 224         "libmsimg32.a",
 225         "libmsvcrt.a",
 226         "libodbc32.a",
 227         "libole32.a",
 228         "liboleaut32.a",
 229         "libopengl32.a",
 230         "libpsapi.a",
 231         "librpcrt4.a",
 232         "libsecur32.a",
 233         "libsetupapi.a",
 234         "libshell32.a",
 235         "libsynchronization.a",
 236         "libuser32.a",
 237         "libuserenv.a",
 238         "libuuid.a",
 239         "libwinhttp.a",
 240         "libwinmm.a",
 241         "libwinspool.a",
 242         "libws2_32.a",
 243         "libwsock32.a",
 244     ];
 245
 246     //Find mingw artifacts we want to bundle
 247     let target_tools = find_files(&target_tools, &bin_path);
 248     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 249     let target_libs = find_files(&target_libs, &lib_path);
 250
 251     // Copy runtime dlls next to rustc.exe
 252     let dist_bin_dir = rust_root.join("bin/");
 253     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 254     for src in rustc_dlls {
 255         builder.copy_to_folder(&src, &dist_bin_dir);
 256     }
 257
 258     //Copy platform tools to platform-specific bin directory
 259     let target_bin_dir = plat_root
 260         .join("lib")
 261         .join("rustlib")
 262         .join(target.triple)
 263         .join("bin")
 264         .join("self-contained");
 265     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 266     for src in target_tools {
 267         builder.copy_to_folder(&src, &target_bin_dir);
 268     }
 269
 270     // Warn windows-gnu users that the bundled GCC cannot compile C files
 271     builder.create(
 272         &target_bin_dir.join("GCC-WARNING.txt"),
 273         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 274          used as a linker. In order to be able to compile projects containing C code use \
 275          the GCC provided by MinGW or Cygwin.",
 276     );
 277
 278     //Copy platform libs to platform-specific lib directory
 279     let target_lib_dir = plat_root
 280         .join("lib")
 281         .join("rustlib")
 282         .join(target.triple)
 283         .join("lib")
 284         .join("self-contained");
 285     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 286     for src in target_libs {
 287         builder.copy_to_folder(&src, &target_lib_dir);
 288     }
 289 }
 290
 291 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 292 pub struct Mingw {
 293     pub host: TargetSelection,
 294 }
 295
 296 impl Step for Mingw {
 297     type Output = Option<GeneratedTarball>;
 298     const DEFAULT: bool = true;
 299
 300     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 301         run.alias("rust-mingw")
 302     }
 303
 304     fn make_run(run: RunConfig<'_>) {
 305         run.builder.ensure(Mingw { host: run.target });
 306     }
 307
 308     /// Builds the `rust-mingw` installer component.
 309     ///
 310     /// This contains all the bits and pieces to run the MinGW Windows targets
 311     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 312     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 313         let host = self.host;
 314         if !host.ends_with("pc-windows-gnu") {
 315             return None;
 316         }
 317
 318         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 319         tarball.set_product_name("Rust MinGW");
 320
 321         // The first argument is a "temporary directory" which is just
 322         // thrown away (this contains the runtime DLLs included in the rustc package
 323         // above) and the second argument is where to place all the MinGW components
 324         // (which is what we want).
 325         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 326
 327         Some(tarball.generate())
 328     }
 329 }
 330
 331 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 332 pub struct Rustc {
 333     pub compiler: Compiler,
 334 }
 335
 336 impl Step for Rustc {
 337     type Output = GeneratedTarball;
 338     const DEFAULT: bool = true;
 339     const ONLY_HOSTS: bool = true;
 340
 341     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 342         run.alias("rustc")
 343     }
 344
 345     fn make_run(run: RunConfig<'_>) {
 346         run.builder
 347             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 348     }
 349
 350     /// Creates the `rustc` installer component.
 351     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 352         let compiler = self.compiler;
 353         let host = self.compiler.host;
 354
 355         let tarball = Tarball::new(builder, "rustc", &host.triple);
 356
 357         // Prepare the rustc "image", what will actually end up getting installed
 358         prepare_image(builder, compiler, tarball.image_dir());
 359
 360         // On MinGW we've got a few runtime DLL dependencies that we need to
 361         // include. The first argument to this script is where to put these DLLs
 362         // (the image we're creating), and the second argument is a junk directory
 363         // to ignore all other MinGW stuff the script creates.
 364         //
 365         // On 32-bit MinGW we're always including a DLL which needs some extra
 366         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 367         // anything requiring us to distribute a license, but it's likely the
 368         // install will *also* include the rust-mingw package, which also needs
 369         // licenses, so to be safe we just include it here in all MinGW packages.
 370         if host.ends_with("pc-windows-gnu") {
 371             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 372             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 373         }
 374
 375         return tarball.generate();
 376
 377         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 378             let host = compiler.host;
 379             let src = builder.sysroot(compiler);
 380
 381             // Copy rustc/rustdoc binaries
 382             t!(fs::create_dir_all(image.join("bin")));
 383             builder.cp_r(&src.join("bin"), &image.join("bin"));
 384
 385             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 386
 387             let ra_proc_macro_srv = builder
 388                 .ensure(tool::RustAnalyzerProcMacroSrv {
 389                     compiler: builder.compiler_for(
 390                         compiler.stage,
 391                         builder.config.build,
 392                         compiler.host,
 393                     ),
 394                     target: compiler.host,
 395                 })
 396                 .expect("rust-analyzer-proc-macro-server always builds");
 397             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 398
 399             let libdir_relative = builder.libdir_relative(compiler);
 400
 401             // Copy runtime DLLs needed by the compiler
 402             if libdir_relative.to_str() != Some("bin") {
 403                 let libdir = builder.rustc_libdir(compiler);
 404                 for entry in builder.read_dir(&libdir) {
 405                     let name = entry.file_name();
 406                     if let Some(s) = name.to_str() {
 407                         if is_dylib(s) {
 408                             // Don't use custom libdir here because ^lib/ will be resolved again
 409                             // with installer
 410                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 411                         }
 412                     }
 413                 }
 414             }
 415
 416             // Copy over the codegen backends
 417             let backends_src = builder.sysroot_codegen_backends(compiler);
 418             let backends_rel = backends_src
 419                 .strip_prefix(&src)
 420                 .unwrap()
 421                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 422                 .unwrap();
 423             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 424             let backends_dst = image.join("lib").join(&backends_rel);
 425
 426             t!(fs::create_dir_all(&backends_dst));
 427             builder.cp_r(&backends_src, &backends_dst);
 428
 429             // Copy libLLVM.so to the lib dir as well, if needed. While not
 430             // technically needed by rustc itself it's needed by lots of other
 431             // components like the llvm tools and LLD. LLD is included below and
 432             // tools/LLDB come later, so let's just throw it in the rustc
 433             // component for now.
 434             maybe_install_llvm_runtime(builder, host, image);
 435
 436             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 437             t!(fs::create_dir_all(&dst_dir));
 438
 439             // Copy over lld if it's there
 440             if builder.config.lld_enabled {
 441                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 442                 let rust_lld = exe("rust-lld", compiler.host);
 443                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 444                 // for `-Z gcc-ld=lld`
 445                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 446                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 447                 t!(fs::create_dir(&gcc_lld_dst_dir));
 448                 for name in crate::LLD_FILE_NAMES {
 449                     let exe_name = exe(name, compiler.host);
 450                     builder
 451                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 452                 }
 453             }
 454
 455             // Man pages
 456             t!(fs::create_dir_all(image.join("share/man/man1")));
 457             let man_src = builder.src.join("src/doc/man");
 458             let man_dst = image.join("share/man/man1");
 459
 460             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 461             // to hardlink, and we don't want to edit the source templates
 462             for file_entry in builder.read_dir(&man_src) {
 463                 let page_src = file_entry.path();
 464                 let page_dst = man_dst.join(file_entry.file_name());
 465                 let src_text = t!(std::fs::read_to_string(&page_src));
 466                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 467                 t!(std::fs::write(&page_dst, &new_text));
 468                 t!(fs::copy(&page_src, &page_dst));
 469             }
 470
 471             // Debugger scripts
 472             builder
 473                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 474
 475             // Misc license info
 476             let cp = |file: &str| {
 477                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 478             };
 479             cp("COPYRIGHT");
 480             cp("LICENSE-APACHE");
 481             cp("LICENSE-MIT");
 482             cp("README.md");
 483         }
 484     }
 485 }
 486
 487 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 488 pub struct DebuggerScripts {
 489     pub sysroot: Interned<PathBuf>,
 490     pub host: TargetSelection,
 491 }
 492
 493 impl Step for DebuggerScripts {
 494     type Output = ();
 495
 496     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 497         run.never()
 498     }
 499
 500     /// Copies debugger scripts for `target` into the `sysroot` specified.
 501     fn run(self, builder: &Builder<'_>) {
 502         let host = self.host;
 503         let sysroot = self.sysroot;
 504         let dst = sysroot.join("lib/rustlib/etc");
 505         t!(fs::create_dir_all(&dst));
 506         let cp_debugger_script = |file: &str| {
 507             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 508         };
 509         if host.contains("windows-msvc") {
 510             // windbg debugger scripts
 511             builder.install(
 512                 &builder.src.join("src/etc/rust-windbg.cmd"),
 513                 &sysroot.join("bin"),
 514                 0o755,
 515             );
 516
 517             cp_debugger_script("natvis/intrinsic.natvis");
 518             cp_debugger_script("natvis/liballoc.natvis");
 519             cp_debugger_script("natvis/libcore.natvis");
 520             cp_debugger_script("natvis/libstd.natvis");
 521         } else {
 522             cp_debugger_script("rust_types.py");
 523
 524             // gdb debugger scripts
 525             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 526             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 527
 528             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 529             cp_debugger_script("gdb_lookup.py");
 530             cp_debugger_script("gdb_providers.py");
 531
 532             // lldb debugger scripts
 533             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 534
 535             cp_debugger_script("lldb_lookup.py");
 536             cp_debugger_script("lldb_providers.py");
 537             cp_debugger_script("lldb_commands")
 538         }
 539     }
 540 }
 541
 542 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 543     // The only true set of target libraries came from the build triple, so
 544     // let's reduce redundant work by only producing archives from that host.
 545     if compiler.host != builder.config.build {
 546         builder.info("\tskipping, not a build host");
 547         true
 548     } else {
 549         false
 550     }
 551 }
 552
 553 /// Copy stamped files into an image's `target/lib` directory.
 554 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 555     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 556     let self_contained_dst = dst.join("self-contained");
 557     t!(fs::create_dir_all(&dst));
 558     t!(fs::create_dir_all(&self_contained_dst));
 559     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 560         if dependency_type == DependencyType::TargetSelfContained {
 561             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 562         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 563             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 564         }
 565     }
 566 }
 567
 568 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 569 pub struct Std {
 570     pub compiler: Compiler,
 571     pub target: TargetSelection,
 572 }
 573
 574 impl Step for Std {
 575     type Output = Option<GeneratedTarball>;
 576     const DEFAULT: bool = true;
 577
 578     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 579         run.alias("rust-std")
 580     }
 581
 582     fn make_run(run: RunConfig<'_>) {
 583         run.builder.ensure(Std {
 584             compiler: run.builder.compiler_for(
 585                 run.builder.top_stage,
 586                 run.builder.config.build,
 587                 run.target,
 588             ),
 589             target: run.target,
 590         });
 591     }
 592
 593     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 594         let compiler = self.compiler;
 595         let target = self.target;
 596
 597         if skip_host_target_lib(builder, compiler) {
 598             return None;
 599         }
 600
 601         builder.ensure(compile::Std::new(compiler, target));
 602
 603         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 604         tarball.include_target_in_component_name(true);
 605
 606         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 607         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 608         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 609
 610         Some(tarball.generate())
 611     }
 612 }
 613
 614 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 615 pub struct RustcDev {
 616     pub compiler: Compiler,
 617     pub target: TargetSelection,
 618 }
 619
 620 impl Step for RustcDev {
 621     type Output = Option<GeneratedTarball>;
 622     const DEFAULT: bool = true;
 623     const ONLY_HOSTS: bool = true;
 624
 625     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 626         run.alias("rustc-dev")
 627     }
 628
 629     fn make_run(run: RunConfig<'_>) {
 630         run.builder.ensure(RustcDev {
 631             compiler: run.builder.compiler_for(
 632                 run.builder.top_stage,
 633                 run.builder.config.build,
 634                 run.target,
 635             ),
 636             target: run.target,
 637         });
 638     }
 639
 640     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 641         let compiler = self.compiler;
 642         let target = self.target;
 643         if skip_host_target_lib(builder, compiler) {
 644             return None;
 645         }
 646
 647         builder.ensure(compile::Rustc::new(compiler, target));
 648
 649         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 650
 651         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 652         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 653         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 654
 655         let src_files = &["Cargo.lock"];
 656         // This is the reduced set of paths which will become the rustc-dev component
 657         // (essentially the compiler crates and all of their path dependencies).
 658         copy_src_dirs(
 659             builder,
 660             &builder.src,
 661             &["compiler"],
 662             &[],
 663             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 664         );
 665         for file in src_files {
 666             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 667         }
 668
 669         Some(tarball.generate())
 670     }
 671 }
 672
 673 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 674 pub struct Analysis {
 675     pub compiler: Compiler,
 676     pub target: TargetSelection,
 677 }
 678
 679 impl Step for Analysis {
 680     type Output = Option<GeneratedTarball>;
 681     const DEFAULT: bool = true;
 682
 683     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 684         let default = should_build_extended_tool(&run.builder, "analysis");
 685         run.alias("rust-analysis").default_condition(default)
 686     }
 687
 688     fn make_run(run: RunConfig<'_>) {
 689         run.builder.ensure(Analysis {
 690             // Find the actual compiler (handling the full bootstrap option) which
 691             // produced the save-analysis data because that data isn't copied
 692             // through the sysroot uplifting.
 693             compiler: run.builder.compiler_for(
 694                 run.builder.top_stage,
 695                 run.builder.config.build,
 696                 run.target,
 697             ),
 698             target: run.target,
 699         });
 700     }
 701
 702     /// Creates a tarball of save-analysis metadata, if available.
 703     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 704         let compiler = self.compiler;
 705         let target = self.target;
 706         if compiler.host != builder.config.build {
 707             return None;
 708         }
 709
 710         builder.ensure(compile::Std::new(compiler, target));
 711         let src = builder
 712             .stage_out(compiler, Mode::Std)
 713             .join(target.triple)
 714             .join(builder.cargo_dir())
 715             .join("deps")
 716             .join("save-analysis");
 717
 718         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 719         tarball.include_target_in_component_name(true);
 720         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 721         Some(tarball.generate())
 722     }
 723 }
 724
 725 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 726 /// `dst_dir`.
 727 fn copy_src_dirs(
 728     builder: &Builder<'_>,
 729     base: &Path,
 730     src_dirs: &[&str],
 731     exclude_dirs: &[&str],
 732     dst_dir: &Path,
 733 ) {
 734     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 735         let spath = match path.to_str() {
 736             Some(path) => path,
 737             None => return false,
 738         };
 739         if spath.ends_with('~') || spath.ends_with(".pyc") {
 740             return false;
 741         }
 742
 743         const LLVM_PROJECTS: &[&str] = &[
 744             "llvm-project/clang",
 745             "llvm-project\\clang",
 746             "llvm-project/libunwind",
 747             "llvm-project\\libunwind",
 748             "llvm-project/lld",
 749             "llvm-project\\lld",
 750             "llvm-project/lldb",
 751             "llvm-project\\lldb",
 752             "llvm-project/llvm",
 753             "llvm-project\\llvm",
 754             "llvm-project/compiler-rt",
 755             "llvm-project\\compiler-rt",
 756             "llvm-project/cmake",
 757             "llvm-project\\cmake",
 758         ];
 759         if spath.contains("llvm-project")
 760             && !spath.ends_with("llvm-project")
 761             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 762         {
 763             return false;
 764         }
 765
 766         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 767         if LLVM_TEST.iter().any(|path| spath.contains(path))
 768             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 769         {
 770             return false;
 771         }
 772
 773         let full_path = Path::new(dir).join(path);
 774         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 775             return false;
 776         }
 777
 778         let excludes = [
 779             "CVS",
 780             "RCS",
 781             "SCCS",
 782             ".git",
 783             ".gitignore",
 784             ".gitmodules",
 785             ".gitattributes",
 786             ".cvsignore",
 787             ".svn",
 788             ".arch-ids",
 789             "{arch}",
 790             "=RELEASE-ID",
 791             "=meta-update",
 792             "=update",
 793             ".bzr",
 794             ".bzrignore",
 795             ".bzrtags",
 796             ".hg",
 797             ".hgignore",
 798             ".hgrags",
 799             "_darcs",
 800         ];
 801         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 802     }
 803
 804     // Copy the directories using our filter
 805     for item in src_dirs {
 806         let dst = &dst_dir.join(item);
 807         t!(fs::create_dir_all(dst));
 808         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 809     }
 810 }
 811
 812 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 813 pub struct Src;
 814
 815 impl Step for Src {
 816     /// The output path of the src installer tarball
 817     type Output = GeneratedTarball;
 818     const DEFAULT: bool = true;
 819     const ONLY_HOSTS: bool = true;
 820
 821     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 822         run.alias("rust-src")
 823     }
 824
 825     fn make_run(run: RunConfig<'_>) {
 826         run.builder.ensure(Src);
 827     }
 828
 829     /// Creates the `rust-src` installer component
 830     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 831         let tarball = Tarball::new_targetless(builder, "rust-src");
 832
 833         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 834         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 835         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 836         // and fix them...
 837         //
 838         // NOTE: if you update the paths here, you also should update the "virtual" path
 839         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 840         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 841
 842         let src_files = ["Cargo.lock"];
 843         // This is the reduced set of paths which will become the rust-src component
 844         // (essentially libstd and all of its path dependencies).
 845         copy_src_dirs(
 846             builder,
 847             &builder.src,
 848             &["library", "src/llvm-project/libunwind"],
 849             &[
 850                 // not needed and contains symlinks which rustup currently
 851                 // chokes on when unpacking.
 852                 "library/backtrace/crates",
 853                 // these are 30MB combined and aren't necessary for building
 854                 // the standard library.
 855                 "library/stdarch/Cargo.toml",
 856                 "library/stdarch/crates/stdarch-verify",
 857                 "library/stdarch/crates/intrinsic-test",
 858             ],
 859             &dst_src,
 860         );
 861         for file in src_files.iter() {
 862             builder.copy(&builder.src.join(file), &dst_src.join(file));
 863         }
 864
 865         tarball.generate()
 866     }
 867 }
 868
 869 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 870 pub struct PlainSourceTarball;
 871
 872 impl Step for PlainSourceTarball {
 873     /// Produces the location of the tarball generated
 874     type Output = GeneratedTarball;
 875     const DEFAULT: bool = true;
 876     const ONLY_HOSTS: bool = true;
 877
 878     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 879         let builder = run.builder;
 880         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 881     }
 882
 883     fn make_run(run: RunConfig<'_>) {
 884         run.builder.ensure(PlainSourceTarball);
 885     }
 886
 887     /// Creates the plain source tarball
 888     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 889         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 890         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 891         // It also contains symbolic links, unlike other any other dist tarball.
 892         // It's used for distros building rustc from source in a pre-vendored environment.
 893         let mut tarball = Tarball::new(builder, "rustc", "src");
 894         tarball.permit_symlinks(true);
 895         let plain_dst_src = tarball.image_dir();
 896
 897         // This is the set of root paths which will become part of the source package
 898         let src_files = [
 899             "COPYRIGHT",
 900             "LICENSE-APACHE",
 901             "LICENSE-MIT",
 902             "CONTRIBUTING.md",
 903             "README.md",
 904             "RELEASES.md",
 905             "configure",
 906             "x.py",
 907             "config.toml.example",
 908             "Cargo.toml",
 909             "Cargo.lock",
 910         ];
 911         let src_dirs = ["src", "compiler", "library"];
 912
 913         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 914
 915         // Copy the files normally
 916         for item in &src_files {
 917             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 918         }
 919
 920         // Create the version file
 921         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 922         if let Some(info) = builder.rust_info.info() {
 923             channel::write_commit_hash_file(&plain_dst_src, &info.sha);
 924             channel::write_commit_info_file(&plain_dst_src, info);
 925         }
 926
 927         // If we're building from git sources, we need to vendor a complete distribution.
 928         if builder.rust_info.is_managed_git_subrepository() {
 929             // Ensure we have the submodules checked out.
 930             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 931
 932             // Vendor all Cargo dependencies
 933             let mut cmd = Command::new(&builder.initial_cargo);
 934             cmd.arg("vendor")
 935                 .arg("--sync")
 936                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 937                 .arg("--sync")
 938                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 939                 .arg("--sync")
 940                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 941                 .current_dir(&plain_dst_src);
 942
 943             let config = if !builder.config.dry_run {
 944                 t!(String::from_utf8(t!(cmd.output()).stdout))
 945             } else {
 946                 String::new()
 947             };
 948
 949             let cargo_config_dir = plain_dst_src.join(".cargo");
 950             builder.create_dir(&cargo_config_dir);
 951             builder.create(&cargo_config_dir.join("config.toml"), &config);
 952         }
 953
 954         tarball.bare()
 955     }
 956 }
 957
 958 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 959 pub struct Cargo {
 960     pub compiler: Compiler,
 961     pub target: TargetSelection,
 962 }
 963
 964 impl Step for Cargo {
 965     type Output = Option<GeneratedTarball>;
 966     const DEFAULT: bool = true;
 967     const ONLY_HOSTS: bool = true;
 968
 969     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 970         let default = should_build_extended_tool(&run.builder, "cargo");
 971         run.alias("cargo").default_condition(default)
 972     }
 973
 974     fn make_run(run: RunConfig<'_>) {
 975         run.builder.ensure(Cargo {
 976             compiler: run.builder.compiler_for(
 977                 run.builder.top_stage,
 978                 run.builder.config.build,
 979                 run.target,
 980             ),
 981             target: run.target,
 982         });
 983     }
 984
 985     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 986         let compiler = self.compiler;
 987         let target = self.target;
 988
 989         let cargo = builder.ensure(tool::Cargo { compiler, target });
 990         let src = builder.src.join("src/tools/cargo");
 991         let etc = src.join("src/etc");
 992
 993         // Prepare the image directory
 994         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 995         tarball.set_overlay(OverlayKind::Cargo);
 996
 997         tarball.add_file(&cargo, "bin", 0o755);
 998         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 999         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1000         tarball.add_dir(etc.join("man"), "share/man/man1");
1001         tarball.add_legal_and_readme_to("share/doc/cargo");
1002
1003         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1004             let dirent = dirent.expect("read dir entry");
1005             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1006                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1007             }
1008         }
1009
1010         Some(tarball.generate())
1011     }
1012 }
1013
1014 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1015 pub struct Rls {
1016     pub compiler: Compiler,
1017     pub target: TargetSelection,
1018 }
1019
1020 impl Step for Rls {
1021     type Output = Option<GeneratedTarball>;
1022     const ONLY_HOSTS: bool = true;
1023     const DEFAULT: bool = true;
1024
1025     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1026         let default = should_build_extended_tool(&run.builder, "rls");
1027         run.alias("rls").default_condition(default)
1028     }
1029
1030     fn make_run(run: RunConfig<'_>) {
1031         run.builder.ensure(Rls {
1032             compiler: run.builder.compiler_for(
1033                 run.builder.top_stage,
1034                 run.builder.config.build,
1035                 run.target,
1036             ),
1037             target: run.target,
1038         });
1039     }
1040
1041     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1042         let compiler = self.compiler;
1043         let target = self.target;
1044
1045         let rls = builder
1046             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1047             .expect("rls expected to build");
1048
1049         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1050         tarball.set_overlay(OverlayKind::RLS);
1051         tarball.is_preview(true);
1052         tarball.add_file(rls, "bin", 0o755);
1053         tarball.add_legal_and_readme_to("share/doc/rls");
1054         Some(tarball.generate())
1055     }
1056 }
1057
1058 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1059 pub struct RustAnalyzer {
1060     pub compiler: Compiler,
1061     pub target: TargetSelection,
1062 }
1063
1064 impl Step for RustAnalyzer {
1065     type Output = Option<GeneratedTarball>;
1066     const DEFAULT: bool = true;
1067     const ONLY_HOSTS: bool = true;
1068
1069     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1070         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1071         run.alias("rust-analyzer").default_condition(default)
1072     }
1073
1074     fn make_run(run: RunConfig<'_>) {
1075         run.builder.ensure(RustAnalyzer {
1076             compiler: run.builder.compiler_for(
1077                 run.builder.top_stage,
1078                 run.builder.config.build,
1079                 run.target,
1080             ),
1081             target: run.target,
1082         });
1083     }
1084
1085     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1086         let compiler = self.compiler;
1087         let target = self.target;
1088
1089         if target.contains("riscv64") {
1090             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1091             // to build. See #74813 for details.
1092             return None;
1093         }
1094
1095         let rust_analyzer = builder
1096             .ensure(tool::RustAnalyzer { compiler, target })
1097             .expect("rust-analyzer always builds");
1098
1099         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1100         tarball.set_overlay(OverlayKind::RustAnalyzer);
1101         tarball.is_preview(true);
1102         tarball.add_file(rust_analyzer, "bin", 0o755);
1103         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1104         Some(tarball.generate())
1105     }
1106 }
1107
1108 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1109 pub struct Clippy {
1110     pub compiler: Compiler,
1111     pub target: TargetSelection,
1112 }
1113
1114 impl Step for Clippy {
1115     type Output = Option<GeneratedTarball>;
1116     const DEFAULT: bool = true;
1117     const ONLY_HOSTS: bool = true;
1118
1119     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1120         let default = should_build_extended_tool(&run.builder, "clippy");
1121         run.alias("clippy").default_condition(default)
1122     }
1123
1124     fn make_run(run: RunConfig<'_>) {
1125         run.builder.ensure(Clippy {
1126             compiler: run.builder.compiler_for(
1127                 run.builder.top_stage,
1128                 run.builder.config.build,
1129                 run.target,
1130             ),
1131             target: run.target,
1132         });
1133     }
1134
1135     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1136         let compiler = self.compiler;
1137         let target = self.target;
1138
1139         // Prepare the image directory
1140         // We expect clippy to build, because we've exited this step above if tool
1141         // state for clippy isn't testing.
1142         let clippy = builder
1143             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1144             .expect("clippy expected to build - essential tool");
1145         let cargoclippy = builder
1146             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1147             .expect("clippy expected to build - essential tool");
1148
1149         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1150         tarball.set_overlay(OverlayKind::Clippy);
1151         tarball.is_preview(true);
1152         tarball.add_file(clippy, "bin", 0o755);
1153         tarball.add_file(cargoclippy, "bin", 0o755);
1154         tarball.add_legal_and_readme_to("share/doc/clippy");
1155         Some(tarball.generate())
1156     }
1157 }
1158
1159 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1160 pub struct Miri {
1161     pub compiler: Compiler,
1162     pub target: TargetSelection,
1163 }
1164
1165 impl Step for Miri {
1166     type Output = Option<GeneratedTarball>;
1167     const DEFAULT: bool = true;
1168     const ONLY_HOSTS: bool = true;
1169
1170     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1171         let default = should_build_extended_tool(&run.builder, "miri");
1172         run.alias("miri").default_condition(default)
1173     }
1174
1175     fn make_run(run: RunConfig<'_>) {
1176         run.builder.ensure(Miri {
1177             compiler: run.builder.compiler_for(
1178                 run.builder.top_stage,
1179                 run.builder.config.build,
1180                 run.target,
1181             ),
1182             target: run.target,
1183         });
1184     }
1185
1186     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1187         // This prevents miri from being built for "dist" or "install"
1188         // on the stable/beta channels. It is a nightly-only tool and should
1189         // not be included.
1190         if !builder.build.unstable_features() {
1191             return None;
1192         }
1193         let compiler = self.compiler;
1194         let target = self.target;
1195
1196         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1197         let cargomiri =
1198             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1199
1200         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1201         tarball.set_overlay(OverlayKind::Miri);
1202         tarball.is_preview(true);
1203         tarball.add_file(miri, "bin", 0o755);
1204         tarball.add_file(cargomiri, "bin", 0o755);
1205         tarball.add_legal_and_readme_to("share/doc/miri");
1206         Some(tarball.generate())
1207     }
1208 }
1209
1210 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1211 pub struct Rustfmt {
1212     pub compiler: Compiler,
1213     pub target: TargetSelection,
1214 }
1215
1216 impl Step for Rustfmt {
1217     type Output = Option<GeneratedTarball>;
1218     const DEFAULT: bool = true;
1219     const ONLY_HOSTS: bool = true;
1220
1221     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1222         let default = should_build_extended_tool(&run.builder, "rustfmt");
1223         run.alias("rustfmt").default_condition(default)
1224     }
1225
1226     fn make_run(run: RunConfig<'_>) {
1227         run.builder.ensure(Rustfmt {
1228             compiler: run.builder.compiler_for(
1229                 run.builder.top_stage,
1230                 run.builder.config.build,
1231                 run.target,
1232             ),
1233             target: run.target,
1234         });
1235     }
1236
1237     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1238         let compiler = self.compiler;
1239         let target = self.target;
1240
1241         let rustfmt = builder
1242             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1243             .expect("rustfmt expected to build - essential tool");
1244         let cargofmt = builder
1245             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1246             .expect("cargo fmt expected to build - essential tool");
1247         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1248         tarball.set_overlay(OverlayKind::Rustfmt);
1249         tarball.is_preview(true);
1250         tarball.add_file(rustfmt, "bin", 0o755);
1251         tarball.add_file(cargofmt, "bin", 0o755);
1252         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1253         Some(tarball.generate())
1254     }
1255 }
1256
1257 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1258 pub struct RustDemangler {
1259     pub compiler: Compiler,
1260     pub target: TargetSelection,
1261 }
1262
1263 impl Step for RustDemangler {
1264     type Output = Option<GeneratedTarball>;
1265     const DEFAULT: bool = true;
1266     const ONLY_HOSTS: bool = true;
1267
1268     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1269         // While other tools use `should_build_extended_tool` to decide whether to be run by
1270         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1271         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1272         // we run the step by default when only `extended = true`, and decide whether to actually
1273         // run it or not later.
1274         let default = run.builder.config.extended;
1275         run.alias("rust-demangler").default_condition(default)
1276     }
1277
1278     fn make_run(run: RunConfig<'_>) {
1279         run.builder.ensure(RustDemangler {
1280             compiler: run.builder.compiler_for(
1281                 run.builder.top_stage,
1282                 run.builder.config.build,
1283                 run.target,
1284             ),
1285             target: run.target,
1286         });
1287     }
1288
1289     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1290         let compiler = self.compiler;
1291         let target = self.target;
1292
1293         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1294         let condition = should_build_extended_tool(builder, "rust-demangler")
1295             || builder.config.profiler_enabled(target);
1296         if builder.config.extended && !condition {
1297             return None;
1298         }
1299
1300         let rust_demangler = builder
1301             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1302             .expect("rust-demangler expected to build - in-tree tool");
1303
1304         // Prepare the image directory
1305         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1306         tarball.set_overlay(OverlayKind::RustDemangler);
1307         tarball.is_preview(true);
1308         tarball.add_file(&rust_demangler, "bin", 0o755);
1309         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1310         Some(tarball.generate())
1311     }
1312 }
1313
1314 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1315 pub struct Extended {
1316     stage: u32,
1317     host: TargetSelection,
1318     target: TargetSelection,
1319 }
1320
1321 impl Step for Extended {
1322     type Output = ();
1323     const DEFAULT: bool = true;
1324     const ONLY_HOSTS: bool = true;
1325
1326     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1327         let builder = run.builder;
1328         run.alias("extended").default_condition(builder.config.extended)
1329     }
1330
1331     fn make_run(run: RunConfig<'_>) {
1332         run.builder.ensure(Extended {
1333             stage: run.builder.top_stage,
1334             host: run.builder.config.build,
1335             target: run.target,
1336         });
1337     }
1338
1339     /// Creates a combined installer for the specified target in the provided stage.
1340     fn run(self, builder: &Builder<'_>) {
1341         let target = self.target;
1342         let stage = self.stage;
1343         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1344
1345         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1346
1347         let mut tarballs = Vec::new();
1348         let mut built_tools = HashSet::new();
1349         macro_rules! add_component {
1350             ($name:expr => $step:expr) => {
1351                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1352                     tarballs.push(tarball);
1353                     built_tools.insert($name);
1354                 }
1355             };
1356         }
1357
1358         // When rust-std package split from rustc, we needed to ensure that during
1359         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1360         // the std files during uninstall. To do this ensure that rustc comes
1361         // before rust-std in the list below.
1362         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1363         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1364
1365         if target.ends_with("windows-gnu") {
1366             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1367         }
1368
1369         add_component!("rust-docs" => Docs { host: target });
1370         add_component!("rust-json-docs" => JsonDocs { host: target });
1371         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1372         add_component!("cargo" => Cargo { compiler, target });
1373         add_component!("rustfmt" => Rustfmt { compiler, target });
1374         add_component!("rls" => Rls { compiler, target });
1375         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1376         add_component!("llvm-components" => LlvmTools { target });
1377         add_component!("clippy" => Clippy { compiler, target });
1378         add_component!("miri" => Miri { compiler, target });
1379         add_component!("analysis" => Analysis { compiler, target });
1380
1381         let etc = builder.src.join("src/etc/installer");
1382
1383         // Avoid producing tarballs during a dry run.
1384         if builder.config.dry_run {
1385             return;
1386         }
1387
1388         let tarball = Tarball::new(builder, "rust", &target.triple);
1389         let generated = tarball.combine(&tarballs);
1390
1391         let tmp = tmpdir(builder).join("combined-tarball");
1392         let work = generated.work_dir();
1393
1394         let mut license = String::new();
1395         license += &builder.read(&builder.src.join("COPYRIGHT"));
1396         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1397         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1398         license.push('\n');
1399         license.push('\n');
1400
1401         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1402         let mut rtf = rtf.to_string();
1403         rtf.push('\n');
1404         for line in license.lines() {
1405             rtf.push_str(line);
1406             rtf.push_str("\\line ");
1407         }
1408         rtf.push('}');
1409
1410         fn filter(contents: &str, marker: &str) -> String {
1411             let start = format!("tool-{}-start", marker);
1412             let end = format!("tool-{}-end", marker);
1413             let mut lines = Vec::new();
1414             let mut omitted = false;
1415             for line in contents.lines() {
1416                 if line.contains(&start) {
1417                     omitted = true;
1418                 } else if line.contains(&end) {
1419                     omitted = false;
1420                 } else if !omitted {
1421                     lines.push(line);
1422                 }
1423             }
1424
1425             lines.join("\n")
1426         }
1427
1428         let xform = |p: &Path| {
1429             let mut contents = t!(fs::read_to_string(p));
1430             for tool in &["rust-demangler"] {
1431                 if !built_tools.contains(tool) {
1432                     contents = filter(&contents, tool);
1433                 }
1434             }
1435             let ret = tmp.join(p.file_name().unwrap());
1436             t!(fs::write(&ret, &contents));
1437             ret
1438         };
1439
1440         if target.contains("apple-darwin") {
1441             builder.info("building pkg installer");
1442             let pkg = tmp.join("pkg");
1443             let _ = fs::remove_dir_all(&pkg);
1444
1445             let pkgbuild = |component: &str| {
1446                 let mut cmd = Command::new("pkgbuild");
1447                 cmd.arg("--identifier")
1448                     .arg(format!("org.rust-lang.{}", component))
1449                     .arg("--scripts")
1450                     .arg(pkg.join(component))
1451                     .arg("--nopayload")
1452                     .arg(pkg.join(component).with_extension("pkg"));
1453                 builder.run(&mut cmd);
1454             };
1455
1456             let prepare = |name: &str| {
1457                 builder.create_dir(&pkg.join(name));
1458                 builder.cp_r(
1459                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1460                     &pkg.join(name),
1461                 );
1462                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1463                 pkgbuild(name);
1464             };
1465             prepare("rustc");
1466             prepare("cargo");
1467             prepare("rust-std");
1468             prepare("rust-analysis");
1469             prepare("clippy");
1470             prepare("miri");
1471             prepare("rust-analyzer");
1472             for tool in &["rust-docs", "rust-demangler"] {
1473                 if built_tools.contains(tool) {
1474                     prepare(tool);
1475                 }
1476             }
1477             // create an 'uninstall' package
1478             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1479             pkgbuild("uninstall");
1480
1481             builder.create_dir(&pkg.join("res"));
1482             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1483             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1484             let mut cmd = Command::new("productbuild");
1485             cmd.arg("--distribution")
1486                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1487                 .arg("--resources")
1488                 .arg(pkg.join("res"))
1489                 .arg(distdir(builder).join(format!(
1490                     "{}-{}.pkg",
1491                     pkgname(builder, "rust"),
1492                     target.triple
1493                 )))
1494                 .arg("--package-path")
1495                 .arg(&pkg);
1496             let _time = timeit(builder);
1497             builder.run(&mut cmd);
1498         }
1499
1500         if target.contains("windows") {
1501             let exe = tmp.join("exe");
1502             let _ = fs::remove_dir_all(&exe);
1503
1504             let prepare = |name: &str| {
1505                 builder.create_dir(&exe.join(name));
1506                 let dir = if name == "rust-std" || name == "rust-analysis" {
1507                     format!("{}-{}", name, target.triple)
1508                 } else if name == "rust-analyzer" {
1509                     "rust-analyzer-preview".to_string()
1510                 } else if name == "clippy" {
1511                     "clippy-preview".to_string()
1512                 } else if name == "rust-demangler" {
1513                     "rust-demangler-preview".to_string()
1514                 } else if name == "miri" {
1515                     "miri-preview".to_string()
1516                 } else {
1517                     name.to_string()
1518                 };
1519                 builder.cp_r(
1520                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1521                     &exe.join(name),
1522                 );
1523                 builder.remove(&exe.join(name).join("manifest.in"));
1524             };
1525             prepare("rustc");
1526             prepare("cargo");
1527             prepare("rust-analysis");
1528             prepare("rust-docs");
1529             prepare("rust-std");
1530             prepare("clippy");
1531             prepare("miri");
1532             prepare("rust-analyzer");
1533             for tool in &["rust-demangler"] {
1534                 if built_tools.contains(tool) {
1535                     prepare(tool);
1536                 }
1537             }
1538             if target.ends_with("windows-gnu") {
1539                 prepare("rust-mingw");
1540             }
1541
1542             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1543
1544             // Generate msi installer
1545             let wix_path = env::var_os("WIX")
1546                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1547             let wix = PathBuf::from(wix_path);
1548             let heat = wix.join("bin/heat.exe");
1549             let candle = wix.join("bin/candle.exe");
1550             let light = wix.join("bin/light.exe");
1551
1552             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1553             builder.run(
1554                 Command::new(&heat)
1555                     .current_dir(&exe)
1556                     .arg("dir")
1557                     .arg("rustc")
1558                     .args(&heat_flags)
1559                     .arg("-cg")
1560                     .arg("RustcGroup")
1561                     .arg("-dr")
1562                     .arg("Rustc")
1563                     .arg("-var")
1564                     .arg("var.RustcDir")
1565                     .arg("-out")
1566                     .arg(exe.join("RustcGroup.wxs")),
1567             );
1568             builder.run(
1569                 Command::new(&heat)
1570                     .current_dir(&exe)
1571                     .arg("dir")
1572                     .arg("rust-docs")
1573                     .args(&heat_flags)
1574                     .arg("-cg")
1575                     .arg("DocsGroup")
1576                     .arg("-dr")
1577                     .arg("Docs")
1578                     .arg("-var")
1579                     .arg("var.DocsDir")
1580                     .arg("-out")
1581                     .arg(exe.join("DocsGroup.wxs"))
1582                     .arg("-t")
1583                     .arg(etc.join("msi/squash-components.xsl")),
1584             );
1585             builder.run(
1586                 Command::new(&heat)
1587                     .current_dir(&exe)
1588                     .arg("dir")
1589                     .arg("cargo")
1590                     .args(&heat_flags)
1591                     .arg("-cg")
1592                     .arg("CargoGroup")
1593                     .arg("-dr")
1594                     .arg("Cargo")
1595                     .arg("-var")
1596                     .arg("var.CargoDir")
1597                     .arg("-out")
1598                     .arg(exe.join("CargoGroup.wxs"))
1599                     .arg("-t")
1600                     .arg(etc.join("msi/remove-duplicates.xsl")),
1601             );
1602             builder.run(
1603                 Command::new(&heat)
1604                     .current_dir(&exe)
1605                     .arg("dir")
1606                     .arg("rust-std")
1607                     .args(&heat_flags)
1608                     .arg("-cg")
1609                     .arg("StdGroup")
1610                     .arg("-dr")
1611                     .arg("Std")
1612                     .arg("-var")
1613                     .arg("var.StdDir")
1614                     .arg("-out")
1615                     .arg(exe.join("StdGroup.wxs")),
1616             );
1617             builder.run(
1618                 Command::new(&heat)
1619                     .current_dir(&exe)
1620                     .arg("dir")
1621                     .arg("rust-analyzer")
1622                     .args(&heat_flags)
1623                     .arg("-cg")
1624                     .arg("RustAnalyzerGroup")
1625                     .arg("-dr")
1626                     .arg("RustAnalyzer")
1627                     .arg("-var")
1628                     .arg("var.RustAnalyzerDir")
1629                     .arg("-out")
1630                     .arg(exe.join("RustAnalyzerGroup.wxs"))
1631                     .arg("-t")
1632                     .arg(etc.join("msi/remove-duplicates.xsl")),
1633             );
1634             builder.run(
1635                 Command::new(&heat)
1636                     .current_dir(&exe)
1637                     .arg("dir")
1638                     .arg("clippy")
1639                     .args(&heat_flags)
1640                     .arg("-cg")
1641                     .arg("ClippyGroup")
1642                     .arg("-dr")
1643                     .arg("Clippy")
1644                     .arg("-var")
1645                     .arg("var.ClippyDir")
1646                     .arg("-out")
1647                     .arg(exe.join("ClippyGroup.wxs"))
1648                     .arg("-t")
1649                     .arg(etc.join("msi/remove-duplicates.xsl")),
1650             );
1651             if built_tools.contains("rust-demangler") {
1652                 builder.run(
1653                     Command::new(&heat)
1654                         .current_dir(&exe)
1655                         .arg("dir")
1656                         .arg("rust-demangler")
1657                         .args(&heat_flags)
1658                         .arg("-cg")
1659                         .arg("RustDemanglerGroup")
1660                         .arg("-dr")
1661                         .arg("RustDemangler")
1662                         .arg("-var")
1663                         .arg("var.RustDemanglerDir")
1664                         .arg("-out")
1665                         .arg(exe.join("RustDemanglerGroup.wxs"))
1666                         .arg("-t")
1667                         .arg(etc.join("msi/remove-duplicates.xsl")),
1668                 );
1669             }
1670             builder.run(
1671                 Command::new(&heat)
1672                     .current_dir(&exe)
1673                     .arg("dir")
1674                     .arg("miri")
1675                     .args(&heat_flags)
1676                     .arg("-cg")
1677                     .arg("MiriGroup")
1678                     .arg("-dr")
1679                     .arg("Miri")
1680                     .arg("-var")
1681                     .arg("var.MiriDir")
1682                     .arg("-out")
1683                     .arg(exe.join("MiriGroup.wxs"))
1684                     .arg("-t")
1685                     .arg(etc.join("msi/remove-duplicates.xsl")),
1686             );
1687             builder.run(
1688                 Command::new(&heat)
1689                     .current_dir(&exe)
1690                     .arg("dir")
1691                     .arg("rust-analysis")
1692                     .args(&heat_flags)
1693                     .arg("-cg")
1694                     .arg("AnalysisGroup")
1695                     .arg("-dr")
1696                     .arg("Analysis")
1697                     .arg("-var")
1698                     .arg("var.AnalysisDir")
1699                     .arg("-out")
1700                     .arg(exe.join("AnalysisGroup.wxs"))
1701                     .arg("-t")
1702                     .arg(etc.join("msi/remove-duplicates.xsl")),
1703             );
1704             if target.ends_with("windows-gnu") {
1705                 builder.run(
1706                     Command::new(&heat)
1707                         .current_dir(&exe)
1708                         .arg("dir")
1709                         .arg("rust-mingw")
1710                         .args(&heat_flags)
1711                         .arg("-cg")
1712                         .arg("GccGroup")
1713                         .arg("-dr")
1714                         .arg("Gcc")
1715                         .arg("-var")
1716                         .arg("var.GccDir")
1717                         .arg("-out")
1718                         .arg(exe.join("GccGroup.wxs")),
1719                 );
1720             }
1721
1722             let candle = |input: &Path| {
1723                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1724                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1725                 let mut cmd = Command::new(&candle);
1726                 cmd.current_dir(&exe)
1727                     .arg("-nologo")
1728                     .arg("-dRustcDir=rustc")
1729                     .arg("-dDocsDir=rust-docs")
1730                     .arg("-dCargoDir=cargo")
1731                     .arg("-dStdDir=rust-std")
1732                     .arg("-dAnalysisDir=rust-analysis")
1733                     .arg("-dClippyDir=clippy")
1734                     .arg("-dMiriDir=miri")
1735                     .arg("-arch")
1736                     .arg(&arch)
1737                     .arg("-out")
1738                     .arg(&output)
1739                     .arg(&input);
1740                 add_env(builder, &mut cmd, target);
1741
1742                 if built_tools.contains("rust-demangler") {
1743                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1744                 }
1745                 if built_tools.contains("rust-analyzer") {
1746                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1747                 }
1748                 if target.ends_with("windows-gnu") {
1749                     cmd.arg("-dGccDir=rust-mingw");
1750                 }
1751                 builder.run(&mut cmd);
1752             };
1753             candle(&xform(&etc.join("msi/rust.wxs")));
1754             candle(&etc.join("msi/ui.wxs"));
1755             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1756             candle("RustcGroup.wxs".as_ref());
1757             candle("DocsGroup.wxs".as_ref());
1758             candle("CargoGroup.wxs".as_ref());
1759             candle("StdGroup.wxs".as_ref());
1760             candle("ClippyGroup.wxs".as_ref());
1761             candle("MiriGroup.wxs".as_ref());
1762             if built_tools.contains("rust-demangler") {
1763                 candle("RustDemanglerGroup.wxs".as_ref());
1764             }
1765             if built_tools.contains("rust-analyzer") {
1766                 candle("RustAnalyzerGroup.wxs".as_ref());
1767             }
1768             candle("AnalysisGroup.wxs".as_ref());
1769
1770             if target.ends_with("windows-gnu") {
1771                 candle("GccGroup.wxs".as_ref());
1772             }
1773
1774             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1775             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1776             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1777
1778             builder.info(&format!("building `msi` installer with {:?}", light));
1779             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1780             let mut cmd = Command::new(&light);
1781             cmd.arg("-nologo")
1782                 .arg("-ext")
1783                 .arg("WixUIExtension")
1784                 .arg("-ext")
1785                 .arg("WixUtilExtension")
1786                 .arg("-out")
1787                 .arg(exe.join(&filename))
1788                 .arg("rust.wixobj")
1789                 .arg("ui.wixobj")
1790                 .arg("rustwelcomedlg.wixobj")
1791                 .arg("RustcGroup.wixobj")
1792                 .arg("DocsGroup.wixobj")
1793                 .arg("CargoGroup.wixobj")
1794                 .arg("StdGroup.wixobj")
1795                 .arg("AnalysisGroup.wixobj")
1796                 .arg("ClippyGroup.wixobj")
1797                 .arg("MiriGroup.wixobj")
1798                 .current_dir(&exe);
1799
1800             if built_tools.contains("rust-analyzer") {
1801                 cmd.arg("RustAnalyzerGroup.wixobj");
1802             }
1803             if built_tools.contains("rust-demangler") {
1804                 cmd.arg("RustDemanglerGroup.wixobj");
1805             }
1806
1807             if target.ends_with("windows-gnu") {
1808                 cmd.arg("GccGroup.wixobj");
1809             }
1810             // ICE57 wrongly complains about the shortcuts
1811             cmd.arg("-sice:ICE57");
1812
1813             let _time = timeit(builder);
1814             builder.run(&mut cmd);
1815
1816             if !builder.config.dry_run {
1817                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1818             }
1819         }
1820     }
1821 }
1822
1823 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1824     let mut parts = builder.version.split('.');
1825     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1826         .env("CFG_RELEASE_NUM", &builder.version)
1827         .env("CFG_RELEASE", builder.rust_release())
1828         .env("CFG_VER_MAJOR", parts.next().unwrap())
1829         .env("CFG_VER_MINOR", parts.next().unwrap())
1830         .env("CFG_VER_PATCH", parts.next().unwrap())
1831         .env("CFG_VER_BUILD", "0") // just needed to build
1832         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1833         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1834         .env("CFG_BUILD", target.triple)
1835         .env("CFG_CHANNEL", &builder.config.channel);
1836
1837     if target.contains("windows-gnullvm") {
1838         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1839     } else if target.contains("windows-gnu") {
1840         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1841     } else {
1842         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1843     }
1844 }
1845
1846 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1847 ///
1848 /// Returns whether the files were actually copied.
1849 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1850     if !builder.is_rust_llvm(target) {
1851         // If the LLVM was externally provided, then we don't currently copy
1852         // artifacts into the sysroot. This is not necessarily the right
1853         // choice (in particular, it will require the LLVM dylib to be in
1854         // the linker's load path at runtime), but the common use case for
1855         // external LLVMs is distribution provided LLVMs, and in that case
1856         // they're usually in the standard search path (e.g., /usr/lib) and
1857         // copying them here is going to cause problems as we may end up
1858         // with the wrong files and isn't what distributions want.
1859         //
1860         // This behavior may be revisited in the future though.
1861         //
1862         // If the LLVM is coming from ourselves (just from CI) though, we
1863         // still want to install it, as it otherwise won't be available.
1864         return false;
1865     }
1866
1867     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1868     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1869     // clear why this is the case, though. llvm-config will emit the versioned
1870     // paths and we don't want those in the sysroot (as we're expecting
1871     // unversioned paths).
1872     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1873         let src_libdir = builder.llvm_out(target).join("lib");
1874         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1875         if llvm_dylib_path.exists() {
1876             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1877         }
1878         !builder.config.dry_run
1879     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1880         let mut cmd = Command::new(llvm_config);
1881         cmd.arg("--libfiles");
1882         builder.verbose(&format!("running {:?}", cmd));
1883         let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) };
1884         let build_llvm_out = &builder.llvm_out(builder.config.build);
1885         let target_llvm_out = &builder.llvm_out(target);
1886         for file in files.trim_end().split(' ') {
1887             // If we're not using a custom LLVM, make sure we package for the target.
1888             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1889                 target_llvm_out.join(relative_path)
1890             } else {
1891                 PathBuf::from(file)
1892             };
1893             builder.install(&file, dst_libdir, 0o644);
1894         }
1895         !builder.config.dry_run
1896     } else {
1897         false
1898     }
1899 }
1900
1901 /// Maybe add libLLVM.so to the target lib-dir for linking.
1902 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1903     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1904     // We do not need to copy LLVM files into the sysroot if it is not
1905     // dynamically linked; it is already included into librustc_llvm
1906     // statically.
1907     if builder.llvm_link_shared() {
1908         maybe_install_llvm(builder, target, &dst_libdir);
1909     }
1910 }
1911
1912 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1913 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1914     let dst_libdir =
1915         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1916     // We do not need to copy LLVM files into the sysroot if it is not
1917     // dynamically linked; it is already included into librustc_llvm
1918     // statically.
1919     if builder.llvm_link_shared() {
1920         maybe_install_llvm(builder, target, &dst_libdir);
1921     }
1922 }
1923
1924 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1925 pub struct LlvmTools {
1926     pub target: TargetSelection,
1927 }
1928
1929 impl Step for LlvmTools {
1930     type Output = Option<GeneratedTarball>;
1931     const ONLY_HOSTS: bool = true;
1932     const DEFAULT: bool = true;
1933
1934     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1935         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1936         // FIXME: allow using the names of the tools themselves?
1937         run.alias("llvm-tools").default_condition(default)
1938     }
1939
1940     fn make_run(run: RunConfig<'_>) {
1941         run.builder.ensure(LlvmTools { target: run.target });
1942     }
1943
1944     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1945         let target = self.target;
1946
1947         /* run only if llvm-config isn't used */
1948         if let Some(config) = builder.config.target_config.get(&target) {
1949             if let Some(ref _s) = config.llvm_config {
1950                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1951                 return None;
1952             }
1953         }
1954
1955         builder.ensure(crate::native::Llvm { target });
1956
1957         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1958         tarball.set_overlay(OverlayKind::LLVM);
1959         tarball.is_preview(true);
1960
1961         // Prepare the image directory
1962         let src_bindir = builder.llvm_out(target).join("bin");
1963         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1964         for tool in LLVM_TOOLS {
1965             let exe = src_bindir.join(exe(tool, target));
1966             tarball.add_file(&exe, &dst_bindir, 0o755);
1967         }
1968
1969         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1970         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1971         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1972         // compiler libraries.
1973         maybe_install_llvm_target(builder, target, tarball.image_dir());
1974
1975         Some(tarball.generate())
1976     }
1977 }
1978
1979 // Tarball intended for internal consumption to ease rustc/std development.
1980 //
1981 // Should not be considered stable by end users.
1982 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1983 pub struct RustDev {
1984     pub target: TargetSelection,
1985 }
1986
1987 impl Step for RustDev {
1988     type Output = Option<GeneratedTarball>;
1989     const DEFAULT: bool = true;
1990     const ONLY_HOSTS: bool = true;
1991
1992     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1993         run.alias("rust-dev")
1994     }
1995
1996     fn make_run(run: RunConfig<'_>) {
1997         run.builder.ensure(RustDev { target: run.target });
1998     }
1999
2000     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2001         let target = self.target;
2002
2003         /* run only if llvm-config isn't used */
2004         if let Some(config) = builder.config.target_config.get(&target) {
2005             if let Some(ref _s) = config.llvm_config {
2006                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2007                 return None;
2008             }
2009         }
2010
2011         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2012         tarball.set_overlay(OverlayKind::LLVM);
2013
2014         builder.ensure(crate::native::Llvm { target });
2015
2016         let src_bindir = builder.llvm_out(target).join("bin");
2017         // If updating this list, you likely want to change
2018         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2019         // will not pick up the extra file until LLVM gets bumped.
2020         for bin in &[
2021             "llvm-config",
2022             "llvm-ar",
2023             "llvm-objdump",
2024             "llvm-profdata",
2025             "llvm-bcanalyzer",
2026             "llvm-cov",
2027             "llvm-dwp",
2028             "llvm-nm",
2029             "llvm-dwarfdump",
2030             "llvm-dis",
2031             "llvm-tblgen",
2032         ] {
2033             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2034         }
2035
2036         // We don't build LLD on some platforms, so only add it if it exists
2037         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2038         if lld_path.exists() {
2039             tarball.add_file(lld_path, "bin", 0o755);
2040         }
2041
2042         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2043
2044         // Copy the include directory as well; needed mostly to build
2045         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2046         // just broadly useful to be able to link against the bundled LLVM.
2047         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2048
2049         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2050         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2051         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2052         // compiler libraries.
2053         let dst_libdir = tarball.image_dir().join("lib");
2054         maybe_install_llvm(builder, target, &dst_libdir);
2055         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2056         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2057
2058         Some(tarball.generate())
2059     }
2060 }
2061
2062 // Tarball intended for internal consumption to ease rustc/std development.
2063 //
2064 // Should not be considered stable by end users.
2065 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2066 pub struct Bootstrap {
2067     pub target: TargetSelection,
2068 }
2069
2070 impl Step for Bootstrap {
2071     type Output = Option<GeneratedTarball>;
2072     const DEFAULT: bool = false;
2073     const ONLY_HOSTS: bool = true;
2074
2075     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2076         run.alias("bootstrap")
2077     }
2078
2079     fn make_run(run: RunConfig<'_>) {
2080         run.builder.ensure(Bootstrap { target: run.target });
2081     }
2082
2083     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2084         let target = self.target;
2085
2086         let tarball = Tarball::new(builder, "bootstrap", &target.triple);
2087
2088         let bootstrap_outdir = &builder.bootstrap_out;
2089         for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
2090             tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
2091         }
2092
2093         Some(tarball.generate())
2094     }
2095 }
2096
2097 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2098 /// release process to avoid cloning the monorepo and building stuff.
2099 ///
2100 /// Should not be considered stable by end users.
2101 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2102 pub struct BuildManifest {
2103     pub target: TargetSelection,
2104 }
2105
2106 impl Step for BuildManifest {
2107     type Output = GeneratedTarball;
2108     const DEFAULT: bool = false;
2109     const ONLY_HOSTS: bool = true;
2110
2111     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2112         run.alias("build-manifest")
2113     }
2114
2115     fn make_run(run: RunConfig<'_>) {
2116         run.builder.ensure(BuildManifest { target: run.target });
2117     }
2118
2119     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2120         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2121
2122         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2123         tarball.add_file(&build_manifest, "bin", 0o755);
2124         tarball.generate()
2125     }
2126 }
2127
2128 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2129 ///
2130 /// Currently this is the PGO profile data.
2131 ///
2132 /// Should not be considered stable by end users.
2133 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2134 pub struct ReproducibleArtifacts {
2135     pub target: TargetSelection,
2136 }
2137
2138 impl Step for ReproducibleArtifacts {
2139     type Output = Option<GeneratedTarball>;
2140     const DEFAULT: bool = true;
2141     const ONLY_HOSTS: bool = true;
2142
2143     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2144         run.alias("reproducible-artifacts")
2145     }
2146
2147     fn make_run(run: RunConfig<'_>) {
2148         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2149     }
2150
2151     fn run(self, builder: &Builder<'_>) -> Self::Output {
2152         let mut added_anything = false;
2153         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2154         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2155             tarball.add_file(path, ".", 0o644);
2156             added_anything = true;
2157         }
2158         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2159             tarball.add_file(path, ".", 0o644);
2160             added_anything = true;
2161         }
2162         if let Some(path) = builder.config.llvm_bolt_profile_use.as_ref() {
2163             tarball.add_file(path, ".", 0o644);
2164             added_anything = true;
2165         }
2166         if added_anything { Some(tarball.generate()) } else { None }
2167     }
2168 }