src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let exe = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 405                 // for `-Z gcc-ld=lld`
 406                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 407                 t!(fs::create_dir(&gcc_lld_dir));
 408                 builder.copy(&src_dir.join(&exe), &gcc_lld_dir.join(&exe));
 409             }
 410
 411             // Copy over llvm-dwp if it's there
 412             let exe = exe("rust-llvm-dwp", compiler.host);
 413             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 414
 415             // Man pages
 416             t!(fs::create_dir_all(image.join("share/man/man1")));
 417             let man_src = builder.src.join("src/doc/man");
 418             let man_dst = image.join("share/man/man1");
 419
 420             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 421             let time = env::var("SOURCE_DATE_EPOCH")
 422                 .map(|timestamp| {
 423                     let epoch = timestamp
 424                         .parse()
 425                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 426                         .unwrap();
 427
 428                     time::at(Timespec::new(epoch, 0))
 429                 })
 430                 .unwrap_or_else(|_| time::now());
 431
 432             let month_year = t!(time::strftime("%B %Y", &time));
 433             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 434             // to hardlink, and we don't want to edit the source templates
 435             for file_entry in builder.read_dir(&man_src) {
 436                 let page_src = file_entry.path();
 437                 let page_dst = man_dst.join(file_entry.file_name());
 438                 t!(fs::copy(&page_src, &page_dst));
 439                 // template in month/year and version number
 440                 builder.replace_in_file(
 441                     &page_dst,
 442                     &[
 443                         ("<INSERT DATE HERE>", &month_year),
 444                         ("<INSERT VERSION HERE>", &builder.version),
 445                     ],
 446                 );
 447             }
 448
 449             // Debugger scripts
 450             builder
 451                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 452
 453             // Misc license info
 454             let cp = |file: &str| {
 455                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 456             };
 457             cp("COPYRIGHT");
 458             cp("LICENSE-APACHE");
 459             cp("LICENSE-MIT");
 460             cp("README.md");
 461         }
 462     }
 463 }
 464
 465 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 466 pub struct DebuggerScripts {
 467     pub sysroot: Interned<PathBuf>,
 468     pub host: TargetSelection,
 469 }
 470
 471 impl Step for DebuggerScripts {
 472     type Output = ();
 473
 474     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 475         run.path("src/lldb_batchmode.py")
 476     }
 477
 478     fn make_run(run: RunConfig<'_>) {
 479         run.builder.ensure(DebuggerScripts {
 480             sysroot: run
 481                 .builder
 482                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 483             host: run.target,
 484         });
 485     }
 486
 487     /// Copies debugger scripts for `target` into the `sysroot` specified.
 488     fn run(self, builder: &Builder<'_>) {
 489         let host = self.host;
 490         let sysroot = self.sysroot;
 491         let dst = sysroot.join("lib/rustlib/etc");
 492         t!(fs::create_dir_all(&dst));
 493         let cp_debugger_script = |file: &str| {
 494             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 495         };
 496         if host.contains("windows-msvc") {
 497             // windbg debugger scripts
 498             builder.install(
 499                 &builder.src.join("src/etc/rust-windbg.cmd"),
 500                 &sysroot.join("bin"),
 501                 0o755,
 502             );
 503
 504             cp_debugger_script("natvis/intrinsic.natvis");
 505             cp_debugger_script("natvis/liballoc.natvis");
 506             cp_debugger_script("natvis/libcore.natvis");
 507             cp_debugger_script("natvis/libstd.natvis");
 508         } else {
 509             cp_debugger_script("rust_types.py");
 510
 511             // gdb debugger scripts
 512             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 513             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 514
 515             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 516             cp_debugger_script("gdb_lookup.py");
 517             cp_debugger_script("gdb_providers.py");
 518
 519             // lldb debugger scripts
 520             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 521
 522             cp_debugger_script("lldb_lookup.py");
 523             cp_debugger_script("lldb_providers.py");
 524             cp_debugger_script("lldb_commands")
 525         }
 526     }
 527 }
 528
 529 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 530     // The only true set of target libraries came from the build triple, so
 531     // let's reduce redundant work by only producing archives from that host.
 532     if compiler.host != builder.config.build {
 533         builder.info("\tskipping, not a build host");
 534         true
 535     } else {
 536         false
 537     }
 538 }
 539
 540 /// Copy stamped files into an image's `target/lib` directory.
 541 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 542     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 543     let self_contained_dst = dst.join("self-contained");
 544     t!(fs::create_dir_all(&dst));
 545     t!(fs::create_dir_all(&self_contained_dst));
 546     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 547         if dependency_type == DependencyType::TargetSelfContained {
 548             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 549         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 550             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: TargetSelection,
 559 }
 560
 561 impl Step for Std {
 562     type Output = Option<GeneratedTarball>;
 563     const DEFAULT: bool = true;
 564
 565     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 566         run.path("library/std")
 567     }
 568
 569     fn make_run(run: RunConfig<'_>) {
 570         run.builder.ensure(Std {
 571             compiler: run.builder.compiler_for(
 572                 run.builder.top_stage,
 573                 run.builder.config.build,
 574                 run.target,
 575             ),
 576             target: run.target,
 577         });
 578     }
 579
 580     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 581         let compiler = self.compiler;
 582         let target = self.target;
 583
 584         if skip_host_target_lib(builder, compiler) {
 585             return None;
 586         }
 587
 588         builder.ensure(compile::Std { compiler, target });
 589
 590         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 591         tarball.include_target_in_component_name(true);
 592
 593         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 594         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 595         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 596
 597         Some(tarball.generate())
 598     }
 599 }
 600
 601 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 602 pub struct RustcDev {
 603     pub compiler: Compiler,
 604     pub target: TargetSelection,
 605 }
 606
 607 impl Step for RustcDev {
 608     type Output = Option<GeneratedTarball>;
 609     const DEFAULT: bool = true;
 610     const ONLY_HOSTS: bool = true;
 611
 612     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 613         run.path("rustc-dev")
 614     }
 615
 616     fn make_run(run: RunConfig<'_>) {
 617         run.builder.ensure(RustcDev {
 618             compiler: run.builder.compiler_for(
 619                 run.builder.top_stage,
 620                 run.builder.config.build,
 621                 run.target,
 622             ),
 623             target: run.target,
 624         });
 625     }
 626
 627     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 628         let compiler = self.compiler;
 629         let target = self.target;
 630         if skip_host_target_lib(builder, compiler) {
 631             return None;
 632         }
 633
 634         builder.ensure(compile::Rustc { compiler, target });
 635
 636         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 637
 638         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 639         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 640         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 641
 642         let src_files = &["Cargo.lock"];
 643         // This is the reduced set of paths which will become the rustc-dev component
 644         // (essentially the compiler crates and all of their path dependencies).
 645         copy_src_dirs(
 646             builder,
 647             &builder.src,
 648             &["compiler"],
 649             &[],
 650             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 651         );
 652         // This particular crate is used as a build dependency of the above.
 653         copy_src_dirs(
 654             builder,
 655             &builder.src,
 656             &["src/build_helper"],
 657             &[],
 658             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 659         );
 660         for file in src_files {
 661             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 662         }
 663
 664         Some(tarball.generate())
 665     }
 666 }
 667
 668 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 669 pub struct Analysis {
 670     pub compiler: Compiler,
 671     pub target: TargetSelection,
 672 }
 673
 674 impl Step for Analysis {
 675     type Output = Option<GeneratedTarball>;
 676     const DEFAULT: bool = true;
 677
 678     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 679         let builder = run.builder;
 680         run.path("analysis").default_condition(builder.config.extended)
 681     }
 682
 683     fn make_run(run: RunConfig<'_>) {
 684         run.builder.ensure(Analysis {
 685             // Find the actual compiler (handling the full bootstrap option) which
 686             // produced the save-analysis data because that data isn't copied
 687             // through the sysroot uplifting.
 688             compiler: run.builder.compiler_for(
 689                 run.builder.top_stage,
 690                 run.builder.config.build,
 691                 run.target,
 692             ),
 693             target: run.target,
 694         });
 695     }
 696
 697     /// Creates a tarball of save-analysis metadata, if available.
 698     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 699         let compiler = self.compiler;
 700         let target = self.target;
 701         assert!(builder.config.extended);
 702         if compiler.host != builder.config.build {
 703             return None;
 704         }
 705
 706         builder.ensure(compile::Std { compiler, target });
 707         let src = builder
 708             .stage_out(compiler, Mode::Std)
 709             .join(target.triple)
 710             .join(builder.cargo_dir())
 711             .join("deps")
 712             .join("save-analysis");
 713
 714         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 715         tarball.include_target_in_component_name(true);
 716         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 717         Some(tarball.generate())
 718     }
 719 }
 720
 721 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 722 /// `dst_dir`.
 723 fn copy_src_dirs(
 724     builder: &Builder<'_>,
 725     base: &Path,
 726     src_dirs: &[&str],
 727     exclude_dirs: &[&str],
 728     dst_dir: &Path,
 729 ) {
 730     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 731         let spath = match path.to_str() {
 732             Some(path) => path,
 733             None => return false,
 734         };
 735         if spath.ends_with('~') || spath.ends_with(".pyc") {
 736             return false;
 737         }
 738
 739         const LLVM_PROJECTS: &[&str] = &[
 740             "llvm-project/clang",
 741             "llvm-project\\clang",
 742             "llvm-project/libunwind",
 743             "llvm-project\\libunwind",
 744             "llvm-project/lld",
 745             "llvm-project\\lld",
 746             "llvm-project/lldb",
 747             "llvm-project\\lldb",
 748             "llvm-project/llvm",
 749             "llvm-project\\llvm",
 750             "llvm-project/compiler-rt",
 751             "llvm-project\\compiler-rt",
 752         ];
 753         if spath.contains("llvm-project")
 754             && !spath.ends_with("llvm-project")
 755             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 756         {
 757             return false;
 758         }
 759
 760         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 761         if LLVM_TEST.iter().any(|path| spath.contains(path))
 762             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 763         {
 764             return false;
 765         }
 766
 767         let full_path = Path::new(dir).join(path);
 768         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 769             return false;
 770         }
 771
 772         let excludes = [
 773             "CVS",
 774             "RCS",
 775             "SCCS",
 776             ".git",
 777             ".gitignore",
 778             ".gitmodules",
 779             ".gitattributes",
 780             ".cvsignore",
 781             ".svn",
 782             ".arch-ids",
 783             "{arch}",
 784             "=RELEASE-ID",
 785             "=meta-update",
 786             "=update",
 787             ".bzr",
 788             ".bzrignore",
 789             ".bzrtags",
 790             ".hg",
 791             ".hgignore",
 792             ".hgrags",
 793             "_darcs",
 794         ];
 795         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 796     }
 797
 798     // Copy the directories using our filter
 799     for item in src_dirs {
 800         let dst = &dst_dir.join(item);
 801         t!(fs::create_dir_all(dst));
 802         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 803     }
 804 }
 805
 806 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 807 pub struct Src;
 808
 809 impl Step for Src {
 810     /// The output path of the src installer tarball
 811     type Output = GeneratedTarball;
 812     const DEFAULT: bool = true;
 813     const ONLY_HOSTS: bool = true;
 814
 815     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 816         run.path("src")
 817     }
 818
 819     fn make_run(run: RunConfig<'_>) {
 820         run.builder.ensure(Src);
 821     }
 822
 823     /// Creates the `rust-src` installer component
 824     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 825         let tarball = Tarball::new_targetless(builder, "rust-src");
 826
 827         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 828         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 829         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 830         // and fix them...
 831         //
 832         // NOTE: if you update the paths here, you also should update the "virtual" path
 833         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 834         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 835
 836         let src_files = ["Cargo.lock"];
 837         // This is the reduced set of paths which will become the rust-src component
 838         // (essentially libstd and all of its path dependencies).
 839         copy_src_dirs(
 840             builder,
 841             &builder.src,
 842             &["library", "src/llvm-project/libunwind"],
 843             &[
 844                 // not needed and contains symlinks which rustup currently
 845                 // chokes on when unpacking.
 846                 "library/backtrace/crates",
 847             ],
 848             &dst_src,
 849         );
 850         for file in src_files.iter() {
 851             builder.copy(&builder.src.join(file), &dst_src.join(file));
 852         }
 853
 854         tarball.generate()
 855     }
 856 }
 857
 858 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 859 pub struct PlainSourceTarball;
 860
 861 impl Step for PlainSourceTarball {
 862     /// Produces the location of the tarball generated
 863     type Output = GeneratedTarball;
 864     const DEFAULT: bool = true;
 865     const ONLY_HOSTS: bool = true;
 866
 867     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 868         let builder = run.builder;
 869         run.path("src").default_condition(builder.config.rust_dist_src)
 870     }
 871
 872     fn make_run(run: RunConfig<'_>) {
 873         run.builder.ensure(PlainSourceTarball);
 874     }
 875
 876     /// Creates the plain source tarball
 877     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 878         let tarball = Tarball::new(builder, "rustc", "src");
 879         let plain_dst_src = tarball.image_dir();
 880
 881         // This is the set of root paths which will become part of the source package
 882         let src_files = [
 883             "COPYRIGHT",
 884             "LICENSE-APACHE",
 885             "LICENSE-MIT",
 886             "CONTRIBUTING.md",
 887             "README.md",
 888             "RELEASES.md",
 889             "configure",
 890             "x.py",
 891             "config.toml.example",
 892             "Cargo.toml",
 893             "Cargo.lock",
 894         ];
 895         let src_dirs = ["src", "compiler", "library"];
 896
 897         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 898
 899         // Copy the files normally
 900         for item in &src_files {
 901             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 902         }
 903
 904         // Create the version file
 905         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 906         if let Some(sha) = builder.rust_sha() {
 907             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 908         }
 909
 910         // If we're building from git sources, we need to vendor a complete distribution.
 911         if builder.rust_info.is_git() {
 912             // Vendor all Cargo dependencies
 913             let mut cmd = Command::new(&builder.initial_cargo);
 914             cmd.arg("vendor")
 915                 .arg("--sync")
 916                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 917                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 918                 .current_dir(&plain_dst_src);
 919             builder.run(&mut cmd);
 920         }
 921
 922         tarball.bare()
 923     }
 924 }
 925
 926 // We have to run a few shell scripts, which choke quite a bit on both `\`
 927 // characters and on `C:\` paths, so normalize both of them away.
 928 pub fn sanitize_sh(path: &Path) -> String {
 929     let path = path.to_str().unwrap().replace("\\", "/");
 930     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 931
 932     fn unc_to_lfs(s: &str) -> &str {
 933         s.strip_prefix("//?/").unwrap_or(s)
 934     }
 935
 936     fn change_drive(s: &str) -> Option<String> {
 937         let mut ch = s.chars();
 938         let drive = ch.next().unwrap_or('C');
 939         if ch.next() != Some(':') {
 940             return None;
 941         }
 942         if ch.next() != Some('/') {
 943             return None;
 944         }
 945         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 946     }
 947 }
 948
 949 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 950 pub struct Cargo {
 951     pub compiler: Compiler,
 952     pub target: TargetSelection,
 953 }
 954
 955 impl Step for Cargo {
 956     type Output = GeneratedTarball;
 957     const ONLY_HOSTS: bool = true;
 958
 959     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 960         run.path("cargo")
 961     }
 962
 963     fn make_run(run: RunConfig<'_>) {
 964         run.builder.ensure(Cargo {
 965             compiler: run.builder.compiler_for(
 966                 run.builder.top_stage,
 967                 run.builder.config.build,
 968                 run.target,
 969             ),
 970             target: run.target,
 971         });
 972     }
 973
 974     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 975         let compiler = self.compiler;
 976         let target = self.target;
 977
 978         let cargo = builder.ensure(tool::Cargo { compiler, target });
 979         let src = builder.src.join("src/tools/cargo");
 980         let etc = src.join("src/etc");
 981
 982         // Prepare the image directory
 983         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 984         tarball.set_overlay(OverlayKind::Cargo);
 985
 986         tarball.add_file(&cargo, "bin", 0o755);
 987         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 988         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 989         tarball.add_dir(etc.join("man"), "share/man/man1");
 990         tarball.add_legal_and_readme_to("share/doc/cargo");
 991
 992         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 993             let dirent = dirent.expect("read dir entry");
 994             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 995                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 996             }
 997         }
 998
 999         tarball.generate()
1000     }
1001 }
1002
1003 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1004 pub struct Rls {
1005     pub compiler: Compiler,
1006     pub target: TargetSelection,
1007 }
1008
1009 impl Step for Rls {
1010     type Output = Option<GeneratedTarball>;
1011     const ONLY_HOSTS: bool = true;
1012
1013     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1014         run.path("rls")
1015     }
1016
1017     fn make_run(run: RunConfig<'_>) {
1018         run.builder.ensure(Rls {
1019             compiler: run.builder.compiler_for(
1020                 run.builder.top_stage,
1021                 run.builder.config.build,
1022                 run.target,
1023             ),
1024             target: run.target,
1025         });
1026     }
1027
1028     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1029         let compiler = self.compiler;
1030         let target = self.target;
1031         assert!(builder.config.extended);
1032
1033         let rls = builder
1034             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1035             .or_else(|| {
1036                 missing_tool("RLS", builder.build.config.missing_tools);
1037                 None
1038             })?;
1039
1040         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1041         tarball.set_overlay(OverlayKind::RLS);
1042         tarball.is_preview(true);
1043         tarball.add_file(rls, "bin", 0o755);
1044         tarball.add_legal_and_readme_to("share/doc/rls");
1045         Some(tarball.generate())
1046     }
1047 }
1048
1049 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1050 pub struct RustAnalyzer {
1051     pub compiler: Compiler,
1052     pub target: TargetSelection,
1053 }
1054
1055 impl Step for RustAnalyzer {
1056     type Output = Option<GeneratedTarball>;
1057     const ONLY_HOSTS: bool = true;
1058
1059     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1060         run.path("rust-analyzer")
1061     }
1062
1063     fn make_run(run: RunConfig<'_>) {
1064         run.builder.ensure(RustAnalyzer {
1065             compiler: run.builder.compiler_for(
1066                 run.builder.top_stage,
1067                 run.builder.config.build,
1068                 run.target,
1069             ),
1070             target: run.target,
1071         });
1072     }
1073
1074     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1075         let compiler = self.compiler;
1076         let target = self.target;
1077         assert!(builder.config.extended);
1078
1079         if target.contains("riscv64") {
1080             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1081             // to build. See #74813 for details.
1082             return None;
1083         }
1084
1085         let rust_analyzer = builder
1086             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1087             .expect("rust-analyzer always builds");
1088
1089         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1090         tarball.set_overlay(OverlayKind::RustAnalyzer);
1091         tarball.is_preview(true);
1092         tarball.add_file(rust_analyzer, "bin", 0o755);
1093         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1094         Some(tarball.generate())
1095     }
1096 }
1097
1098 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1099 pub struct Clippy {
1100     pub compiler: Compiler,
1101     pub target: TargetSelection,
1102 }
1103
1104 impl Step for Clippy {
1105     type Output = GeneratedTarball;
1106     const ONLY_HOSTS: bool = true;
1107
1108     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1109         run.path("clippy")
1110     }
1111
1112     fn make_run(run: RunConfig<'_>) {
1113         run.builder.ensure(Clippy {
1114             compiler: run.builder.compiler_for(
1115                 run.builder.top_stage,
1116                 run.builder.config.build,
1117                 run.target,
1118             ),
1119             target: run.target,
1120         });
1121     }
1122
1123     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1124         let compiler = self.compiler;
1125         let target = self.target;
1126         assert!(builder.config.extended);
1127
1128         // Prepare the image directory
1129         // We expect clippy to build, because we've exited this step above if tool
1130         // state for clippy isn't testing.
1131         let clippy = builder
1132             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1133             .expect("clippy expected to build - essential tool");
1134         let cargoclippy = builder
1135             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1136             .expect("clippy expected to build - essential tool");
1137
1138         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1139         tarball.set_overlay(OverlayKind::Clippy);
1140         tarball.is_preview(true);
1141         tarball.add_file(clippy, "bin", 0o755);
1142         tarball.add_file(cargoclippy, "bin", 0o755);
1143         tarball.add_legal_and_readme_to("share/doc/clippy");
1144         tarball.generate()
1145     }
1146 }
1147
1148 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1149 pub struct Miri {
1150     pub compiler: Compiler,
1151     pub target: TargetSelection,
1152 }
1153
1154 impl Step for Miri {
1155     type Output = Option<GeneratedTarball>;
1156     const ONLY_HOSTS: bool = true;
1157
1158     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1159         run.path("miri")
1160     }
1161
1162     fn make_run(run: RunConfig<'_>) {
1163         run.builder.ensure(Miri {
1164             compiler: run.builder.compiler_for(
1165                 run.builder.top_stage,
1166                 run.builder.config.build,
1167                 run.target,
1168             ),
1169             target: run.target,
1170         });
1171     }
1172
1173     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1174         let compiler = self.compiler;
1175         let target = self.target;
1176         assert!(builder.config.extended);
1177
1178         let miri = builder
1179             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1180             .or_else(|| {
1181                 missing_tool("miri", builder.build.config.missing_tools);
1182                 None
1183             })?;
1184         let cargomiri = builder
1185             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1186             .or_else(|| {
1187                 missing_tool("cargo miri", builder.build.config.missing_tools);
1188                 None
1189             })?;
1190
1191         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1192         tarball.set_overlay(OverlayKind::Miri);
1193         tarball.is_preview(true);
1194         tarball.add_file(miri, "bin", 0o755);
1195         tarball.add_file(cargomiri, "bin", 0o755);
1196         tarball.add_legal_and_readme_to("share/doc/miri");
1197         Some(tarball.generate())
1198     }
1199 }
1200
1201 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1202 pub struct Rustfmt {
1203     pub compiler: Compiler,
1204     pub target: TargetSelection,
1205 }
1206
1207 impl Step for Rustfmt {
1208     type Output = Option<GeneratedTarball>;
1209     const ONLY_HOSTS: bool = true;
1210
1211     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1212         run.path("rustfmt")
1213     }
1214
1215     fn make_run(run: RunConfig<'_>) {
1216         run.builder.ensure(Rustfmt {
1217             compiler: run.builder.compiler_for(
1218                 run.builder.top_stage,
1219                 run.builder.config.build,
1220                 run.target,
1221             ),
1222             target: run.target,
1223         });
1224     }
1225
1226     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1227         let compiler = self.compiler;
1228         let target = self.target;
1229
1230         let rustfmt = builder
1231             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1232             .or_else(|| {
1233                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1234                 None
1235             })?;
1236         let cargofmt = builder
1237             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1238             .or_else(|| {
1239                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1240                 None
1241             })?;
1242
1243         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1244         tarball.set_overlay(OverlayKind::Rustfmt);
1245         tarball.is_preview(true);
1246         tarball.add_file(rustfmt, "bin", 0o755);
1247         tarball.add_file(cargofmt, "bin", 0o755);
1248         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1249         Some(tarball.generate())
1250     }
1251 }
1252
1253 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1254 pub struct RustDemangler {
1255     pub compiler: Compiler,
1256     pub target: TargetSelection,
1257 }
1258
1259 impl Step for RustDemangler {
1260     type Output = Option<GeneratedTarball>;
1261     const ONLY_HOSTS: bool = true;
1262
1263     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1264         run.path("rust-demangler")
1265     }
1266
1267     fn make_run(run: RunConfig<'_>) {
1268         run.builder.ensure(RustDemangler {
1269             compiler: run.builder.compiler_for(
1270                 run.builder.top_stage,
1271                 run.builder.config.build,
1272                 run.target,
1273             ),
1274             target: run.target,
1275         });
1276     }
1277
1278     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1279         let compiler = self.compiler;
1280         let target = self.target;
1281         assert!(builder.config.extended);
1282
1283         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1284         let profiler = builder.config.profiler_enabled(target);
1285         if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
1286             return None;
1287         }
1288
1289         let rust_demangler = builder
1290             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1291             .expect("rust-demangler expected to build - in-tree tool");
1292
1293         // Prepare the image directory
1294         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1295         tarball.set_overlay(OverlayKind::RustDemangler);
1296         tarball.is_preview(true);
1297         tarball.add_file(&rust_demangler, "bin", 0o755);
1298         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1299         Some(tarball.generate())
1300     }
1301 }
1302
1303 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1304 pub struct Extended {
1305     stage: u32,
1306     host: TargetSelection,
1307     target: TargetSelection,
1308 }
1309
1310 impl Step for Extended {
1311     type Output = ();
1312     const DEFAULT: bool = true;
1313     const ONLY_HOSTS: bool = true;
1314
1315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1316         let builder = run.builder;
1317         run.path("extended").default_condition(builder.config.extended)
1318     }
1319
1320     fn make_run(run: RunConfig<'_>) {
1321         run.builder.ensure(Extended {
1322             stage: run.builder.top_stage,
1323             host: run.builder.config.build,
1324             target: run.target,
1325         });
1326     }
1327
1328     /// Creates a combined installer for the specified target in the provided stage.
1329     fn run(self, builder: &Builder<'_>) {
1330         let target = self.target;
1331         let stage = self.stage;
1332         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1333
1334         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1335
1336         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1337         let cargo_installer = builder.ensure(Cargo { compiler, target });
1338         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1339         let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
1340         let rls_installer = builder.ensure(Rls { compiler, target });
1341         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1342         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1343         let clippy_installer = builder.ensure(Clippy { compiler, target });
1344         let miri_installer = builder.ensure(Miri { compiler, target });
1345         let mingw_installer = builder.ensure(Mingw { host: target });
1346         let analysis_installer = builder.ensure(Analysis { compiler, target });
1347
1348         let docs_installer = builder.ensure(Docs { host: target });
1349         let std_installer = builder.ensure(Std { compiler, target });
1350
1351         let etc = builder.src.join("src/etc/installer");
1352
1353         // Avoid producing tarballs during a dry run.
1354         if builder.config.dry_run {
1355             return;
1356         }
1357
1358         // When rust-std package split from rustc, we needed to ensure that during
1359         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1360         // the std files during uninstall. To do this ensure that rustc comes
1361         // before rust-std in the list below.
1362         let mut tarballs = Vec::new();
1363         tarballs.push(rustc_installer);
1364         tarballs.push(cargo_installer);
1365         tarballs.push(clippy_installer);
1366         tarballs.extend(rust_demangler_installer.clone());
1367         tarballs.extend(rls_installer.clone());
1368         tarballs.extend(rust_analyzer_installer.clone());
1369         tarballs.extend(miri_installer.clone());
1370         tarballs.extend(rustfmt_installer.clone());
1371         tarballs.extend(llvm_tools_installer);
1372         if let Some(analysis_installer) = analysis_installer {
1373             tarballs.push(analysis_installer);
1374         }
1375         tarballs.push(std_installer.expect("missing std"));
1376         if let Some(docs_installer) = docs_installer {
1377             tarballs.push(docs_installer);
1378         }
1379         if target.contains("pc-windows-gnu") {
1380             tarballs.push(mingw_installer.unwrap());
1381         }
1382
1383         let tarball = Tarball::new(builder, "rust", &target.triple);
1384         let generated = tarball.combine(&tarballs);
1385
1386         let tmp = tmpdir(builder).join("combined-tarball");
1387         let work = generated.work_dir();
1388
1389         let mut license = String::new();
1390         license += &builder.read(&builder.src.join("COPYRIGHT"));
1391         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1392         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1393         license.push('\n');
1394         license.push('\n');
1395
1396         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1397         let mut rtf = rtf.to_string();
1398         rtf.push('\n');
1399         for line in license.lines() {
1400             rtf.push_str(line);
1401             rtf.push_str("\\line ");
1402         }
1403         rtf.push('}');
1404
1405         fn filter(contents: &str, marker: &str) -> String {
1406             let start = format!("tool-{}-start", marker);
1407             let end = format!("tool-{}-end", marker);
1408             let mut lines = Vec::new();
1409             let mut omitted = false;
1410             for line in contents.lines() {
1411                 if line.contains(&start) {
1412                     omitted = true;
1413                 } else if line.contains(&end) {
1414                     omitted = false;
1415                 } else if !omitted {
1416                     lines.push(line);
1417                 }
1418             }
1419
1420             lines.join("\n")
1421         }
1422
1423         let xform = |p: &Path| {
1424             let mut contents = t!(fs::read_to_string(p));
1425             if rust_demangler_installer.is_none() {
1426                 contents = filter(&contents, "rust-demangler");
1427             }
1428             if rls_installer.is_none() {
1429                 contents = filter(&contents, "rls");
1430             }
1431             if rust_analyzer_installer.is_none() {
1432                 contents = filter(&contents, "rust-analyzer");
1433             }
1434             if miri_installer.is_none() {
1435                 contents = filter(&contents, "miri");
1436             }
1437             if rustfmt_installer.is_none() {
1438                 contents = filter(&contents, "rustfmt");
1439             }
1440             let ret = tmp.join(p.file_name().unwrap());
1441             t!(fs::write(&ret, &contents));
1442             ret
1443         };
1444
1445         if target.contains("apple-darwin") {
1446             builder.info("building pkg installer");
1447             let pkg = tmp.join("pkg");
1448             let _ = fs::remove_dir_all(&pkg);
1449
1450             let pkgbuild = |component: &str| {
1451                 let mut cmd = Command::new("pkgbuild");
1452                 cmd.arg("--identifier")
1453                     .arg(format!("org.rust-lang.{}", component))
1454                     .arg("--scripts")
1455                     .arg(pkg.join(component))
1456                     .arg("--nopayload")
1457                     .arg(pkg.join(component).with_extension("pkg"));
1458                 builder.run(&mut cmd);
1459             };
1460
1461             let prepare = |name: &str| {
1462                 builder.create_dir(&pkg.join(name));
1463                 builder.cp_r(
1464                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1465                     &pkg.join(name),
1466                 );
1467                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1468                 pkgbuild(name);
1469             };
1470             prepare("rustc");
1471             prepare("cargo");
1472             prepare("rust-docs");
1473             prepare("rust-std");
1474             prepare("rust-analysis");
1475             prepare("clippy");
1476             if rust_demangler_installer.is_some() {
1477                 prepare("rust-demangler");
1478             }
1479             if rls_installer.is_some() {
1480                 prepare("rls");
1481             }
1482             if rust_analyzer_installer.is_some() {
1483                 prepare("rust-analyzer");
1484             }
1485             if miri_installer.is_some() {
1486                 prepare("miri");
1487             }
1488
1489             // create an 'uninstall' package
1490             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1491             pkgbuild("uninstall");
1492
1493             builder.create_dir(&pkg.join("res"));
1494             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1495             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1496             let mut cmd = Command::new("productbuild");
1497             cmd.arg("--distribution")
1498                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1499                 .arg("--resources")
1500                 .arg(pkg.join("res"))
1501                 .arg(distdir(builder).join(format!(
1502                     "{}-{}.pkg",
1503                     pkgname(builder, "rust"),
1504                     target.triple
1505                 )))
1506                 .arg("--package-path")
1507                 .arg(&pkg);
1508             let _time = timeit(builder);
1509             builder.run(&mut cmd);
1510         }
1511
1512         if target.contains("windows") {
1513             let exe = tmp.join("exe");
1514             let _ = fs::remove_dir_all(&exe);
1515
1516             let prepare = |name: &str| {
1517                 builder.create_dir(&exe.join(name));
1518                 let dir = if name == "rust-std" || name == "rust-analysis" {
1519                     format!("{}-{}", name, target.triple)
1520                 } else if name == "rls" {
1521                     "rls-preview".to_string()
1522                 } else if name == "rust-analyzer" {
1523                     "rust-analyzer-preview".to_string()
1524                 } else if name == "clippy" {
1525                     "clippy-preview".to_string()
1526                 } else if name == "rust-demangler" {
1527                     "rust-demangler-preview".to_string()
1528                 } else if name == "miri" {
1529                     "miri-preview".to_string()
1530                 } else {
1531                     name.to_string()
1532                 };
1533                 builder.cp_r(
1534                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1535                     &exe.join(name),
1536                 );
1537                 builder.remove(&exe.join(name).join("manifest.in"));
1538             };
1539             prepare("rustc");
1540             prepare("cargo");
1541             prepare("rust-analysis");
1542             prepare("rust-docs");
1543             prepare("rust-std");
1544             prepare("clippy");
1545             if rust_demangler_installer.is_some() {
1546                 prepare("rust-demangler");
1547             }
1548             if rls_installer.is_some() {
1549                 prepare("rls");
1550             }
1551             if rust_analyzer_installer.is_some() {
1552                 prepare("rust-analyzer");
1553             }
1554             if miri_installer.is_some() {
1555                 prepare("miri");
1556             }
1557             if target.contains("windows-gnu") {
1558                 prepare("rust-mingw");
1559             }
1560
1561             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1562
1563             // Generate msi installer
1564             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1565             let heat = wix.join("bin/heat.exe");
1566             let candle = wix.join("bin/candle.exe");
1567             let light = wix.join("bin/light.exe");
1568
1569             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1570             builder.run(
1571                 Command::new(&heat)
1572                     .current_dir(&exe)
1573                     .arg("dir")
1574                     .arg("rustc")
1575                     .args(&heat_flags)
1576                     .arg("-cg")
1577                     .arg("RustcGroup")
1578                     .arg("-dr")
1579                     .arg("Rustc")
1580                     .arg("-var")
1581                     .arg("var.RustcDir")
1582                     .arg("-out")
1583                     .arg(exe.join("RustcGroup.wxs")),
1584             );
1585             builder.run(
1586                 Command::new(&heat)
1587                     .current_dir(&exe)
1588                     .arg("dir")
1589                     .arg("rust-docs")
1590                     .args(&heat_flags)
1591                     .arg("-cg")
1592                     .arg("DocsGroup")
1593                     .arg("-dr")
1594                     .arg("Docs")
1595                     .arg("-var")
1596                     .arg("var.DocsDir")
1597                     .arg("-out")
1598                     .arg(exe.join("DocsGroup.wxs"))
1599                     .arg("-t")
1600                     .arg(etc.join("msi/squash-components.xsl")),
1601             );
1602             builder.run(
1603                 Command::new(&heat)
1604                     .current_dir(&exe)
1605                     .arg("dir")
1606                     .arg("cargo")
1607                     .args(&heat_flags)
1608                     .arg("-cg")
1609                     .arg("CargoGroup")
1610                     .arg("-dr")
1611                     .arg("Cargo")
1612                     .arg("-var")
1613                     .arg("var.CargoDir")
1614                     .arg("-out")
1615                     .arg(exe.join("CargoGroup.wxs"))
1616                     .arg("-t")
1617                     .arg(etc.join("msi/remove-duplicates.xsl")),
1618             );
1619             builder.run(
1620                 Command::new(&heat)
1621                     .current_dir(&exe)
1622                     .arg("dir")
1623                     .arg("rust-std")
1624                     .args(&heat_flags)
1625                     .arg("-cg")
1626                     .arg("StdGroup")
1627                     .arg("-dr")
1628                     .arg("Std")
1629                     .arg("-var")
1630                     .arg("var.StdDir")
1631                     .arg("-out")
1632                     .arg(exe.join("StdGroup.wxs")),
1633             );
1634             if rls_installer.is_some() {
1635                 builder.run(
1636                     Command::new(&heat)
1637                         .current_dir(&exe)
1638                         .arg("dir")
1639                         .arg("rls")
1640                         .args(&heat_flags)
1641                         .arg("-cg")
1642                         .arg("RlsGroup")
1643                         .arg("-dr")
1644                         .arg("Rls")
1645                         .arg("-var")
1646                         .arg("var.RlsDir")
1647                         .arg("-out")
1648                         .arg(exe.join("RlsGroup.wxs"))
1649                         .arg("-t")
1650                         .arg(etc.join("msi/remove-duplicates.xsl")),
1651                 );
1652             }
1653             if rust_analyzer_installer.is_some() {
1654                 builder.run(
1655                     Command::new(&heat)
1656                         .current_dir(&exe)
1657                         .arg("dir")
1658                         .arg("rust-analyzer")
1659                         .args(&heat_flags)
1660                         .arg("-cg")
1661                         .arg("RustAnalyzerGroup")
1662                         .arg("-dr")
1663                         .arg("RustAnalyzer")
1664                         .arg("-var")
1665                         .arg("var.RustAnalyzerDir")
1666                         .arg("-out")
1667                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1668                         .arg("-t")
1669                         .arg(etc.join("msi/remove-duplicates.xsl")),
1670                 );
1671             }
1672             builder.run(
1673                 Command::new(&heat)
1674                     .current_dir(&exe)
1675                     .arg("dir")
1676                     .arg("clippy")
1677                     .args(&heat_flags)
1678                     .arg("-cg")
1679                     .arg("ClippyGroup")
1680                     .arg("-dr")
1681                     .arg("Clippy")
1682                     .arg("-var")
1683                     .arg("var.ClippyDir")
1684                     .arg("-out")
1685                     .arg(exe.join("ClippyGroup.wxs"))
1686                     .arg("-t")
1687                     .arg(etc.join("msi/remove-duplicates.xsl")),
1688             );
1689             if rust_demangler_installer.is_some() {
1690                 builder.run(
1691                     Command::new(&heat)
1692                         .current_dir(&exe)
1693                         .arg("dir")
1694                         .arg("rust-demangler")
1695                         .args(&heat_flags)
1696                         .arg("-cg")
1697                         .arg("RustDemanglerGroup")
1698                         .arg("-dr")
1699                         .arg("RustDemangler")
1700                         .arg("-var")
1701                         .arg("var.RustDemanglerDir")
1702                         .arg("-out")
1703                         .arg(exe.join("RustDemanglerGroup.wxs"))
1704                         .arg("-t")
1705                         .arg(etc.join("msi/remove-duplicates.xsl")),
1706                 );
1707             }
1708             if miri_installer.is_some() {
1709                 builder.run(
1710                     Command::new(&heat)
1711                         .current_dir(&exe)
1712                         .arg("dir")
1713                         .arg("miri")
1714                         .args(&heat_flags)
1715                         .arg("-cg")
1716                         .arg("MiriGroup")
1717                         .arg("-dr")
1718                         .arg("Miri")
1719                         .arg("-var")
1720                         .arg("var.MiriDir")
1721                         .arg("-out")
1722                         .arg(exe.join("MiriGroup.wxs"))
1723                         .arg("-t")
1724                         .arg(etc.join("msi/remove-duplicates.xsl")),
1725                 );
1726             }
1727             builder.run(
1728                 Command::new(&heat)
1729                     .current_dir(&exe)
1730                     .arg("dir")
1731                     .arg("rust-analysis")
1732                     .args(&heat_flags)
1733                     .arg("-cg")
1734                     .arg("AnalysisGroup")
1735                     .arg("-dr")
1736                     .arg("Analysis")
1737                     .arg("-var")
1738                     .arg("var.AnalysisDir")
1739                     .arg("-out")
1740                     .arg(exe.join("AnalysisGroup.wxs"))
1741                     .arg("-t")
1742                     .arg(etc.join("msi/remove-duplicates.xsl")),
1743             );
1744             if target.contains("windows-gnu") {
1745                 builder.run(
1746                     Command::new(&heat)
1747                         .current_dir(&exe)
1748                         .arg("dir")
1749                         .arg("rust-mingw")
1750                         .args(&heat_flags)
1751                         .arg("-cg")
1752                         .arg("GccGroup")
1753                         .arg("-dr")
1754                         .arg("Gcc")
1755                         .arg("-var")
1756                         .arg("var.GccDir")
1757                         .arg("-out")
1758                         .arg(exe.join("GccGroup.wxs")),
1759                 );
1760             }
1761
1762             let candle = |input: &Path| {
1763                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1764                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1765                 let mut cmd = Command::new(&candle);
1766                 cmd.current_dir(&exe)
1767                     .arg("-nologo")
1768                     .arg("-dRustcDir=rustc")
1769                     .arg("-dDocsDir=rust-docs")
1770                     .arg("-dCargoDir=cargo")
1771                     .arg("-dStdDir=rust-std")
1772                     .arg("-dAnalysisDir=rust-analysis")
1773                     .arg("-dClippyDir=clippy")
1774                     .arg("-arch")
1775                     .arg(&arch)
1776                     .arg("-out")
1777                     .arg(&output)
1778                     .arg(&input);
1779                 add_env(builder, &mut cmd, target);
1780
1781                 if rust_demangler_installer.is_some() {
1782                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1783                 }
1784                 if rls_installer.is_some() {
1785                     cmd.arg("-dRlsDir=rls");
1786                 }
1787                 if rust_analyzer_installer.is_some() {
1788                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1789                 }
1790                 if miri_installer.is_some() {
1791                     cmd.arg("-dMiriDir=miri");
1792                 }
1793                 if target.contains("windows-gnu") {
1794                     cmd.arg("-dGccDir=rust-mingw");
1795                 }
1796                 builder.run(&mut cmd);
1797             };
1798             candle(&xform(&etc.join("msi/rust.wxs")));
1799             candle(&etc.join("msi/ui.wxs"));
1800             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1801             candle("RustcGroup.wxs".as_ref());
1802             candle("DocsGroup.wxs".as_ref());
1803             candle("CargoGroup.wxs".as_ref());
1804             candle("StdGroup.wxs".as_ref());
1805             candle("ClippyGroup.wxs".as_ref());
1806             if rust_demangler_installer.is_some() {
1807                 candle("RustDemanglerGroup.wxs".as_ref());
1808             }
1809             if rls_installer.is_some() {
1810                 candle("RlsGroup.wxs".as_ref());
1811             }
1812             if rust_analyzer_installer.is_some() {
1813                 candle("RustAnalyzerGroup.wxs".as_ref());
1814             }
1815             if miri_installer.is_some() {
1816                 candle("MiriGroup.wxs".as_ref());
1817             }
1818             candle("AnalysisGroup.wxs".as_ref());
1819
1820             if target.contains("windows-gnu") {
1821                 candle("GccGroup.wxs".as_ref());
1822             }
1823
1824             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1825             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1826             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1827
1828             builder.info(&format!("building `msi` installer with {:?}", light));
1829             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1830             let mut cmd = Command::new(&light);
1831             cmd.arg("-nologo")
1832                 .arg("-ext")
1833                 .arg("WixUIExtension")
1834                 .arg("-ext")
1835                 .arg("WixUtilExtension")
1836                 .arg("-out")
1837                 .arg(exe.join(&filename))
1838                 .arg("rust.wixobj")
1839                 .arg("ui.wixobj")
1840                 .arg("rustwelcomedlg.wixobj")
1841                 .arg("RustcGroup.wixobj")
1842                 .arg("DocsGroup.wixobj")
1843                 .arg("CargoGroup.wixobj")
1844                 .arg("StdGroup.wixobj")
1845                 .arg("AnalysisGroup.wixobj")
1846                 .arg("ClippyGroup.wixobj")
1847                 .current_dir(&exe);
1848
1849             if rls_installer.is_some() {
1850                 cmd.arg("RlsGroup.wixobj");
1851             }
1852             if rust_analyzer_installer.is_some() {
1853                 cmd.arg("RustAnalyzerGroup.wixobj");
1854             }
1855             if rust_demangler_installer.is_some() {
1856                 cmd.arg("RustDemanglerGroup.wixobj");
1857             }
1858             if miri_installer.is_some() {
1859                 cmd.arg("MiriGroup.wixobj");
1860             }
1861
1862             if target.contains("windows-gnu") {
1863                 cmd.arg("GccGroup.wixobj");
1864             }
1865             // ICE57 wrongly complains about the shortcuts
1866             cmd.arg("-sice:ICE57");
1867
1868             let _time = timeit(builder);
1869             builder.run(&mut cmd);
1870
1871             if !builder.config.dry_run {
1872                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1873             }
1874         }
1875     }
1876 }
1877
1878 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1879     let mut parts = builder.version.split('.');
1880     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1881         .env("CFG_RELEASE_NUM", &builder.version)
1882         .env("CFG_RELEASE", builder.rust_release())
1883         .env("CFG_VER_MAJOR", parts.next().unwrap())
1884         .env("CFG_VER_MINOR", parts.next().unwrap())
1885         .env("CFG_VER_PATCH", parts.next().unwrap())
1886         .env("CFG_VER_BUILD", "0") // just needed to build
1887         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1888         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1889         .env("CFG_BUILD", target.triple)
1890         .env("CFG_CHANNEL", &builder.config.channel);
1891
1892     if target.contains("windows-gnu") {
1893         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1894     } else {
1895         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1896     }
1897
1898     if target.contains("x86_64") {
1899         cmd.env("CFG_PLATFORM", "x64");
1900     } else {
1901         cmd.env("CFG_PLATFORM", "x86");
1902     }
1903 }
1904
1905 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1906 ///
1907
1908 /// Returns whether the files were actually copied.
1909 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1910     if let Some(config) = builder.config.target_config.get(&target) {
1911         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1912             // If the LLVM was externally provided, then we don't currently copy
1913             // artifacts into the sysroot. This is not necessarily the right
1914             // choice (in particular, it will require the LLVM dylib to be in
1915             // the linker's load path at runtime), but the common use case for
1916             // external LLVMs is distribution provided LLVMs, and in that case
1917             // they're usually in the standard search path (e.g., /usr/lib) and
1918             // copying them here is going to cause problems as we may end up
1919             // with the wrong files and isn't what distributions want.
1920             //
1921             // This behavior may be revisited in the future though.
1922             //
1923             // If the LLVM is coming from ourselves (just from CI) though, we
1924             // still want to install it, as it otherwise won't be available.
1925             return false;
1926         }
1927     }
1928
1929     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1930     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1931     // clear why this is the case, though. llvm-config will emit the versioned
1932     // paths and we don't want those in the sysroot (as we're expecting
1933     // unversioned paths).
1934     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1935         let src_libdir = builder.llvm_out(target).join("lib");
1936         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1937         if llvm_dylib_path.exists() {
1938             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1939         }
1940         !builder.config.dry_run
1941     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1942         let mut cmd = Command::new(llvm_config);
1943         cmd.arg("--libfiles");
1944         builder.verbose(&format!("running {:?}", cmd));
1945         let files = output(&mut cmd);
1946         for file in files.trim_end().split(' ') {
1947             builder.install(Path::new(file), dst_libdir, 0o644);
1948         }
1949         !builder.config.dry_run
1950     } else {
1951         false
1952     }
1953 }
1954
1955 /// Maybe add libLLVM.so to the target lib-dir for linking.
1956 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1957     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1958     // We do not need to copy LLVM files into the sysroot if it is not
1959     // dynamically linked; it is already included into librustc_llvm
1960     // statically.
1961     if builder.config.llvm_link_shared {
1962         maybe_install_llvm(builder, target, &dst_libdir);
1963     }
1964 }
1965
1966 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1967 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1968     let dst_libdir =
1969         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1970     // We do not need to copy LLVM files into the sysroot if it is not
1971     // dynamically linked; it is already included into librustc_llvm
1972     // statically.
1973     if builder.config.llvm_link_shared {
1974         maybe_install_llvm(builder, target, &dst_libdir);
1975     }
1976 }
1977
1978 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1979 pub struct LlvmTools {
1980     pub target: TargetSelection,
1981 }
1982
1983 impl Step for LlvmTools {
1984     type Output = Option<GeneratedTarball>;
1985     const ONLY_HOSTS: bool = true;
1986
1987     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1988         run.path("llvm-tools")
1989     }
1990
1991     fn make_run(run: RunConfig<'_>) {
1992         run.builder.ensure(LlvmTools { target: run.target });
1993     }
1994
1995     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1996         let target = self.target;
1997         assert!(builder.config.extended);
1998
1999         /* run only if llvm-config isn't used */
2000         if let Some(config) = builder.config.target_config.get(&target) {
2001             if let Some(ref _s) = config.llvm_config {
2002                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2003                 return None;
2004             }
2005         }
2006
2007         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2008         tarball.set_overlay(OverlayKind::LLVM);
2009         tarball.is_preview(true);
2010
2011         // Prepare the image directory
2012         let src_bindir = builder.llvm_out(target).join("bin");
2013         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2014         for tool in LLVM_TOOLS {
2015             let exe = src_bindir.join(exe(tool, target));
2016             tarball.add_file(&exe, &dst_bindir, 0o755);
2017         }
2018
2019         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2020         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2021         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2022         // compiler libraries.
2023         maybe_install_llvm_target(builder, target, tarball.image_dir());
2024
2025         Some(tarball.generate())
2026     }
2027 }
2028
2029 // Tarball intended for internal consumption to ease rustc/std development.
2030 //
2031 // Should not be considered stable by end users.
2032 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2033 pub struct RustDev {
2034     pub target: TargetSelection,
2035 }
2036
2037 impl Step for RustDev {
2038     type Output = Option<GeneratedTarball>;
2039     const DEFAULT: bool = true;
2040     const ONLY_HOSTS: bool = true;
2041
2042     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2043         run.path("rust-dev")
2044     }
2045
2046     fn make_run(run: RunConfig<'_>) {
2047         run.builder.ensure(RustDev { target: run.target });
2048     }
2049
2050     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2051         let target = self.target;
2052
2053         /* run only if llvm-config isn't used */
2054         if let Some(config) = builder.config.target_config.get(&target) {
2055             if let Some(ref _s) = config.llvm_config {
2056                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2057                 return None;
2058             }
2059         }
2060
2061         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2062         tarball.set_overlay(OverlayKind::LLVM);
2063
2064         let src_bindir = builder.llvm_out(target).join("bin");
2065         for bin in &[
2066             "llvm-config",
2067             "llvm-ar",
2068             "llvm-objdump",
2069             "llvm-profdata",
2070             "llvm-bcanalyzer",
2071             "llvm-cov",
2072             "llvm-dwp",
2073         ] {
2074             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2075         }
2076         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2077
2078         // Copy the include directory as well; needed mostly to build
2079         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2080         // just broadly useful to be able to link against the bundled LLVM.
2081         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2082
2083         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2084         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2085         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2086         // compiler libraries.
2087         let dst_libdir = tarball.image_dir().join("lib");
2088         maybe_install_llvm(builder, target, &dst_libdir);
2089         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2090         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2091
2092         Some(tarball.generate())
2093     }
2094 }
2095
2096 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2097 /// release process to avoid cloning the monorepo and building stuff.
2098 ///
2099 /// Should not be considered stable by end users.
2100 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2101 pub struct BuildManifest {
2102     pub target: TargetSelection,
2103 }
2104
2105 impl Step for BuildManifest {
2106     type Output = GeneratedTarball;
2107     const DEFAULT: bool = false;
2108     const ONLY_HOSTS: bool = true;
2109
2110     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2111         run.path("src/tools/build-manifest")
2112     }
2113
2114     fn make_run(run: RunConfig<'_>) {
2115         run.builder.ensure(BuildManifest { target: run.target });
2116     }
2117
2118     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2119         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2120
2121         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2122         tarball.add_file(&build_manifest, "bin", 0o755);
2123         tarball.generate()
2124     }
2125 }
2126
2127 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2128 ///
2129 /// Currently this is the PGO profile data.
2130 ///
2131 /// Should not be considered stable by end users.
2132 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2133 pub struct ReproducibleArtifacts {
2134     pub target: TargetSelection,
2135 }
2136
2137 impl Step for ReproducibleArtifacts {
2138     type Output = Option<GeneratedTarball>;
2139     const DEFAULT: bool = true;
2140     const ONLY_HOSTS: bool = true;
2141
2142     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2143         run.path("reproducible")
2144     }
2145
2146     fn make_run(run: RunConfig<'_>) {
2147         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2148     }
2149
2150     fn run(self, builder: &Builder<'_>) -> Self::Output {
2151         let path = builder.config.rust_profile_use.as_ref()?;
2152
2153         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2154         tarball.add_file(path, ".", 0o644);
2155         Some(tarball.generate())
2156     }
2157 }