src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14 use std::process::Command;
  15
  16 use build_helper::{output, t};
  17
  18 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn missing_tool(tool_name: &str, skip: bool) {
  41     if skip {
  42         println!("Unable to build {}, skipping dist", tool_name)
  43     } else {
  44         panic!("Unable to build {}", tool_name)
  45     }
  46 }
  47
  48 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  49 pub struct Docs {
  50     pub host: TargetSelection,
  51 }
  52
  53 impl Step for Docs {
  54     type Output = Option<GeneratedTarball>;
  55     const DEFAULT: bool = true;
  56
  57     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  58         run.path("src/doc")
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         if !builder.config.docs {
  69             return None;
  70         }
  71         builder.default_doc(&[]);
  72
  73         let dest = "share/doc/rust/html";
  74
  75         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  76         tarball.set_product_name("Rust Documentation");
  77         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  78         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  79         Some(tarball.generate())
  80     }
  81 }
  82
  83 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  84 pub struct RustcDocs {
  85     pub host: TargetSelection,
  86 }
  87
  88 impl Step for RustcDocs {
  89     type Output = Option<GeneratedTarball>;
  90     const DEFAULT: bool = true;
  91
  92     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  93         run.path("src/librustc")
  94     }
  95
  96     fn make_run(run: RunConfig<'_>) {
  97         run.builder.ensure(RustcDocs { host: run.target });
  98     }
  99
 100     /// Builds the `rustc-docs` installer component.
 101     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 102         let host = self.host;
 103         if !builder.config.compiler_docs {
 104             return None;
 105         }
 106         builder.default_doc(&[]);
 107
 108         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 109         tarball.set_product_name("Rustc Documentation");
 110         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 111         Some(tarball.generate())
 112     }
 113 }
 114
 115 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 116     let mut found = Vec::with_capacity(files.len());
 117
 118     for file in files {
 119         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 120
 121         if let Some(file_path) = file_path {
 122             found.push(file_path);
 123         } else {
 124             panic!("Could not find '{}' in {:?}", file, path);
 125         }
 126     }
 127
 128     found
 129 }
 130
 131 fn make_win_dist(
 132     rust_root: &Path,
 133     plat_root: &Path,
 134     target: TargetSelection,
 135     builder: &Builder<'_>,
 136 ) {
 137     //Ask gcc where it keeps its stuff
 138     let mut cmd = Command::new(builder.cc(target));
 139     cmd.arg("-print-search-dirs");
 140     let gcc_out = output(&mut cmd);
 141
 142     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 143     let mut lib_path = Vec::new();
 144
 145     for line in gcc_out.lines() {
 146         let idx = line.find(':').unwrap();
 147         let key = &line[..idx];
 148         let trim_chars: &[_] = &[' ', '='];
 149         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 150
 151         if key == "programs" {
 152             bin_path.extend(value);
 153         } else if key == "libraries" {
 154             lib_path.extend(value);
 155         }
 156     }
 157
 158     let compiler = if target == "i686-pc-windows-gnu" {
 159         "i686-w64-mingw32-gcc.exe"
 160     } else if target == "x86_64-pc-windows-gnu" {
 161         "x86_64-w64-mingw32-gcc.exe"
 162     } else {
 163         "gcc.exe"
 164     };
 165     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 166     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 167     if target.starts_with("i686-") {
 168         rustc_dlls.push("libgcc_s_dw2-1.dll");
 169     } else {
 170         rustc_dlls.push("libgcc_s_seh-1.dll");
 171     }
 172
 173     let target_libs = [
 174         //MinGW libs
 175         "libgcc.a",
 176         "libgcc_eh.a",
 177         "libgcc_s.a",
 178         "libm.a",
 179         "libmingw32.a",
 180         "libmingwex.a",
 181         "libstdc++.a",
 182         "libiconv.a",
 183         "libmoldname.a",
 184         "libpthread.a",
 185         //Windows import libs
 186         "libadvapi32.a",
 187         "libbcrypt.a",
 188         "libcomctl32.a",
 189         "libcomdlg32.a",
 190         "libcredui.a",
 191         "libcrypt32.a",
 192         "libdbghelp.a",
 193         "libgdi32.a",
 194         "libimagehlp.a",
 195         "libiphlpapi.a",
 196         "libkernel32.a",
 197         "libmsimg32.a",
 198         "libmsvcrt.a",
 199         "libodbc32.a",
 200         "libole32.a",
 201         "liboleaut32.a",
 202         "libopengl32.a",
 203         "libpsapi.a",
 204         "librpcrt4.a",
 205         "libsecur32.a",
 206         "libsetupapi.a",
 207         "libshell32.a",
 208         "libsynchronization.a",
 209         "libuser32.a",
 210         "libuserenv.a",
 211         "libuuid.a",
 212         "libwinhttp.a",
 213         "libwinmm.a",
 214         "libwinspool.a",
 215         "libws2_32.a",
 216         "libwsock32.a",
 217     ];
 218
 219     //Find mingw artifacts we want to bundle
 220     let target_tools = find_files(&target_tools, &bin_path);
 221     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 222     let target_libs = find_files(&target_libs, &lib_path);
 223
 224     // Copy runtime dlls next to rustc.exe
 225     let dist_bin_dir = rust_root.join("bin/");
 226     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 227     for src in rustc_dlls {
 228         builder.copy_to_folder(&src, &dist_bin_dir);
 229     }
 230
 231     //Copy platform tools to platform-specific bin directory
 232     let target_bin_dir = plat_root
 233         .join("lib")
 234         .join("rustlib")
 235         .join(target.triple)
 236         .join("bin")
 237         .join("self-contained");
 238     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 239     for src in target_tools {
 240         builder.copy_to_folder(&src, &target_bin_dir);
 241     }
 242
 243     // Warn windows-gnu users that the bundled GCC cannot compile C files
 244     builder.create(
 245         &target_bin_dir.join("GCC-WARNING.txt"),
 246         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 247          used as a linker. In order to be able to compile projects containing C code use \
 248          the GCC provided by MinGW or Cygwin.",
 249     );
 250
 251     //Copy platform libs to platform-specific lib directory
 252     let target_lib_dir = plat_root
 253         .join("lib")
 254         .join("rustlib")
 255         .join(target.triple)
 256         .join("lib")
 257         .join("self-contained");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         builder.copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     pub host: TargetSelection,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<GeneratedTarball>;
 271     const DEFAULT: bool = true;
 272
 273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 274         run.never()
 275     }
 276
 277     fn make_run(run: RunConfig<'_>) {
 278         run.builder.ensure(Mingw { host: run.target });
 279     }
 280
 281     /// Builds the `rust-mingw` installer component.
 282     ///
 283     /// This contains all the bits and pieces to run the MinGW Windows targets
 284     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 285     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 286         let host = self.host;
 287         if !host.contains("pc-windows-gnu") {
 288             return None;
 289         }
 290
 291         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 292         tarball.set_product_name("Rust MinGW");
 293
 294         // The first argument is a "temporary directory" which is just
 295         // thrown away (this contains the runtime DLLs included in the rustc package
 296         // above) and the second argument is where to place all the MinGW components
 297         // (which is what we want).
 298         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 299
 300         Some(tarball.generate())
 301     }
 302 }
 303
 304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 305 pub struct Rustc {
 306     pub compiler: Compiler,
 307 }
 308
 309 impl Step for Rustc {
 310     type Output = GeneratedTarball;
 311     const DEFAULT: bool = true;
 312     const ONLY_HOSTS: bool = true;
 313
 314     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 315         run.path("src/librustc")
 316     }
 317
 318     fn make_run(run: RunConfig<'_>) {
 319         run.builder
 320             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 321     }
 322
 323     /// Creates the `rustc` installer component.
 324     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 325         let compiler = self.compiler;
 326         let host = self.compiler.host;
 327
 328         let tarball = Tarball::new(builder, "rustc", &host.triple);
 329
 330         // Prepare the rustc "image", what will actually end up getting installed
 331         prepare_image(builder, compiler, tarball.image_dir());
 332
 333         // On MinGW we've got a few runtime DLL dependencies that we need to
 334         // include. The first argument to this script is where to put these DLLs
 335         // (the image we're creating), and the second argument is a junk directory
 336         // to ignore all other MinGW stuff the script creates.
 337         //
 338         // On 32-bit MinGW we're always including a DLL which needs some extra
 339         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 340         // anything requiring us to distribute a license, but it's likely the
 341         // install will *also* include the rust-mingw package, which also needs
 342         // licenses, so to be safe we just include it here in all MinGW packages.
 343         if host.contains("pc-windows-gnu") {
 344             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 345             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 346         }
 347
 348         return tarball.generate();
 349
 350         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 351             let host = compiler.host;
 352             let src = builder.sysroot(compiler);
 353
 354             // Copy rustc/rustdoc binaries
 355             t!(fs::create_dir_all(image.join("bin")));
 356             builder.cp_r(&src.join("bin"), &image.join("bin"));
 357
 358             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 359
 360             let libdir_relative = builder.libdir_relative(compiler);
 361
 362             // Copy runtime DLLs needed by the compiler
 363             if libdir_relative.to_str() != Some("bin") {
 364                 let libdir = builder.rustc_libdir(compiler);
 365                 for entry in builder.read_dir(&libdir) {
 366                     let name = entry.file_name();
 367                     if let Some(s) = name.to_str() {
 368                         if is_dylib(s) {
 369                             // Don't use custom libdir here because ^lib/ will be resolved again
 370                             // with installer
 371                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 372                         }
 373                     }
 374                 }
 375             }
 376
 377             // Copy over the codegen backends
 378             let backends_src = builder.sysroot_codegen_backends(compiler);
 379             let backends_rel = backends_src
 380                 .strip_prefix(&src)
 381                 .unwrap()
 382                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 383                 .unwrap();
 384             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 385             let backends_dst = image.join("lib").join(&backends_rel);
 386
 387             t!(fs::create_dir_all(&backends_dst));
 388             builder.cp_r(&backends_src, &backends_dst);
 389
 390             // Copy libLLVM.so to the lib dir as well, if needed. While not
 391             // technically needed by rustc itself it's needed by lots of other
 392             // components like the llvm tools and LLD. LLD is included below and
 393             // tools/LLDB come later, so let's just throw it in the rustc
 394             // component for now.
 395             maybe_install_llvm_runtime(builder, host, image);
 396
 397             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let exe = exe("rust-lld", compiler.host);
 404                 builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 405                 // for `-Z gcc-ld=lld`
 406                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 407                 t!(fs::create_dir(&gcc_lld_dir));
 408                 builder.copy(&src_dir.join(&exe), &gcc_lld_dir.join(&exe));
 409             }
 410
 411             // Copy over llvm-dwp if it's there
 412             let exe = exe("rust-llvm-dwp", compiler.host);
 413             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 414
 415             // Man pages
 416             t!(fs::create_dir_all(image.join("share/man/man1")));
 417             let man_src = builder.src.join("src/doc/man");
 418             let man_dst = image.join("share/man/man1");
 419
 420             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 421             let time = env::var("SOURCE_DATE_EPOCH")
 422                 .map(|timestamp| {
 423                     let epoch = timestamp
 424                         .parse()
 425                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 426                         .unwrap();
 427
 428                     time::at(Timespec::new(epoch, 0))
 429                 })
 430                 .unwrap_or_else(|_| time::now());
 431
 432             let month_year = t!(time::strftime("%B %Y", &time));
 433             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 434             // to hardlink, and we don't want to edit the source templates
 435             for file_entry in builder.read_dir(&man_src) {
 436                 let page_src = file_entry.path();
 437                 let page_dst = man_dst.join(file_entry.file_name());
 438                 t!(fs::copy(&page_src, &page_dst));
 439                 // template in month/year and version number
 440                 builder.replace_in_file(
 441                     &page_dst,
 442                     &[
 443                         ("<INSERT DATE HERE>", &month_year),
 444                         ("<INSERT VERSION HERE>", &builder.version),
 445                     ],
 446                 );
 447             }
 448
 449             // Debugger scripts
 450             builder
 451                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 452
 453             // Misc license info
 454             let cp = |file: &str| {
 455                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 456             };
 457             cp("COPYRIGHT");
 458             cp("LICENSE-APACHE");
 459             cp("LICENSE-MIT");
 460             cp("README.md");
 461         }
 462     }
 463 }
 464
 465 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 466 pub struct DebuggerScripts {
 467     pub sysroot: Interned<PathBuf>,
 468     pub host: TargetSelection,
 469 }
 470
 471 impl Step for DebuggerScripts {
 472     type Output = ();
 473
 474     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 475         run.path("src/lldb_batchmode.py")
 476     }
 477
 478     fn make_run(run: RunConfig<'_>) {
 479         run.builder.ensure(DebuggerScripts {
 480             sysroot: run
 481                 .builder
 482                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 483             host: run.target,
 484         });
 485     }
 486
 487     /// Copies debugger scripts for `target` into the `sysroot` specified.
 488     fn run(self, builder: &Builder<'_>) {
 489         let host = self.host;
 490         let sysroot = self.sysroot;
 491         let dst = sysroot.join("lib/rustlib/etc");
 492         t!(fs::create_dir_all(&dst));
 493         let cp_debugger_script = |file: &str| {
 494             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 495         };
 496         if host.contains("windows-msvc") {
 497             // windbg debugger scripts
 498             builder.install(
 499                 &builder.src.join("src/etc/rust-windbg.cmd"),
 500                 &sysroot.join("bin"),
 501                 0o755,
 502             );
 503
 504             cp_debugger_script("natvis/intrinsic.natvis");
 505             cp_debugger_script("natvis/liballoc.natvis");
 506             cp_debugger_script("natvis/libcore.natvis");
 507             cp_debugger_script("natvis/libstd.natvis");
 508         } else {
 509             cp_debugger_script("rust_types.py");
 510
 511             // gdb debugger scripts
 512             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 513             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 514
 515             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 516             cp_debugger_script("gdb_lookup.py");
 517             cp_debugger_script("gdb_providers.py");
 518
 519             // lldb debugger scripts
 520             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 521
 522             cp_debugger_script("lldb_lookup.py");
 523             cp_debugger_script("lldb_providers.py");
 524             cp_debugger_script("lldb_commands")
 525         }
 526     }
 527 }
 528
 529 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 530     // The only true set of target libraries came from the build triple, so
 531     // let's reduce redundant work by only producing archives from that host.
 532     if compiler.host != builder.config.build {
 533         builder.info("\tskipping, not a build host");
 534         true
 535     } else {
 536         false
 537     }
 538 }
 539
 540 /// Copy stamped files into an image's `target/lib` directory.
 541 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 542     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 543     let self_contained_dst = dst.join("self-contained");
 544     t!(fs::create_dir_all(&dst));
 545     t!(fs::create_dir_all(&self_contained_dst));
 546     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 547         if dependency_type == DependencyType::TargetSelfContained {
 548             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 549         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 550             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: TargetSelection,
 559 }
 560
 561 impl Step for Std {
 562     type Output = Option<GeneratedTarball>;
 563     const DEFAULT: bool = true;
 564
 565     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 566         run.path("library/std")
 567     }
 568
 569     fn make_run(run: RunConfig<'_>) {
 570         run.builder.ensure(Std {
 571             compiler: run.builder.compiler_for(
 572                 run.builder.top_stage,
 573                 run.builder.config.build,
 574                 run.target,
 575             ),
 576             target: run.target,
 577         });
 578     }
 579
 580     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 581         let compiler = self.compiler;
 582         let target = self.target;
 583
 584         if skip_host_target_lib(builder, compiler) {
 585             return None;
 586         }
 587
 588         builder.ensure(compile::Std { compiler, target });
 589
 590         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 591         tarball.include_target_in_component_name(true);
 592
 593         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 594         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 595         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 596
 597         Some(tarball.generate())
 598     }
 599 }
 600
 601 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 602 pub struct RustcDev {
 603     pub compiler: Compiler,
 604     pub target: TargetSelection,
 605 }
 606
 607 impl Step for RustcDev {
 608     type Output = Option<GeneratedTarball>;
 609     const DEFAULT: bool = true;
 610     const ONLY_HOSTS: bool = true;
 611
 612     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 613         run.path("rustc-dev")
 614     }
 615
 616     fn make_run(run: RunConfig<'_>) {
 617         run.builder.ensure(RustcDev {
 618             compiler: run.builder.compiler_for(
 619                 run.builder.top_stage,
 620                 run.builder.config.build,
 621                 run.target,
 622             ),
 623             target: run.target,
 624         });
 625     }
 626
 627     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 628         let compiler = self.compiler;
 629         let target = self.target;
 630         if skip_host_target_lib(builder, compiler) {
 631             return None;
 632         }
 633
 634         builder.ensure(compile::Rustc { compiler, target });
 635
 636         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 637
 638         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 639         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 640         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 641
 642         let src_files = &["Cargo.lock"];
 643         // This is the reduced set of paths which will become the rustc-dev component
 644         // (essentially the compiler crates and all of their path dependencies).
 645         copy_src_dirs(
 646             builder,
 647             &builder.src,
 648             &["compiler"],
 649             &[],
 650             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 651         );
 652         // This particular crate is used as a build dependency of the above.
 653         copy_src_dirs(
 654             builder,
 655             &builder.src,
 656             &["src/build_helper"],
 657             &[],
 658             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 659         );
 660         for file in src_files {
 661             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 662         }
 663
 664         Some(tarball.generate())
 665     }
 666 }
 667
 668 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 669 pub struct Analysis {
 670     pub compiler: Compiler,
 671     pub target: TargetSelection,
 672 }
 673
 674 impl Step for Analysis {
 675     type Output = Option<GeneratedTarball>;
 676     const DEFAULT: bool = true;
 677
 678     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 679         let builder = run.builder;
 680         run.path("analysis").default_condition(builder.config.extended)
 681     }
 682
 683     fn make_run(run: RunConfig<'_>) {
 684         run.builder.ensure(Analysis {
 685             // Find the actual compiler (handling the full bootstrap option) which
 686             // produced the save-analysis data because that data isn't copied
 687             // through the sysroot uplifting.
 688             compiler: run.builder.compiler_for(
 689                 run.builder.top_stage,
 690                 run.builder.config.build,
 691                 run.target,
 692             ),
 693             target: run.target,
 694         });
 695     }
 696
 697     /// Creates a tarball of save-analysis metadata, if available.
 698     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 699         let compiler = self.compiler;
 700         let target = self.target;
 701         assert!(builder.config.extended);
 702         if compiler.host != builder.config.build {
 703             return None;
 704         }
 705
 706         builder.ensure(compile::Std { compiler, target });
 707         let src = builder
 708             .stage_out(compiler, Mode::Std)
 709             .join(target.triple)
 710             .join(builder.cargo_dir())
 711             .join("deps")
 712             .join("save-analysis");
 713
 714         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 715         tarball.include_target_in_component_name(true);
 716         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 717         Some(tarball.generate())
 718     }
 719 }
 720
 721 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 722 /// `dst_dir`.
 723 fn copy_src_dirs(
 724     builder: &Builder<'_>,
 725     base: &Path,
 726     src_dirs: &[&str],
 727     exclude_dirs: &[&str],
 728     dst_dir: &Path,
 729 ) {
 730     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 731         let spath = match path.to_str() {
 732             Some(path) => path,
 733             None => return false,
 734         };
 735         if spath.ends_with('~') || spath.ends_with(".pyc") {
 736             return false;
 737         }
 738
 739         const LLVM_PROJECTS: &[&str] = &[
 740             "llvm-project/clang",
 741             "llvm-project\\clang",
 742             "llvm-project/libunwind",
 743             "llvm-project\\libunwind",
 744             "llvm-project/lld",
 745             "llvm-project\\lld",
 746             "llvm-project/lldb",
 747             "llvm-project\\lldb",
 748             "llvm-project/llvm",
 749             "llvm-project\\llvm",
 750             "llvm-project/compiler-rt",
 751             "llvm-project\\compiler-rt",
 752         ];
 753         if spath.contains("llvm-project")
 754             && !spath.ends_with("llvm-project")
 755             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 756         {
 757             return false;
 758         }
 759
 760         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 761         if LLVM_TEST.iter().any(|path| spath.contains(path))
 762             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 763         {
 764             return false;
 765         }
 766
 767         let full_path = Path::new(dir).join(path);
 768         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 769             return false;
 770         }
 771
 772         let excludes = [
 773             "CVS",
 774             "RCS",
 775             "SCCS",
 776             ".git",
 777             ".gitignore",
 778             ".gitmodules",
 779             ".gitattributes",
 780             ".cvsignore",
 781             ".svn",
 782             ".arch-ids",
 783             "{arch}",
 784             "=RELEASE-ID",
 785             "=meta-update",
 786             "=update",
 787             ".bzr",
 788             ".bzrignore",
 789             ".bzrtags",
 790             ".hg",
 791             ".hgignore",
 792             ".hgrags",
 793             "_darcs",
 794         ];
 795         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 796     }
 797
 798     // Copy the directories using our filter
 799     for item in src_dirs {
 800         let dst = &dst_dir.join(item);
 801         t!(fs::create_dir_all(dst));
 802         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 803     }
 804 }
 805
 806 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 807 pub struct Src;
 808
 809 impl Step for Src {
 810     /// The output path of the src installer tarball
 811     type Output = GeneratedTarball;
 812     const DEFAULT: bool = true;
 813     const ONLY_HOSTS: bool = true;
 814
 815     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 816         run.path("src")
 817     }
 818
 819     fn make_run(run: RunConfig<'_>) {
 820         run.builder.ensure(Src);
 821     }
 822
 823     /// Creates the `rust-src` installer component
 824     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 825         let tarball = Tarball::new_targetless(builder, "rust-src");
 826
 827         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 828         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 829         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 830         // and fix them...
 831         //
 832         // NOTE: if you update the paths here, you also should update the "virtual" path
 833         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 834         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 835
 836         let src_files = ["Cargo.lock"];
 837         // This is the reduced set of paths which will become the rust-src component
 838         // (essentially libstd and all of its path dependencies).
 839         copy_src_dirs(
 840             builder,
 841             &builder.src,
 842             &["library", "src/llvm-project/libunwind"],
 843             &[
 844                 // not needed and contains symlinks which rustup currently
 845                 // chokes on when unpacking.
 846                 "library/backtrace/crates",
 847             ],
 848             &dst_src,
 849         );
 850         for file in src_files.iter() {
 851             builder.copy(&builder.src.join(file), &dst_src.join(file));
 852         }
 853
 854         tarball.generate()
 855     }
 856 }
 857
 858 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 859 pub struct PlainSourceTarball;
 860
 861 impl Step for PlainSourceTarball {
 862     /// Produces the location of the tarball generated
 863     type Output = GeneratedTarball;
 864     const DEFAULT: bool = true;
 865     const ONLY_HOSTS: bool = true;
 866
 867     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 868         let builder = run.builder;
 869         run.path("src").default_condition(builder.config.rust_dist_src)
 870     }
 871
 872     fn make_run(run: RunConfig<'_>) {
 873         run.builder.ensure(PlainSourceTarball);
 874     }
 875
 876     /// Creates the plain source tarball
 877     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 878         let tarball = Tarball::new(builder, "rustc", "src");
 879         let plain_dst_src = tarball.image_dir();
 880
 881         // This is the set of root paths which will become part of the source package
 882         let src_files = [
 883             "COPYRIGHT",
 884             "LICENSE-APACHE",
 885             "LICENSE-MIT",
 886             "CONTRIBUTING.md",
 887             "README.md",
 888             "RELEASES.md",
 889             "configure",
 890             "x.py",
 891             "config.toml.example",
 892             "Cargo.toml",
 893             "Cargo.lock",
 894         ];
 895         let src_dirs = ["src", "compiler", "library"];
 896
 897         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 898
 899         // Copy the files normally
 900         for item in &src_files {
 901             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 902         }
 903
 904         // Create the version file
 905         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 906         if let Some(sha) = builder.rust_sha() {
 907             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 908         }
 909
 910         // If we're building from git sources, we need to vendor a complete distribution.
 911         if builder.rust_info.is_git() {
 912             // Vendor all Cargo dependencies
 913             let mut cmd = Command::new(&builder.initial_cargo);
 914             cmd.arg("vendor")
 915                 .arg("--sync")
 916                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 917                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 918                 .current_dir(&plain_dst_src);
 919             builder.run(&mut cmd);
 920         }
 921
 922         tarball.bare()
 923     }
 924 }
 925
 926 // We have to run a few shell scripts, which choke quite a bit on both `\`
 927 // characters and on `C:\` paths, so normalize both of them away.
 928 pub fn sanitize_sh(path: &Path) -> String {
 929     let path = path.to_str().unwrap().replace("\\", "/");
 930     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 931
 932     fn unc_to_lfs(s: &str) -> &str {
 933         s.strip_prefix("//?/").unwrap_or(s)
 934     }
 935
 936     fn change_drive(s: &str) -> Option<String> {
 937         let mut ch = s.chars();
 938         let drive = ch.next().unwrap_or('C');
 939         if ch.next() != Some(':') {
 940             return None;
 941         }
 942         if ch.next() != Some('/') {
 943             return None;
 944         }
 945         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 946     }
 947 }
 948
 949 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 950 pub struct Cargo {
 951     pub compiler: Compiler,
 952     pub target: TargetSelection,
 953 }
 954
 955 impl Step for Cargo {
 956     type Output = GeneratedTarball;
 957     const ONLY_HOSTS: bool = true;
 958
 959     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 960         run.path("cargo")
 961     }
 962
 963     fn make_run(run: RunConfig<'_>) {
 964         run.builder.ensure(Cargo {
 965             compiler: run.builder.compiler_for(
 966                 run.builder.top_stage,
 967                 run.builder.config.build,
 968                 run.target,
 969             ),
 970             target: run.target,
 971         });
 972     }
 973
 974     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 975         let compiler = self.compiler;
 976         let target = self.target;
 977
 978         let cargo = builder.ensure(tool::Cargo { compiler, target });
 979         let src = builder.src.join("src/tools/cargo");
 980         let etc = src.join("src/etc");
 981
 982         // Prepare the image directory
 983         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 984         tarball.set_overlay(OverlayKind::Cargo);
 985
 986         tarball.add_file(&cargo, "bin", 0o755);
 987         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 988         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 989         tarball.add_dir(etc.join("man"), "share/man/man1");
 990         tarball.add_legal_and_readme_to("share/doc/cargo");
 991
 992         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 993             let dirent = dirent.expect("read dir entry");
 994             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 995                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 996             }
 997         }
 998
 999         tarball.generate()
1000     }
1001 }
1002
1003 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1004 pub struct Rls {
1005     pub compiler: Compiler,
1006     pub target: TargetSelection,
1007 }
1008
1009 impl Step for Rls {
1010     type Output = Option<GeneratedTarball>;
1011     const ONLY_HOSTS: bool = true;
1012
1013     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1014         run.path("rls")
1015     }
1016
1017     fn make_run(run: RunConfig<'_>) {
1018         run.builder.ensure(Rls {
1019             compiler: run.builder.compiler_for(
1020                 run.builder.top_stage,
1021                 run.builder.config.build,
1022                 run.target,
1023             ),
1024             target: run.target,
1025         });
1026     }
1027
1028     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1029         let compiler = self.compiler;
1030         let target = self.target;
1031         assert!(builder.config.extended);
1032
1033         let rls = builder
1034             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1035             .or_else(|| {
1036                 missing_tool("RLS", builder.build.config.missing_tools);
1037                 None
1038             })?;
1039
1040         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1041         tarball.set_overlay(OverlayKind::RLS);
1042         tarball.is_preview(true);
1043         tarball.add_file(rls, "bin", 0o755);
1044         tarball.add_legal_and_readme_to("share/doc/rls");
1045         Some(tarball.generate())
1046     }
1047 }
1048
1049 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1050 pub struct RustAnalyzer {
1051     pub compiler: Compiler,
1052     pub target: TargetSelection,
1053 }
1054
1055 impl Step for RustAnalyzer {
1056     type Output = Option<GeneratedTarball>;
1057     const ONLY_HOSTS: bool = true;
1058
1059     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1060         run.path("rust-analyzer")
1061     }
1062
1063     fn make_run(run: RunConfig<'_>) {
1064         run.builder.ensure(RustAnalyzer {
1065             compiler: run.builder.compiler_for(
1066                 run.builder.top_stage,
1067                 run.builder.config.build,
1068                 run.target,
1069             ),
1070             target: run.target,
1071         });
1072     }
1073
1074     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1075         let compiler = self.compiler;
1076         let target = self.target;
1077         assert!(builder.config.extended);
1078
1079         if target.contains("riscv64") {
1080             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1081             // to build. See #74813 for details.
1082             return None;
1083         }
1084
1085         let rust_analyzer = builder
1086             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1087             .expect("rust-analyzer always builds");
1088
1089         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1090         tarball.set_overlay(OverlayKind::RustAnalyzer);
1091         tarball.is_preview(true);
1092         tarball.add_file(rust_analyzer, "bin", 0o755);
1093         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1094         Some(tarball.generate())
1095     }
1096 }
1097
1098 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1099 pub struct Clippy {
1100     pub compiler: Compiler,
1101     pub target: TargetSelection,
1102 }
1103
1104 impl Step for Clippy {
1105     type Output = GeneratedTarball;
1106     const ONLY_HOSTS: bool = true;
1107
1108     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1109         run.path("clippy")
1110     }
1111
1112     fn make_run(run: RunConfig<'_>) {
1113         run.builder.ensure(Clippy {
1114             compiler: run.builder.compiler_for(
1115                 run.builder.top_stage,
1116                 run.builder.config.build,
1117                 run.target,
1118             ),
1119             target: run.target,
1120         });
1121     }
1122
1123     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
1124         let compiler = self.compiler;
1125         let target = self.target;
1126         assert!(builder.config.extended);
1127
1128         // Prepare the image directory
1129         // We expect clippy to build, because we've exited this step above if tool
1130         // state for clippy isn't testing.
1131         let clippy = builder
1132             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1133             .expect("clippy expected to build - essential tool");
1134         let cargoclippy = builder
1135             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1136             .expect("clippy expected to build - essential tool");
1137
1138         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1139         tarball.set_overlay(OverlayKind::Clippy);
1140         tarball.is_preview(true);
1141         tarball.add_file(clippy, "bin", 0o755);
1142         tarball.add_file(cargoclippy, "bin", 0o755);
1143         tarball.add_legal_and_readme_to("share/doc/clippy");
1144         tarball.generate()
1145     }
1146 }
1147
1148 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1149 pub struct Miri {
1150     pub compiler: Compiler,
1151     pub target: TargetSelection,
1152 }
1153
1154 impl Step for Miri {
1155     type Output = Option<GeneratedTarball>;
1156     const ONLY_HOSTS: bool = true;
1157
1158     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1159         run.path("miri")
1160     }
1161
1162     fn make_run(run: RunConfig<'_>) {
1163         run.builder.ensure(Miri {
1164             compiler: run.builder.compiler_for(
1165                 run.builder.top_stage,
1166                 run.builder.config.build,
1167                 run.target,
1168             ),
1169             target: run.target,
1170         });
1171     }
1172
1173     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1174         if !builder.build.unstable_features() {
1175             return None;
1176         }
1177         let compiler = self.compiler;
1178         let target = self.target;
1179         assert!(builder.config.extended);
1180
1181         let miri = builder
1182             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1183             .or_else(|| {
1184                 missing_tool("miri", builder.build.config.missing_tools);
1185                 None
1186             })?;
1187         let cargomiri = builder
1188             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1189             .or_else(|| {
1190                 missing_tool("cargo miri", builder.build.config.missing_tools);
1191                 None
1192             })?;
1193
1194         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1195         tarball.set_overlay(OverlayKind::Miri);
1196         tarball.is_preview(true);
1197         tarball.add_file(miri, "bin", 0o755);
1198         tarball.add_file(cargomiri, "bin", 0o755);
1199         tarball.add_legal_and_readme_to("share/doc/miri");
1200         Some(tarball.generate())
1201     }
1202 }
1203
1204 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1205 pub struct Rustfmt {
1206     pub compiler: Compiler,
1207     pub target: TargetSelection,
1208 }
1209
1210 impl Step for Rustfmt {
1211     type Output = Option<GeneratedTarball>;
1212     const ONLY_HOSTS: bool = true;
1213
1214     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1215         run.path("rustfmt")
1216     }
1217
1218     fn make_run(run: RunConfig<'_>) {
1219         run.builder.ensure(Rustfmt {
1220             compiler: run.builder.compiler_for(
1221                 run.builder.top_stage,
1222                 run.builder.config.build,
1223                 run.target,
1224             ),
1225             target: run.target,
1226         });
1227     }
1228
1229     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1230         let compiler = self.compiler;
1231         let target = self.target;
1232
1233         let rustfmt = builder
1234             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1235             .or_else(|| {
1236                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1237                 None
1238             })?;
1239         let cargofmt = builder
1240             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1241             .or_else(|| {
1242                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1243                 None
1244             })?;
1245
1246         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1247         tarball.set_overlay(OverlayKind::Rustfmt);
1248         tarball.is_preview(true);
1249         tarball.add_file(rustfmt, "bin", 0o755);
1250         tarball.add_file(cargofmt, "bin", 0o755);
1251         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1252         Some(tarball.generate())
1253     }
1254 }
1255
1256 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1257 pub struct RustDemangler {
1258     pub compiler: Compiler,
1259     pub target: TargetSelection,
1260 }
1261
1262 impl Step for RustDemangler {
1263     type Output = Option<GeneratedTarball>;
1264     const ONLY_HOSTS: bool = true;
1265
1266     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1267         run.path("rust-demangler")
1268     }
1269
1270     fn make_run(run: RunConfig<'_>) {
1271         run.builder.ensure(RustDemangler {
1272             compiler: run.builder.compiler_for(
1273                 run.builder.top_stage,
1274                 run.builder.config.build,
1275                 run.target,
1276             ),
1277             target: run.target,
1278         });
1279     }
1280
1281     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1282         let compiler = self.compiler;
1283         let target = self.target;
1284         assert!(builder.config.extended);
1285
1286         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1287         let profiler = builder.config.profiler_enabled(target);
1288         if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
1289             return None;
1290         }
1291
1292         let rust_demangler = builder
1293             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1294             .expect("rust-demangler expected to build - in-tree tool");
1295
1296         // Prepare the image directory
1297         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1298         tarball.set_overlay(OverlayKind::RustDemangler);
1299         tarball.is_preview(true);
1300         tarball.add_file(&rust_demangler, "bin", 0o755);
1301         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1302         Some(tarball.generate())
1303     }
1304 }
1305
1306 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1307 pub struct Extended {
1308     stage: u32,
1309     host: TargetSelection,
1310     target: TargetSelection,
1311 }
1312
1313 impl Step for Extended {
1314     type Output = ();
1315     const DEFAULT: bool = true;
1316     const ONLY_HOSTS: bool = true;
1317
1318     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1319         let builder = run.builder;
1320         run.path("extended").default_condition(builder.config.extended)
1321     }
1322
1323     fn make_run(run: RunConfig<'_>) {
1324         run.builder.ensure(Extended {
1325             stage: run.builder.top_stage,
1326             host: run.builder.config.build,
1327             target: run.target,
1328         });
1329     }
1330
1331     /// Creates a combined installer for the specified target in the provided stage.
1332     fn run(self, builder: &Builder<'_>) {
1333         let target = self.target;
1334         let stage = self.stage;
1335         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1336
1337         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1338
1339         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
1340         let cargo_installer = builder.ensure(Cargo { compiler, target });
1341         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1342         let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
1343         let rls_installer = builder.ensure(Rls { compiler, target });
1344         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
1345         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1346         let clippy_installer = builder.ensure(Clippy { compiler, target });
1347         let miri_installer = builder.ensure(Miri { compiler, target });
1348         let mingw_installer = builder.ensure(Mingw { host: target });
1349         let analysis_installer = builder.ensure(Analysis { compiler, target });
1350
1351         let docs_installer = builder.ensure(Docs { host: target });
1352         let std_installer = builder.ensure(Std { compiler, target });
1353
1354         let etc = builder.src.join("src/etc/installer");
1355
1356         // Avoid producing tarballs during a dry run.
1357         if builder.config.dry_run {
1358             return;
1359         }
1360
1361         // When rust-std package split from rustc, we needed to ensure that during
1362         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1363         // the std files during uninstall. To do this ensure that rustc comes
1364         // before rust-std in the list below.
1365         let mut tarballs = Vec::new();
1366         tarballs.push(rustc_installer);
1367         tarballs.push(cargo_installer);
1368         tarballs.push(clippy_installer);
1369         tarballs.extend(rust_demangler_installer.clone());
1370         tarballs.extend(rls_installer.clone());
1371         tarballs.extend(rust_analyzer_installer.clone());
1372         tarballs.extend(miri_installer.clone());
1373         tarballs.extend(rustfmt_installer.clone());
1374         tarballs.extend(llvm_tools_installer);
1375         if let Some(analysis_installer) = analysis_installer {
1376             tarballs.push(analysis_installer);
1377         }
1378         tarballs.push(std_installer.expect("missing std"));
1379         if let Some(docs_installer) = docs_installer {
1380             tarballs.push(docs_installer);
1381         }
1382         if target.contains("pc-windows-gnu") {
1383             tarballs.push(mingw_installer.unwrap());
1384         }
1385
1386         let tarball = Tarball::new(builder, "rust", &target.triple);
1387         let generated = tarball.combine(&tarballs);
1388
1389         let tmp = tmpdir(builder).join("combined-tarball");
1390         let work = generated.work_dir();
1391
1392         let mut license = String::new();
1393         license += &builder.read(&builder.src.join("COPYRIGHT"));
1394         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1395         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1396         license.push('\n');
1397         license.push('\n');
1398
1399         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1400         let mut rtf = rtf.to_string();
1401         rtf.push('\n');
1402         for line in license.lines() {
1403             rtf.push_str(line);
1404             rtf.push_str("\\line ");
1405         }
1406         rtf.push('}');
1407
1408         fn filter(contents: &str, marker: &str) -> String {
1409             let start = format!("tool-{}-start", marker);
1410             let end = format!("tool-{}-end", marker);
1411             let mut lines = Vec::new();
1412             let mut omitted = false;
1413             for line in contents.lines() {
1414                 if line.contains(&start) {
1415                     omitted = true;
1416                 } else if line.contains(&end) {
1417                     omitted = false;
1418                 } else if !omitted {
1419                     lines.push(line);
1420                 }
1421             }
1422
1423             lines.join("\n")
1424         }
1425
1426         let xform = |p: &Path| {
1427             let mut contents = t!(fs::read_to_string(p));
1428             if rust_demangler_installer.is_none() {
1429                 contents = filter(&contents, "rust-demangler");
1430             }
1431             if rls_installer.is_none() {
1432                 contents = filter(&contents, "rls");
1433             }
1434             if rust_analyzer_installer.is_none() {
1435                 contents = filter(&contents, "rust-analyzer");
1436             }
1437             if miri_installer.is_none() {
1438                 contents = filter(&contents, "miri");
1439             }
1440             if rustfmt_installer.is_none() {
1441                 contents = filter(&contents, "rustfmt");
1442             }
1443             let ret = tmp.join(p.file_name().unwrap());
1444             t!(fs::write(&ret, &contents));
1445             ret
1446         };
1447
1448         if target.contains("apple-darwin") {
1449             builder.info("building pkg installer");
1450             let pkg = tmp.join("pkg");
1451             let _ = fs::remove_dir_all(&pkg);
1452
1453             let pkgbuild = |component: &str| {
1454                 let mut cmd = Command::new("pkgbuild");
1455                 cmd.arg("--identifier")
1456                     .arg(format!("org.rust-lang.{}", component))
1457                     .arg("--scripts")
1458                     .arg(pkg.join(component))
1459                     .arg("--nopayload")
1460                     .arg(pkg.join(component).with_extension("pkg"));
1461                 builder.run(&mut cmd);
1462             };
1463
1464             let prepare = |name: &str| {
1465                 builder.create_dir(&pkg.join(name));
1466                 builder.cp_r(
1467                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1468                     &pkg.join(name),
1469                 );
1470                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1471                 pkgbuild(name);
1472             };
1473             prepare("rustc");
1474             prepare("cargo");
1475             prepare("rust-docs");
1476             prepare("rust-std");
1477             prepare("rust-analysis");
1478             prepare("clippy");
1479             if rust_demangler_installer.is_some() {
1480                 prepare("rust-demangler");
1481             }
1482             if rls_installer.is_some() {
1483                 prepare("rls");
1484             }
1485             if rust_analyzer_installer.is_some() {
1486                 prepare("rust-analyzer");
1487             }
1488             if miri_installer.is_some() {
1489                 prepare("miri");
1490             }
1491
1492             // create an 'uninstall' package
1493             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1494             pkgbuild("uninstall");
1495
1496             builder.create_dir(&pkg.join("res"));
1497             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1498             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1499             let mut cmd = Command::new("productbuild");
1500             cmd.arg("--distribution")
1501                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1502                 .arg("--resources")
1503                 .arg(pkg.join("res"))
1504                 .arg(distdir(builder).join(format!(
1505                     "{}-{}.pkg",
1506                     pkgname(builder, "rust"),
1507                     target.triple
1508                 )))
1509                 .arg("--package-path")
1510                 .arg(&pkg);
1511             let _time = timeit(builder);
1512             builder.run(&mut cmd);
1513         }
1514
1515         if target.contains("windows") {
1516             let exe = tmp.join("exe");
1517             let _ = fs::remove_dir_all(&exe);
1518
1519             let prepare = |name: &str| {
1520                 builder.create_dir(&exe.join(name));
1521                 let dir = if name == "rust-std" || name == "rust-analysis" {
1522                     format!("{}-{}", name, target.triple)
1523                 } else if name == "rls" {
1524                     "rls-preview".to_string()
1525                 } else if name == "rust-analyzer" {
1526                     "rust-analyzer-preview".to_string()
1527                 } else if name == "clippy" {
1528                     "clippy-preview".to_string()
1529                 } else if name == "rust-demangler" {
1530                     "rust-demangler-preview".to_string()
1531                 } else if name == "miri" {
1532                     "miri-preview".to_string()
1533                 } else {
1534                     name.to_string()
1535                 };
1536                 builder.cp_r(
1537                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1538                     &exe.join(name),
1539                 );
1540                 builder.remove(&exe.join(name).join("manifest.in"));
1541             };
1542             prepare("rustc");
1543             prepare("cargo");
1544             prepare("rust-analysis");
1545             prepare("rust-docs");
1546             prepare("rust-std");
1547             prepare("clippy");
1548             if rust_demangler_installer.is_some() {
1549                 prepare("rust-demangler");
1550             }
1551             if rls_installer.is_some() {
1552                 prepare("rls");
1553             }
1554             if rust_analyzer_installer.is_some() {
1555                 prepare("rust-analyzer");
1556             }
1557             if miri_installer.is_some() {
1558                 prepare("miri");
1559             }
1560             if target.contains("windows-gnu") {
1561                 prepare("rust-mingw");
1562             }
1563
1564             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1565
1566             // Generate msi installer
1567             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1568             let heat = wix.join("bin/heat.exe");
1569             let candle = wix.join("bin/candle.exe");
1570             let light = wix.join("bin/light.exe");
1571
1572             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1573             builder.run(
1574                 Command::new(&heat)
1575                     .current_dir(&exe)
1576                     .arg("dir")
1577                     .arg("rustc")
1578                     .args(&heat_flags)
1579                     .arg("-cg")
1580                     .arg("RustcGroup")
1581                     .arg("-dr")
1582                     .arg("Rustc")
1583                     .arg("-var")
1584                     .arg("var.RustcDir")
1585                     .arg("-out")
1586                     .arg(exe.join("RustcGroup.wxs")),
1587             );
1588             builder.run(
1589                 Command::new(&heat)
1590                     .current_dir(&exe)
1591                     .arg("dir")
1592                     .arg("rust-docs")
1593                     .args(&heat_flags)
1594                     .arg("-cg")
1595                     .arg("DocsGroup")
1596                     .arg("-dr")
1597                     .arg("Docs")
1598                     .arg("-var")
1599                     .arg("var.DocsDir")
1600                     .arg("-out")
1601                     .arg(exe.join("DocsGroup.wxs"))
1602                     .arg("-t")
1603                     .arg(etc.join("msi/squash-components.xsl")),
1604             );
1605             builder.run(
1606                 Command::new(&heat)
1607                     .current_dir(&exe)
1608                     .arg("dir")
1609                     .arg("cargo")
1610                     .args(&heat_flags)
1611                     .arg("-cg")
1612                     .arg("CargoGroup")
1613                     .arg("-dr")
1614                     .arg("Cargo")
1615                     .arg("-var")
1616                     .arg("var.CargoDir")
1617                     .arg("-out")
1618                     .arg(exe.join("CargoGroup.wxs"))
1619                     .arg("-t")
1620                     .arg(etc.join("msi/remove-duplicates.xsl")),
1621             );
1622             builder.run(
1623                 Command::new(&heat)
1624                     .current_dir(&exe)
1625                     .arg("dir")
1626                     .arg("rust-std")
1627                     .args(&heat_flags)
1628                     .arg("-cg")
1629                     .arg("StdGroup")
1630                     .arg("-dr")
1631                     .arg("Std")
1632                     .arg("-var")
1633                     .arg("var.StdDir")
1634                     .arg("-out")
1635                     .arg(exe.join("StdGroup.wxs")),
1636             );
1637             if rls_installer.is_some() {
1638                 builder.run(
1639                     Command::new(&heat)
1640                         .current_dir(&exe)
1641                         .arg("dir")
1642                         .arg("rls")
1643                         .args(&heat_flags)
1644                         .arg("-cg")
1645                         .arg("RlsGroup")
1646                         .arg("-dr")
1647                         .arg("Rls")
1648                         .arg("-var")
1649                         .arg("var.RlsDir")
1650                         .arg("-out")
1651                         .arg(exe.join("RlsGroup.wxs"))
1652                         .arg("-t")
1653                         .arg(etc.join("msi/remove-duplicates.xsl")),
1654                 );
1655             }
1656             if rust_analyzer_installer.is_some() {
1657                 builder.run(
1658                     Command::new(&heat)
1659                         .current_dir(&exe)
1660                         .arg("dir")
1661                         .arg("rust-analyzer")
1662                         .args(&heat_flags)
1663                         .arg("-cg")
1664                         .arg("RustAnalyzerGroup")
1665                         .arg("-dr")
1666                         .arg("RustAnalyzer")
1667                         .arg("-var")
1668                         .arg("var.RustAnalyzerDir")
1669                         .arg("-out")
1670                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1671                         .arg("-t")
1672                         .arg(etc.join("msi/remove-duplicates.xsl")),
1673                 );
1674             }
1675             builder.run(
1676                 Command::new(&heat)
1677                     .current_dir(&exe)
1678                     .arg("dir")
1679                     .arg("clippy")
1680                     .args(&heat_flags)
1681                     .arg("-cg")
1682                     .arg("ClippyGroup")
1683                     .arg("-dr")
1684                     .arg("Clippy")
1685                     .arg("-var")
1686                     .arg("var.ClippyDir")
1687                     .arg("-out")
1688                     .arg(exe.join("ClippyGroup.wxs"))
1689                     .arg("-t")
1690                     .arg(etc.join("msi/remove-duplicates.xsl")),
1691             );
1692             if rust_demangler_installer.is_some() {
1693                 builder.run(
1694                     Command::new(&heat)
1695                         .current_dir(&exe)
1696                         .arg("dir")
1697                         .arg("rust-demangler")
1698                         .args(&heat_flags)
1699                         .arg("-cg")
1700                         .arg("RustDemanglerGroup")
1701                         .arg("-dr")
1702                         .arg("RustDemangler")
1703                         .arg("-var")
1704                         .arg("var.RustDemanglerDir")
1705                         .arg("-out")
1706                         .arg(exe.join("RustDemanglerGroup.wxs"))
1707                         .arg("-t")
1708                         .arg(etc.join("msi/remove-duplicates.xsl")),
1709                 );
1710             }
1711             if miri_installer.is_some() {
1712                 builder.run(
1713                     Command::new(&heat)
1714                         .current_dir(&exe)
1715                         .arg("dir")
1716                         .arg("miri")
1717                         .args(&heat_flags)
1718                         .arg("-cg")
1719                         .arg("MiriGroup")
1720                         .arg("-dr")
1721                         .arg("Miri")
1722                         .arg("-var")
1723                         .arg("var.MiriDir")
1724                         .arg("-out")
1725                         .arg(exe.join("MiriGroup.wxs"))
1726                         .arg("-t")
1727                         .arg(etc.join("msi/remove-duplicates.xsl")),
1728                 );
1729             }
1730             builder.run(
1731                 Command::new(&heat)
1732                     .current_dir(&exe)
1733                     .arg("dir")
1734                     .arg("rust-analysis")
1735                     .args(&heat_flags)
1736                     .arg("-cg")
1737                     .arg("AnalysisGroup")
1738                     .arg("-dr")
1739                     .arg("Analysis")
1740                     .arg("-var")
1741                     .arg("var.AnalysisDir")
1742                     .arg("-out")
1743                     .arg(exe.join("AnalysisGroup.wxs"))
1744                     .arg("-t")
1745                     .arg(etc.join("msi/remove-duplicates.xsl")),
1746             );
1747             if target.contains("windows-gnu") {
1748                 builder.run(
1749                     Command::new(&heat)
1750                         .current_dir(&exe)
1751                         .arg("dir")
1752                         .arg("rust-mingw")
1753                         .args(&heat_flags)
1754                         .arg("-cg")
1755                         .arg("GccGroup")
1756                         .arg("-dr")
1757                         .arg("Gcc")
1758                         .arg("-var")
1759                         .arg("var.GccDir")
1760                         .arg("-out")
1761                         .arg(exe.join("GccGroup.wxs")),
1762                 );
1763             }
1764
1765             let candle = |input: &Path| {
1766                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1767                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1768                 let mut cmd = Command::new(&candle);
1769                 cmd.current_dir(&exe)
1770                     .arg("-nologo")
1771                     .arg("-dRustcDir=rustc")
1772                     .arg("-dDocsDir=rust-docs")
1773                     .arg("-dCargoDir=cargo")
1774                     .arg("-dStdDir=rust-std")
1775                     .arg("-dAnalysisDir=rust-analysis")
1776                     .arg("-dClippyDir=clippy")
1777                     .arg("-arch")
1778                     .arg(&arch)
1779                     .arg("-out")
1780                     .arg(&output)
1781                     .arg(&input);
1782                 add_env(builder, &mut cmd, target);
1783
1784                 if rust_demangler_installer.is_some() {
1785                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1786                 }
1787                 if rls_installer.is_some() {
1788                     cmd.arg("-dRlsDir=rls");
1789                 }
1790                 if rust_analyzer_installer.is_some() {
1791                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1792                 }
1793                 if miri_installer.is_some() {
1794                     cmd.arg("-dMiriDir=miri");
1795                 }
1796                 if target.contains("windows-gnu") {
1797                     cmd.arg("-dGccDir=rust-mingw");
1798                 }
1799                 builder.run(&mut cmd);
1800             };
1801             candle(&xform(&etc.join("msi/rust.wxs")));
1802             candle(&etc.join("msi/ui.wxs"));
1803             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1804             candle("RustcGroup.wxs".as_ref());
1805             candle("DocsGroup.wxs".as_ref());
1806             candle("CargoGroup.wxs".as_ref());
1807             candle("StdGroup.wxs".as_ref());
1808             candle("ClippyGroup.wxs".as_ref());
1809             if rust_demangler_installer.is_some() {
1810                 candle("RustDemanglerGroup.wxs".as_ref());
1811             }
1812             if rls_installer.is_some() {
1813                 candle("RlsGroup.wxs".as_ref());
1814             }
1815             if rust_analyzer_installer.is_some() {
1816                 candle("RustAnalyzerGroup.wxs".as_ref());
1817             }
1818             if miri_installer.is_some() {
1819                 candle("MiriGroup.wxs".as_ref());
1820             }
1821             candle("AnalysisGroup.wxs".as_ref());
1822
1823             if target.contains("windows-gnu") {
1824                 candle("GccGroup.wxs".as_ref());
1825             }
1826
1827             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1828             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1829             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1830
1831             builder.info(&format!("building `msi` installer with {:?}", light));
1832             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1833             let mut cmd = Command::new(&light);
1834             cmd.arg("-nologo")
1835                 .arg("-ext")
1836                 .arg("WixUIExtension")
1837                 .arg("-ext")
1838                 .arg("WixUtilExtension")
1839                 .arg("-out")
1840                 .arg(exe.join(&filename))
1841                 .arg("rust.wixobj")
1842                 .arg("ui.wixobj")
1843                 .arg("rustwelcomedlg.wixobj")
1844                 .arg("RustcGroup.wixobj")
1845                 .arg("DocsGroup.wixobj")
1846                 .arg("CargoGroup.wixobj")
1847                 .arg("StdGroup.wixobj")
1848                 .arg("AnalysisGroup.wixobj")
1849                 .arg("ClippyGroup.wixobj")
1850                 .current_dir(&exe);
1851
1852             if rls_installer.is_some() {
1853                 cmd.arg("RlsGroup.wixobj");
1854             }
1855             if rust_analyzer_installer.is_some() {
1856                 cmd.arg("RustAnalyzerGroup.wixobj");
1857             }
1858             if rust_demangler_installer.is_some() {
1859                 cmd.arg("RustDemanglerGroup.wixobj");
1860             }
1861             if miri_installer.is_some() {
1862                 cmd.arg("MiriGroup.wixobj");
1863             }
1864
1865             if target.contains("windows-gnu") {
1866                 cmd.arg("GccGroup.wixobj");
1867             }
1868             // ICE57 wrongly complains about the shortcuts
1869             cmd.arg("-sice:ICE57");
1870
1871             let _time = timeit(builder);
1872             builder.run(&mut cmd);
1873
1874             if !builder.config.dry_run {
1875                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1876             }
1877         }
1878     }
1879 }
1880
1881 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1882     let mut parts = builder.version.split('.');
1883     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1884         .env("CFG_RELEASE_NUM", &builder.version)
1885         .env("CFG_RELEASE", builder.rust_release())
1886         .env("CFG_VER_MAJOR", parts.next().unwrap())
1887         .env("CFG_VER_MINOR", parts.next().unwrap())
1888         .env("CFG_VER_PATCH", parts.next().unwrap())
1889         .env("CFG_VER_BUILD", "0") // just needed to build
1890         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1891         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1892         .env("CFG_BUILD", target.triple)
1893         .env("CFG_CHANNEL", &builder.config.channel);
1894
1895     if target.contains("windows-gnu") {
1896         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1897     } else {
1898         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1899     }
1900
1901     if target.contains("x86_64") {
1902         cmd.env("CFG_PLATFORM", "x64");
1903     } else {
1904         cmd.env("CFG_PLATFORM", "x86");
1905     }
1906 }
1907
1908 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1909 ///
1910
1911 /// Returns whether the files were actually copied.
1912 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1913     if let Some(config) = builder.config.target_config.get(&target) {
1914         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1915             // If the LLVM was externally provided, then we don't currently copy
1916             // artifacts into the sysroot. This is not necessarily the right
1917             // choice (in particular, it will require the LLVM dylib to be in
1918             // the linker's load path at runtime), but the common use case for
1919             // external LLVMs is distribution provided LLVMs, and in that case
1920             // they're usually in the standard search path (e.g., /usr/lib) and
1921             // copying them here is going to cause problems as we may end up
1922             // with the wrong files and isn't what distributions want.
1923             //
1924             // This behavior may be revisited in the future though.
1925             //
1926             // If the LLVM is coming from ourselves (just from CI) though, we
1927             // still want to install it, as it otherwise won't be available.
1928             return false;
1929         }
1930     }
1931
1932     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1933     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1934     // clear why this is the case, though. llvm-config will emit the versioned
1935     // paths and we don't want those in the sysroot (as we're expecting
1936     // unversioned paths).
1937     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1938         let src_libdir = builder.llvm_out(target).join("lib");
1939         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1940         if llvm_dylib_path.exists() {
1941             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1942         }
1943         !builder.config.dry_run
1944     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1945         let mut cmd = Command::new(llvm_config);
1946         cmd.arg("--libfiles");
1947         builder.verbose(&format!("running {:?}", cmd));
1948         let files = output(&mut cmd);
1949         for file in files.trim_end().split(' ') {
1950             builder.install(Path::new(file), dst_libdir, 0o644);
1951         }
1952         !builder.config.dry_run
1953     } else {
1954         false
1955     }
1956 }
1957
1958 /// Maybe add libLLVM.so to the target lib-dir for linking.
1959 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1960     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1961     // We do not need to copy LLVM files into the sysroot if it is not
1962     // dynamically linked; it is already included into librustc_llvm
1963     // statically.
1964     if builder.config.llvm_link_shared {
1965         maybe_install_llvm(builder, target, &dst_libdir);
1966     }
1967 }
1968
1969 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1970 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1971     let dst_libdir =
1972         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1973     // We do not need to copy LLVM files into the sysroot if it is not
1974     // dynamically linked; it is already included into librustc_llvm
1975     // statically.
1976     if builder.config.llvm_link_shared {
1977         maybe_install_llvm(builder, target, &dst_libdir);
1978     }
1979 }
1980
1981 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1982 pub struct LlvmTools {
1983     pub target: TargetSelection,
1984 }
1985
1986 impl Step for LlvmTools {
1987     type Output = Option<GeneratedTarball>;
1988     const ONLY_HOSTS: bool = true;
1989
1990     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1991         run.path("llvm-tools")
1992     }
1993
1994     fn make_run(run: RunConfig<'_>) {
1995         run.builder.ensure(LlvmTools { target: run.target });
1996     }
1997
1998     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1999         let target = self.target;
2000         assert!(builder.config.extended);
2001
2002         /* run only if llvm-config isn't used */
2003         if let Some(config) = builder.config.target_config.get(&target) {
2004             if let Some(ref _s) = config.llvm_config {
2005                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2006                 return None;
2007             }
2008         }
2009
2010         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2011         tarball.set_overlay(OverlayKind::LLVM);
2012         tarball.is_preview(true);
2013
2014         // Prepare the image directory
2015         let src_bindir = builder.llvm_out(target).join("bin");
2016         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2017         for tool in LLVM_TOOLS {
2018             let exe = src_bindir.join(exe(tool, target));
2019             tarball.add_file(&exe, &dst_bindir, 0o755);
2020         }
2021
2022         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2023         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2024         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2025         // compiler libraries.
2026         maybe_install_llvm_target(builder, target, tarball.image_dir());
2027
2028         Some(tarball.generate())
2029     }
2030 }
2031
2032 // Tarball intended for internal consumption to ease rustc/std development.
2033 //
2034 // Should not be considered stable by end users.
2035 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2036 pub struct RustDev {
2037     pub target: TargetSelection,
2038 }
2039
2040 impl Step for RustDev {
2041     type Output = Option<GeneratedTarball>;
2042     const DEFAULT: bool = true;
2043     const ONLY_HOSTS: bool = true;
2044
2045     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2046         run.path("rust-dev")
2047     }
2048
2049     fn make_run(run: RunConfig<'_>) {
2050         run.builder.ensure(RustDev { target: run.target });
2051     }
2052
2053     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2054         let target = self.target;
2055
2056         /* run only if llvm-config isn't used */
2057         if let Some(config) = builder.config.target_config.get(&target) {
2058             if let Some(ref _s) = config.llvm_config {
2059                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2060                 return None;
2061             }
2062         }
2063
2064         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2065         tarball.set_overlay(OverlayKind::LLVM);
2066
2067         let src_bindir = builder.llvm_out(target).join("bin");
2068         for bin in &[
2069             "llvm-config",
2070             "llvm-ar",
2071             "llvm-objdump",
2072             "llvm-profdata",
2073             "llvm-bcanalyzer",
2074             "llvm-cov",
2075             "llvm-dwp",
2076         ] {
2077             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2078         }
2079         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2080
2081         // Copy the include directory as well; needed mostly to build
2082         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2083         // just broadly useful to be able to link against the bundled LLVM.
2084         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2085
2086         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2087         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2088         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2089         // compiler libraries.
2090         let dst_libdir = tarball.image_dir().join("lib");
2091         maybe_install_llvm(builder, target, &dst_libdir);
2092         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2093         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2094
2095         Some(tarball.generate())
2096     }
2097 }
2098
2099 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2100 /// release process to avoid cloning the monorepo and building stuff.
2101 ///
2102 /// Should not be considered stable by end users.
2103 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2104 pub struct BuildManifest {
2105     pub target: TargetSelection,
2106 }
2107
2108 impl Step for BuildManifest {
2109     type Output = GeneratedTarball;
2110     const DEFAULT: bool = false;
2111     const ONLY_HOSTS: bool = true;
2112
2113     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2114         run.path("src/tools/build-manifest")
2115     }
2116
2117     fn make_run(run: RunConfig<'_>) {
2118         run.builder.ensure(BuildManifest { target: run.target });
2119     }
2120
2121     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2122         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2123
2124         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2125         tarball.add_file(&build_manifest, "bin", 0o755);
2126         tarball.generate()
2127     }
2128 }
2129
2130 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2131 ///
2132 /// Currently this is the PGO profile data.
2133 ///
2134 /// Should not be considered stable by end users.
2135 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2136 pub struct ReproducibleArtifacts {
2137     pub target: TargetSelection,
2138 }
2139
2140 impl Step for ReproducibleArtifacts {
2141     type Output = Option<GeneratedTarball>;
2142     const DEFAULT: bool = true;
2143     const ONLY_HOSTS: bool = true;
2144
2145     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2146         run.path("reproducible")
2147     }
2148
2149     fn make_run(run: RunConfig<'_>) {
2150         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2151     }
2152
2153     fn run(self, builder: &Builder<'_>) -> Self::Output {
2154         let path = builder.config.rust_profile_use.as_ref()?;
2155
2156         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2157         tarball.add_file(path, ".", 0o644);
2158         Some(tarball.generate())
2159     }
2160 }