src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.alias("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.alias("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.ends_with("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.alias("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.ends_with("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 for flavor in ["ld", "ld64"] {
 411                     let exe_name = exe(flavor, compiler.host);
 412                     builder
 413                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 414                 }
 415             }
 416
 417             // Man pages
 418             t!(fs::create_dir_all(image.join("share/man/man1")));
 419             let man_src = builder.src.join("src/doc/man");
 420             let man_dst = image.join("share/man/man1");
 421
 422             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 423             // to hardlink, and we don't want to edit the source templates
 424             for file_entry in builder.read_dir(&man_src) {
 425                 let page_src = file_entry.path();
 426                 let page_dst = man_dst.join(file_entry.file_name());
 427                 let src_text = t!(std::fs::read_to_string(&page_src));
 428                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 429                 t!(std::fs::write(&page_dst, &new_text));
 430                 t!(fs::copy(&page_src, &page_dst));
 431             }
 432
 433             // Debugger scripts
 434             builder
 435                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 436
 437             // Misc license info
 438             let cp = |file: &str| {
 439                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 440             };
 441             cp("COPYRIGHT");
 442             cp("LICENSE-APACHE");
 443             cp("LICENSE-MIT");
 444             cp("README.md");
 445         }
 446     }
 447 }
 448
 449 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 450 pub struct DebuggerScripts {
 451     pub sysroot: Interned<PathBuf>,
 452     pub host: TargetSelection,
 453 }
 454
 455 impl Step for DebuggerScripts {
 456     type Output = ();
 457
 458     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 459         run.never()
 460     }
 461
 462     /// Copies debugger scripts for `target` into the `sysroot` specified.
 463     fn run(self, builder: &Builder<'_>) {
 464         let host = self.host;
 465         let sysroot = self.sysroot;
 466         let dst = sysroot.join("lib/rustlib/etc");
 467         t!(fs::create_dir_all(&dst));
 468         let cp_debugger_script = |file: &str| {
 469             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 470         };
 471         if host.contains("windows-msvc") {
 472             // windbg debugger scripts
 473             builder.install(
 474                 &builder.src.join("src/etc/rust-windbg.cmd"),
 475                 &sysroot.join("bin"),
 476                 0o755,
 477             );
 478
 479             cp_debugger_script("natvis/intrinsic.natvis");
 480             cp_debugger_script("natvis/liballoc.natvis");
 481             cp_debugger_script("natvis/libcore.natvis");
 482             cp_debugger_script("natvis/libstd.natvis");
 483         } else {
 484             cp_debugger_script("rust_types.py");
 485
 486             // gdb debugger scripts
 487             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 488             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 489
 490             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 491             cp_debugger_script("gdb_lookup.py");
 492             cp_debugger_script("gdb_providers.py");
 493
 494             // lldb debugger scripts
 495             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 496
 497             cp_debugger_script("lldb_lookup.py");
 498             cp_debugger_script("lldb_providers.py");
 499             cp_debugger_script("lldb_commands")
 500         }
 501     }
 502 }
 503
 504 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 505     // The only true set of target libraries came from the build triple, so
 506     // let's reduce redundant work by only producing archives from that host.
 507     if compiler.host != builder.config.build {
 508         builder.info("\tskipping, not a build host");
 509         true
 510     } else {
 511         false
 512     }
 513 }
 514
 515 /// Copy stamped files into an image's `target/lib` directory.
 516 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 517     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 518     let self_contained_dst = dst.join("self-contained");
 519     t!(fs::create_dir_all(&dst));
 520     t!(fs::create_dir_all(&self_contained_dst));
 521     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 522         if dependency_type == DependencyType::TargetSelfContained {
 523             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 524         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 525             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 526         }
 527     }
 528 }
 529
 530 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 531 pub struct Std {
 532     pub compiler: Compiler,
 533     pub target: TargetSelection,
 534 }
 535
 536 impl Step for Std {
 537     type Output = Option<GeneratedTarball>;
 538     const DEFAULT: bool = true;
 539
 540     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 541         run.alias("rust-std")
 542     }
 543
 544     fn make_run(run: RunConfig<'_>) {
 545         run.builder.ensure(Std {
 546             compiler: run.builder.compiler_for(
 547                 run.builder.top_stage,
 548                 run.builder.config.build,
 549                 run.target,
 550             ),
 551             target: run.target,
 552         });
 553     }
 554
 555     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 556         let compiler = self.compiler;
 557         let target = self.target;
 558
 559         if skip_host_target_lib(builder, compiler) {
 560             return None;
 561         }
 562
 563         builder.ensure(compile::Std { compiler, target });
 564
 565         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 566         tarball.include_target_in_component_name(true);
 567
 568         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 569         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 570         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 571
 572         Some(tarball.generate())
 573     }
 574 }
 575
 576 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 577 pub struct RustcDev {
 578     pub compiler: Compiler,
 579     pub target: TargetSelection,
 580 }
 581
 582 impl Step for RustcDev {
 583     type Output = Option<GeneratedTarball>;
 584     const DEFAULT: bool = true;
 585     const ONLY_HOSTS: bool = true;
 586
 587     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 588         run.alias("rustc-dev")
 589     }
 590
 591     fn make_run(run: RunConfig<'_>) {
 592         run.builder.ensure(RustcDev {
 593             compiler: run.builder.compiler_for(
 594                 run.builder.top_stage,
 595                 run.builder.config.build,
 596                 run.target,
 597             ),
 598             target: run.target,
 599         });
 600     }
 601
 602     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 603         let compiler = self.compiler;
 604         let target = self.target;
 605         if skip_host_target_lib(builder, compiler) {
 606             return None;
 607         }
 608
 609         builder.ensure(compile::Rustc { compiler, target });
 610
 611         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 612
 613         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 614         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 615         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 616
 617         let src_files = &["Cargo.lock"];
 618         // This is the reduced set of paths which will become the rustc-dev component
 619         // (essentially the compiler crates and all of their path dependencies).
 620         copy_src_dirs(
 621             builder,
 622             &builder.src,
 623             &["compiler"],
 624             &[],
 625             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 626         );
 627         for file in src_files {
 628             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 629         }
 630
 631         Some(tarball.generate())
 632     }
 633 }
 634
 635 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 636 pub struct Analysis {
 637     pub compiler: Compiler,
 638     pub target: TargetSelection,
 639 }
 640
 641 impl Step for Analysis {
 642     type Output = Option<GeneratedTarball>;
 643     const DEFAULT: bool = true;
 644
 645     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 646         let default = should_build_extended_tool(&run.builder, "analysis");
 647         run.alias("rust-analysis").default_condition(default)
 648     }
 649
 650     fn make_run(run: RunConfig<'_>) {
 651         run.builder.ensure(Analysis {
 652             // Find the actual compiler (handling the full bootstrap option) which
 653             // produced the save-analysis data because that data isn't copied
 654             // through the sysroot uplifting.
 655             compiler: run.builder.compiler_for(
 656                 run.builder.top_stage,
 657                 run.builder.config.build,
 658                 run.target,
 659             ),
 660             target: run.target,
 661         });
 662     }
 663
 664     /// Creates a tarball of save-analysis metadata, if available.
 665     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 666         let compiler = self.compiler;
 667         let target = self.target;
 668         if compiler.host != builder.config.build {
 669             return None;
 670         }
 671
 672         builder.ensure(compile::Std { compiler, target });
 673         let src = builder
 674             .stage_out(compiler, Mode::Std)
 675             .join(target.triple)
 676             .join(builder.cargo_dir())
 677             .join("deps")
 678             .join("save-analysis");
 679
 680         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 681         tarball.include_target_in_component_name(true);
 682         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 683         Some(tarball.generate())
 684     }
 685 }
 686
 687 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 688 /// `dst_dir`.
 689 fn copy_src_dirs(
 690     builder: &Builder<'_>,
 691     base: &Path,
 692     src_dirs: &[&str],
 693     exclude_dirs: &[&str],
 694     dst_dir: &Path,
 695 ) {
 696     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 697         let spath = match path.to_str() {
 698             Some(path) => path,
 699             None => return false,
 700         };
 701         if spath.ends_with('~') || spath.ends_with(".pyc") {
 702             return false;
 703         }
 704
 705         const LLVM_PROJECTS: &[&str] = &[
 706             "llvm-project/clang",
 707             "llvm-project\\clang",
 708             "llvm-project/libunwind",
 709             "llvm-project\\libunwind",
 710             "llvm-project/lld",
 711             "llvm-project\\lld",
 712             "llvm-project/lldb",
 713             "llvm-project\\lldb",
 714             "llvm-project/llvm",
 715             "llvm-project\\llvm",
 716             "llvm-project/compiler-rt",
 717             "llvm-project\\compiler-rt",
 718             "llvm-project/cmake",
 719             "llvm-project\\cmake",
 720         ];
 721         if spath.contains("llvm-project")
 722             && !spath.ends_with("llvm-project")
 723             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 724         {
 725             return false;
 726         }
 727
 728         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 729         if LLVM_TEST.iter().any(|path| spath.contains(path))
 730             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 731         {
 732             return false;
 733         }
 734
 735         let full_path = Path::new(dir).join(path);
 736         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 737             return false;
 738         }
 739
 740         let excludes = [
 741             "CVS",
 742             "RCS",
 743             "SCCS",
 744             ".git",
 745             ".gitignore",
 746             ".gitmodules",
 747             ".gitattributes",
 748             ".cvsignore",
 749             ".svn",
 750             ".arch-ids",
 751             "{arch}",
 752             "=RELEASE-ID",
 753             "=meta-update",
 754             "=update",
 755             ".bzr",
 756             ".bzrignore",
 757             ".bzrtags",
 758             ".hg",
 759             ".hgignore",
 760             ".hgrags",
 761             "_darcs",
 762         ];
 763         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 764     }
 765
 766     // Copy the directories using our filter
 767     for item in src_dirs {
 768         let dst = &dst_dir.join(item);
 769         t!(fs::create_dir_all(dst));
 770         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 771     }
 772 }
 773
 774 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 775 pub struct Src;
 776
 777 impl Step for Src {
 778     /// The output path of the src installer tarball
 779     type Output = GeneratedTarball;
 780     const DEFAULT: bool = true;
 781     const ONLY_HOSTS: bool = true;
 782
 783     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 784         run.alias("rust-src")
 785     }
 786
 787     fn make_run(run: RunConfig<'_>) {
 788         run.builder.ensure(Src);
 789     }
 790
 791     /// Creates the `rust-src` installer component
 792     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 793         let tarball = Tarball::new_targetless(builder, "rust-src");
 794
 795         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 796         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 797         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 798         // and fix them...
 799         //
 800         // NOTE: if you update the paths here, you also should update the "virtual" path
 801         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 802         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 803
 804         let src_files = ["Cargo.lock"];
 805         // This is the reduced set of paths which will become the rust-src component
 806         // (essentially libstd and all of its path dependencies).
 807         copy_src_dirs(
 808             builder,
 809             &builder.src,
 810             &["library", "src/llvm-project/libunwind"],
 811             &[
 812                 // not needed and contains symlinks which rustup currently
 813                 // chokes on when unpacking.
 814                 "library/backtrace/crates",
 815                 // these are 30MB combined and aren't necessary for building
 816                 // the standard library.
 817                 "library/stdarch/Cargo.toml",
 818                 "library/stdarch/crates/stdarch-verify",
 819                 "library/stdarch/crates/intrinsic-test",
 820             ],
 821             &dst_src,
 822         );
 823         for file in src_files.iter() {
 824             builder.copy(&builder.src.join(file), &dst_src.join(file));
 825         }
 826
 827         tarball.generate()
 828     }
 829 }
 830
 831 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 832 pub struct PlainSourceTarball;
 833
 834 impl Step for PlainSourceTarball {
 835     /// Produces the location of the tarball generated
 836     type Output = GeneratedTarball;
 837     const DEFAULT: bool = true;
 838     const ONLY_HOSTS: bool = true;
 839
 840     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 841         let builder = run.builder;
 842         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 843     }
 844
 845     fn make_run(run: RunConfig<'_>) {
 846         run.builder.ensure(PlainSourceTarball);
 847     }
 848
 849     /// Creates the plain source tarball
 850     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 851         let tarball = Tarball::new(builder, "rustc", "src");
 852         let plain_dst_src = tarball.image_dir();
 853
 854         // This is the set of root paths which will become part of the source package
 855         let src_files = [
 856             "COPYRIGHT",
 857             "LICENSE-APACHE",
 858             "LICENSE-MIT",
 859             "CONTRIBUTING.md",
 860             "README.md",
 861             "RELEASES.md",
 862             "configure",
 863             "x.py",
 864             "config.toml.example",
 865             "Cargo.toml",
 866             "Cargo.lock",
 867         ];
 868         let src_dirs = ["src", "compiler", "library"];
 869
 870         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 871
 872         // Copy the files normally
 873         for item in &src_files {
 874             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 875         }
 876
 877         // Create the version file
 878         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 879         if let Some(sha) = builder.rust_sha() {
 880             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 881         }
 882
 883         // If we're building from git sources, we need to vendor a complete distribution.
 884         if builder.rust_info.is_git() {
 885             // Vendor all Cargo dependencies
 886             let mut cmd = Command::new(&builder.initial_cargo);
 887             cmd.arg("vendor")
 888                 .arg("--sync")
 889                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 890                 .arg("--sync")
 891                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 892                 .current_dir(&plain_dst_src);
 893             builder.run(&mut cmd);
 894         }
 895
 896         tarball.bare()
 897     }
 898 }
 899
 900 // We have to run a few shell scripts, which choke quite a bit on both `\`
 901 // characters and on `C:\` paths, so normalize both of them away.
 902 pub fn sanitize_sh(path: &Path) -> String {
 903     let path = path.to_str().unwrap().replace("\\", "/");
 904     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 905
 906     fn unc_to_lfs(s: &str) -> &str {
 907         s.strip_prefix("//?/").unwrap_or(s)
 908     }
 909
 910     fn change_drive(s: &str) -> Option<String> {
 911         let mut ch = s.chars();
 912         let drive = ch.next().unwrap_or('C');
 913         if ch.next() != Some(':') {
 914             return None;
 915         }
 916         if ch.next() != Some('/') {
 917             return None;
 918         }
 919         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 920     }
 921 }
 922
 923 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 924 pub struct Cargo {
 925     pub compiler: Compiler,
 926     pub target: TargetSelection,
 927 }
 928
 929 impl Step for Cargo {
 930     type Output = Option<GeneratedTarball>;
 931     const DEFAULT: bool = true;
 932     const ONLY_HOSTS: bool = true;
 933
 934     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 935         let default = should_build_extended_tool(&run.builder, "cargo");
 936         run.alias("cargo").default_condition(default)
 937     }
 938
 939     fn make_run(run: RunConfig<'_>) {
 940         run.builder.ensure(Cargo {
 941             compiler: run.builder.compiler_for(
 942                 run.builder.top_stage,
 943                 run.builder.config.build,
 944                 run.target,
 945             ),
 946             target: run.target,
 947         });
 948     }
 949
 950     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 951         let compiler = self.compiler;
 952         let target = self.target;
 953
 954         let cargo = builder.ensure(tool::Cargo { compiler, target });
 955         let src = builder.src.join("src/tools/cargo");
 956         let etc = src.join("src/etc");
 957
 958         // Prepare the image directory
 959         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 960         tarball.set_overlay(OverlayKind::Cargo);
 961
 962         tarball.add_file(&cargo, "bin", 0o755);
 963         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 964         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 965         tarball.add_dir(etc.join("man"), "share/man/man1");
 966         tarball.add_legal_and_readme_to("share/doc/cargo");
 967
 968         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 969             let dirent = dirent.expect("read dir entry");
 970             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 971                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 972             }
 973         }
 974
 975         Some(tarball.generate())
 976     }
 977 }
 978
 979 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 980 pub struct Rls {
 981     pub compiler: Compiler,
 982     pub target: TargetSelection,
 983 }
 984
 985 impl Step for Rls {
 986     type Output = Option<GeneratedTarball>;
 987     const ONLY_HOSTS: bool = true;
 988     const DEFAULT: bool = true;
 989
 990     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 991         let default = should_build_extended_tool(&run.builder, "rls");
 992         run.alias("rls").default_condition(default)
 993     }
 994
 995     fn make_run(run: RunConfig<'_>) {
 996         run.builder.ensure(Rls {
 997             compiler: run.builder.compiler_for(
 998                 run.builder.top_stage,
 999                 run.builder.config.build,
1000                 run.target,
1001             ),
1002             target: run.target,
1003         });
1004     }
1005
1006     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1007         let compiler = self.compiler;
1008         let target = self.target;
1009
1010         let rls = builder
1011             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1012             .or_else(|| {
1013                 missing_tool("RLS", builder.build.config.missing_tools);
1014                 None
1015             })?;
1016
1017         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1018         tarball.set_overlay(OverlayKind::RLS);
1019         tarball.is_preview(true);
1020         tarball.add_file(rls, "bin", 0o755);
1021         tarball.add_legal_and_readme_to("share/doc/rls");
1022         Some(tarball.generate())
1023     }
1024 }
1025
1026 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1027 pub struct RustAnalyzer {
1028     pub compiler: Compiler,
1029     pub target: TargetSelection,
1030 }
1031
1032 impl Step for RustAnalyzer {
1033     type Output = Option<GeneratedTarball>;
1034     const DEFAULT: bool = true;
1035     const ONLY_HOSTS: bool = true;
1036
1037     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1038         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1039         run.alias("rust-analyzer").default_condition(default)
1040     }
1041
1042     fn make_run(run: RunConfig<'_>) {
1043         run.builder.ensure(RustAnalyzer {
1044             compiler: run.builder.compiler_for(
1045                 run.builder.top_stage,
1046                 run.builder.config.build,
1047                 run.target,
1048             ),
1049             target: run.target,
1050         });
1051     }
1052
1053     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1054         // This prevents rust-analyzer from being built for "dist" or "install"
1055         // on the stable/beta channels. It is a nightly-only tool and should
1056         // not be included.
1057         if !builder.build.unstable_features() {
1058             return None;
1059         }
1060         let compiler = self.compiler;
1061         let target = self.target;
1062
1063         if target.contains("riscv64") {
1064             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1065             // to build. See #74813 for details.
1066             return None;
1067         }
1068
1069         let rust_analyzer = builder
1070             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1071             .expect("rust-analyzer always builds");
1072
1073         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1074         tarball.set_overlay(OverlayKind::RustAnalyzer);
1075         tarball.is_preview(true);
1076         tarball.add_file(rust_analyzer, "bin", 0o755);
1077         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1078         Some(tarball.generate())
1079     }
1080 }
1081
1082 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1083 pub struct Clippy {
1084     pub compiler: Compiler,
1085     pub target: TargetSelection,
1086 }
1087
1088 impl Step for Clippy {
1089     type Output = Option<GeneratedTarball>;
1090     const DEFAULT: bool = true;
1091     const ONLY_HOSTS: bool = true;
1092
1093     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1094         let default = should_build_extended_tool(&run.builder, "clippy");
1095         run.alias("clippy").default_condition(default)
1096     }
1097
1098     fn make_run(run: RunConfig<'_>) {
1099         run.builder.ensure(Clippy {
1100             compiler: run.builder.compiler_for(
1101                 run.builder.top_stage,
1102                 run.builder.config.build,
1103                 run.target,
1104             ),
1105             target: run.target,
1106         });
1107     }
1108
1109     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1110         let compiler = self.compiler;
1111         let target = self.target;
1112
1113         // Prepare the image directory
1114         // We expect clippy to build, because we've exited this step above if tool
1115         // state for clippy isn't testing.
1116         let clippy = builder
1117             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1118             .expect("clippy expected to build - essential tool");
1119         let cargoclippy = builder
1120             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1121             .expect("clippy expected to build - essential tool");
1122
1123         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1124         tarball.set_overlay(OverlayKind::Clippy);
1125         tarball.is_preview(true);
1126         tarball.add_file(clippy, "bin", 0o755);
1127         tarball.add_file(cargoclippy, "bin", 0o755);
1128         tarball.add_legal_and_readme_to("share/doc/clippy");
1129         Some(tarball.generate())
1130     }
1131 }
1132
1133 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1134 pub struct Miri {
1135     pub compiler: Compiler,
1136     pub target: TargetSelection,
1137 }
1138
1139 impl Step for Miri {
1140     type Output = Option<GeneratedTarball>;
1141     const DEFAULT: bool = true;
1142     const ONLY_HOSTS: bool = true;
1143
1144     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1145         let default = should_build_extended_tool(&run.builder, "miri");
1146         run.alias("miri").default_condition(default)
1147     }
1148
1149     fn make_run(run: RunConfig<'_>) {
1150         run.builder.ensure(Miri {
1151             compiler: run.builder.compiler_for(
1152                 run.builder.top_stage,
1153                 run.builder.config.build,
1154                 run.target,
1155             ),
1156             target: run.target,
1157         });
1158     }
1159
1160     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1161         // This prevents miri from being built for "dist" or "install"
1162         // on the stable/beta channels. It is a nightly-only tool and should
1163         // not be included.
1164         if !builder.build.unstable_features() {
1165             return None;
1166         }
1167         let compiler = self.compiler;
1168         let target = self.target;
1169
1170         let miri = builder
1171             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1172             .or_else(|| {
1173                 missing_tool("miri", builder.build.config.missing_tools);
1174                 None
1175             })?;
1176         let cargomiri = builder
1177             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1178             .or_else(|| {
1179                 missing_tool("cargo miri", builder.build.config.missing_tools);
1180                 None
1181             })?;
1182
1183         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1184         tarball.set_overlay(OverlayKind::Miri);
1185         tarball.is_preview(true);
1186         tarball.add_file(miri, "bin", 0o755);
1187         tarball.add_file(cargomiri, "bin", 0o755);
1188         tarball.add_legal_and_readme_to("share/doc/miri");
1189         Some(tarball.generate())
1190     }
1191 }
1192
1193 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1194 pub struct Rustfmt {
1195     pub compiler: Compiler,
1196     pub target: TargetSelection,
1197 }
1198
1199 impl Step for Rustfmt {
1200     type Output = Option<GeneratedTarball>;
1201     const DEFAULT: bool = true;
1202     const ONLY_HOSTS: bool = true;
1203
1204     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1205         let default = should_build_extended_tool(&run.builder, "rustfmt");
1206         run.alias("rustfmt").default_condition(default)
1207     }
1208
1209     fn make_run(run: RunConfig<'_>) {
1210         run.builder.ensure(Rustfmt {
1211             compiler: run.builder.compiler_for(
1212                 run.builder.top_stage,
1213                 run.builder.config.build,
1214                 run.target,
1215             ),
1216             target: run.target,
1217         });
1218     }
1219
1220     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1221         let compiler = self.compiler;
1222         let target = self.target;
1223
1224         let rustfmt = builder
1225             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1226             .or_else(|| {
1227                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1228                 None
1229             })?;
1230         let cargofmt = builder
1231             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1232             .or_else(|| {
1233                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1234                 None
1235             })?;
1236
1237         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1238         tarball.set_overlay(OverlayKind::Rustfmt);
1239         tarball.is_preview(true);
1240         tarball.add_file(rustfmt, "bin", 0o755);
1241         tarball.add_file(cargofmt, "bin", 0o755);
1242         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1243         Some(tarball.generate())
1244     }
1245 }
1246
1247 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1248 pub struct RustDemangler {
1249     pub compiler: Compiler,
1250     pub target: TargetSelection,
1251 }
1252
1253 impl Step for RustDemangler {
1254     type Output = Option<GeneratedTarball>;
1255     const DEFAULT: bool = true;
1256     const ONLY_HOSTS: bool = true;
1257
1258     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1259         // While other tools use `should_build_extended_tool` to decide whether to be run by
1260         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1261         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1262         // we run the step by default when only `extended = true`, and decide whether to actually
1263         // run it or not later.
1264         let default = run.builder.config.extended;
1265         run.alias("rust-demangler").default_condition(default)
1266     }
1267
1268     fn make_run(run: RunConfig<'_>) {
1269         run.builder.ensure(RustDemangler {
1270             compiler: run.builder.compiler_for(
1271                 run.builder.top_stage,
1272                 run.builder.config.build,
1273                 run.target,
1274             ),
1275             target: run.target,
1276         });
1277     }
1278
1279     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1280         let compiler = self.compiler;
1281         let target = self.target;
1282
1283         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1284         let condition = should_build_extended_tool(builder, "rust-demangler")
1285             || builder.config.profiler_enabled(target);
1286         if builder.config.extended && !condition {
1287             return None;
1288         }
1289
1290         let rust_demangler = builder
1291             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1292             .expect("rust-demangler expected to build - in-tree tool");
1293
1294         // Prepare the image directory
1295         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1296         tarball.set_overlay(OverlayKind::RustDemangler);
1297         tarball.is_preview(true);
1298         tarball.add_file(&rust_demangler, "bin", 0o755);
1299         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1300         Some(tarball.generate())
1301     }
1302 }
1303
1304 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1305 pub struct Extended {
1306     stage: u32,
1307     host: TargetSelection,
1308     target: TargetSelection,
1309 }
1310
1311 impl Step for Extended {
1312     type Output = ();
1313     const DEFAULT: bool = true;
1314     const ONLY_HOSTS: bool = true;
1315
1316     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1317         let builder = run.builder;
1318         run.alias("extended").default_condition(builder.config.extended)
1319     }
1320
1321     fn make_run(run: RunConfig<'_>) {
1322         run.builder.ensure(Extended {
1323             stage: run.builder.top_stage,
1324             host: run.builder.config.build,
1325             target: run.target,
1326         });
1327     }
1328
1329     /// Creates a combined installer for the specified target in the provided stage.
1330     fn run(self, builder: &Builder<'_>) {
1331         let target = self.target;
1332         let stage = self.stage;
1333         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1334
1335         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1336
1337         let mut tarballs = Vec::new();
1338         let mut built_tools = HashSet::new();
1339         macro_rules! add_component {
1340             ($name:expr => $step:expr) => {
1341                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1342                     tarballs.push(tarball);
1343                     built_tools.insert($name);
1344                 }
1345             };
1346         }
1347
1348         // When rust-std package split from rustc, we needed to ensure that during
1349         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1350         // the std files during uninstall. To do this ensure that rustc comes
1351         // before rust-std in the list below.
1352         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1353         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1354
1355         if target.ends_with("windows-gnu") {
1356             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1357         }
1358
1359         add_component!("rust-docs" => Docs { host: target });
1360         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1361         add_component!("cargo" => Cargo { compiler, target });
1362         add_component!("rustfmt" => Rustfmt { compiler, target });
1363         add_component!("rls" => Rls { compiler, target });
1364         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1365         add_component!("llvm-components" => LlvmTools { target });
1366         add_component!("clippy" => Clippy { compiler, target });
1367         add_component!("miri" => Miri { compiler, target });
1368         add_component!("analysis" => Analysis { compiler, target });
1369
1370         let etc = builder.src.join("src/etc/installer");
1371
1372         // Avoid producing tarballs during a dry run.
1373         if builder.config.dry_run {
1374             return;
1375         }
1376
1377         let tarball = Tarball::new(builder, "rust", &target.triple);
1378         let generated = tarball.combine(&tarballs);
1379
1380         let tmp = tmpdir(builder).join("combined-tarball");
1381         let work = generated.work_dir();
1382
1383         let mut license = String::new();
1384         license += &builder.read(&builder.src.join("COPYRIGHT"));
1385         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1386         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1387         license.push('\n');
1388         license.push('\n');
1389
1390         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1391         let mut rtf = rtf.to_string();
1392         rtf.push('\n');
1393         for line in license.lines() {
1394             rtf.push_str(line);
1395             rtf.push_str("\\line ");
1396         }
1397         rtf.push('}');
1398
1399         fn filter(contents: &str, marker: &str) -> String {
1400             let start = format!("tool-{}-start", marker);
1401             let end = format!("tool-{}-end", marker);
1402             let mut lines = Vec::new();
1403             let mut omitted = false;
1404             for line in contents.lines() {
1405                 if line.contains(&start) {
1406                     omitted = true;
1407                 } else if line.contains(&end) {
1408                     omitted = false;
1409                 } else if !omitted {
1410                     lines.push(line);
1411                 }
1412             }
1413
1414             lines.join("\n")
1415         }
1416
1417         let xform = |p: &Path| {
1418             let mut contents = t!(fs::read_to_string(p));
1419             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1420                 if !built_tools.contains(tool) {
1421                     contents = filter(&contents, tool);
1422                 }
1423             }
1424             let ret = tmp.join(p.file_name().unwrap());
1425             t!(fs::write(&ret, &contents));
1426             ret
1427         };
1428
1429         if target.contains("apple-darwin") {
1430             builder.info("building pkg installer");
1431             let pkg = tmp.join("pkg");
1432             let _ = fs::remove_dir_all(&pkg);
1433
1434             let pkgbuild = |component: &str| {
1435                 let mut cmd = Command::new("pkgbuild");
1436                 cmd.arg("--identifier")
1437                     .arg(format!("org.rust-lang.{}", component))
1438                     .arg("--scripts")
1439                     .arg(pkg.join(component))
1440                     .arg("--nopayload")
1441                     .arg(pkg.join(component).with_extension("pkg"));
1442                 builder.run(&mut cmd);
1443             };
1444
1445             let prepare = |name: &str| {
1446                 builder.create_dir(&pkg.join(name));
1447                 builder.cp_r(
1448                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1449                     &pkg.join(name),
1450                 );
1451                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1452                 pkgbuild(name);
1453             };
1454             prepare("rustc");
1455             prepare("cargo");
1456             prepare("rust-std");
1457             prepare("rust-analysis");
1458             prepare("clippy");
1459             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1460                 if built_tools.contains(tool) {
1461                     prepare(tool);
1462                 }
1463             }
1464             // create an 'uninstall' package
1465             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1466             pkgbuild("uninstall");
1467
1468             builder.create_dir(&pkg.join("res"));
1469             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1470             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1471             let mut cmd = Command::new("productbuild");
1472             cmd.arg("--distribution")
1473                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1474                 .arg("--resources")
1475                 .arg(pkg.join("res"))
1476                 .arg(distdir(builder).join(format!(
1477                     "{}-{}.pkg",
1478                     pkgname(builder, "rust"),
1479                     target.triple
1480                 )))
1481                 .arg("--package-path")
1482                 .arg(&pkg);
1483             let _time = timeit(builder);
1484             builder.run(&mut cmd);
1485         }
1486
1487         if target.contains("windows") {
1488             let exe = tmp.join("exe");
1489             let _ = fs::remove_dir_all(&exe);
1490
1491             let prepare = |name: &str| {
1492                 builder.create_dir(&exe.join(name));
1493                 let dir = if name == "rust-std" || name == "rust-analysis" {
1494                     format!("{}-{}", name, target.triple)
1495                 } else if name == "rls" {
1496                     "rls-preview".to_string()
1497                 } else if name == "rust-analyzer" {
1498                     "rust-analyzer-preview".to_string()
1499                 } else if name == "clippy" {
1500                     "clippy-preview".to_string()
1501                 } else if name == "rust-demangler" {
1502                     "rust-demangler-preview".to_string()
1503                 } else if name == "miri" {
1504                     "miri-preview".to_string()
1505                 } else {
1506                     name.to_string()
1507                 };
1508                 builder.cp_r(
1509                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1510                     &exe.join(name),
1511                 );
1512                 builder.remove(&exe.join(name).join("manifest.in"));
1513             };
1514             prepare("rustc");
1515             prepare("cargo");
1516             prepare("rust-analysis");
1517             prepare("rust-docs");
1518             prepare("rust-std");
1519             prepare("clippy");
1520             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1521                 if built_tools.contains(tool) {
1522                     prepare(tool);
1523                 }
1524             }
1525             if target.ends_with("windows-gnu") {
1526                 prepare("rust-mingw");
1527             }
1528
1529             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1530
1531             // Generate msi installer
1532             let wix_path = env::var_os("WIX")
1533                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1534             let wix = PathBuf::from(wix_path);
1535             let heat = wix.join("bin/heat.exe");
1536             let candle = wix.join("bin/candle.exe");
1537             let light = wix.join("bin/light.exe");
1538
1539             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1540             builder.run(
1541                 Command::new(&heat)
1542                     .current_dir(&exe)
1543                     .arg("dir")
1544                     .arg("rustc")
1545                     .args(&heat_flags)
1546                     .arg("-cg")
1547                     .arg("RustcGroup")
1548                     .arg("-dr")
1549                     .arg("Rustc")
1550                     .arg("-var")
1551                     .arg("var.RustcDir")
1552                     .arg("-out")
1553                     .arg(exe.join("RustcGroup.wxs")),
1554             );
1555             builder.run(
1556                 Command::new(&heat)
1557                     .current_dir(&exe)
1558                     .arg("dir")
1559                     .arg("rust-docs")
1560                     .args(&heat_flags)
1561                     .arg("-cg")
1562                     .arg("DocsGroup")
1563                     .arg("-dr")
1564                     .arg("Docs")
1565                     .arg("-var")
1566                     .arg("var.DocsDir")
1567                     .arg("-out")
1568                     .arg(exe.join("DocsGroup.wxs"))
1569                     .arg("-t")
1570                     .arg(etc.join("msi/squash-components.xsl")),
1571             );
1572             builder.run(
1573                 Command::new(&heat)
1574                     .current_dir(&exe)
1575                     .arg("dir")
1576                     .arg("cargo")
1577                     .args(&heat_flags)
1578                     .arg("-cg")
1579                     .arg("CargoGroup")
1580                     .arg("-dr")
1581                     .arg("Cargo")
1582                     .arg("-var")
1583                     .arg("var.CargoDir")
1584                     .arg("-out")
1585                     .arg(exe.join("CargoGroup.wxs"))
1586                     .arg("-t")
1587                     .arg(etc.join("msi/remove-duplicates.xsl")),
1588             );
1589             builder.run(
1590                 Command::new(&heat)
1591                     .current_dir(&exe)
1592                     .arg("dir")
1593                     .arg("rust-std")
1594                     .args(&heat_flags)
1595                     .arg("-cg")
1596                     .arg("StdGroup")
1597                     .arg("-dr")
1598                     .arg("Std")
1599                     .arg("-var")
1600                     .arg("var.StdDir")
1601                     .arg("-out")
1602                     .arg(exe.join("StdGroup.wxs")),
1603             );
1604             if built_tools.contains("rls") {
1605                 builder.run(
1606                     Command::new(&heat)
1607                         .current_dir(&exe)
1608                         .arg("dir")
1609                         .arg("rls")
1610                         .args(&heat_flags)
1611                         .arg("-cg")
1612                         .arg("RlsGroup")
1613                         .arg("-dr")
1614                         .arg("Rls")
1615                         .arg("-var")
1616                         .arg("var.RlsDir")
1617                         .arg("-out")
1618                         .arg(exe.join("RlsGroup.wxs"))
1619                         .arg("-t")
1620                         .arg(etc.join("msi/remove-duplicates.xsl")),
1621                 );
1622             }
1623             if built_tools.contains("rust-analyzer") {
1624                 builder.run(
1625                     Command::new(&heat)
1626                         .current_dir(&exe)
1627                         .arg("dir")
1628                         .arg("rust-analyzer")
1629                         .args(&heat_flags)
1630                         .arg("-cg")
1631                         .arg("RustAnalyzerGroup")
1632                         .arg("-dr")
1633                         .arg("RustAnalyzer")
1634                         .arg("-var")
1635                         .arg("var.RustAnalyzerDir")
1636                         .arg("-out")
1637                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1638                         .arg("-t")
1639                         .arg(etc.join("msi/remove-duplicates.xsl")),
1640                 );
1641             }
1642             builder.run(
1643                 Command::new(&heat)
1644                     .current_dir(&exe)
1645                     .arg("dir")
1646                     .arg("clippy")
1647                     .args(&heat_flags)
1648                     .arg("-cg")
1649                     .arg("ClippyGroup")
1650                     .arg("-dr")
1651                     .arg("Clippy")
1652                     .arg("-var")
1653                     .arg("var.ClippyDir")
1654                     .arg("-out")
1655                     .arg(exe.join("ClippyGroup.wxs"))
1656                     .arg("-t")
1657                     .arg(etc.join("msi/remove-duplicates.xsl")),
1658             );
1659             if built_tools.contains("rust-demangler") {
1660                 builder.run(
1661                     Command::new(&heat)
1662                         .current_dir(&exe)
1663                         .arg("dir")
1664                         .arg("rust-demangler")
1665                         .args(&heat_flags)
1666                         .arg("-cg")
1667                         .arg("RustDemanglerGroup")
1668                         .arg("-dr")
1669                         .arg("RustDemangler")
1670                         .arg("-var")
1671                         .arg("var.RustDemanglerDir")
1672                         .arg("-out")
1673                         .arg(exe.join("RustDemanglerGroup.wxs"))
1674                         .arg("-t")
1675                         .arg(etc.join("msi/remove-duplicates.xsl")),
1676                 );
1677             }
1678             if built_tools.contains("miri") {
1679                 builder.run(
1680                     Command::new(&heat)
1681                         .current_dir(&exe)
1682                         .arg("dir")
1683                         .arg("miri")
1684                         .args(&heat_flags)
1685                         .arg("-cg")
1686                         .arg("MiriGroup")
1687                         .arg("-dr")
1688                         .arg("Miri")
1689                         .arg("-var")
1690                         .arg("var.MiriDir")
1691                         .arg("-out")
1692                         .arg(exe.join("MiriGroup.wxs"))
1693                         .arg("-t")
1694                         .arg(etc.join("msi/remove-duplicates.xsl")),
1695                 );
1696             }
1697             builder.run(
1698                 Command::new(&heat)
1699                     .current_dir(&exe)
1700                     .arg("dir")
1701                     .arg("rust-analysis")
1702                     .args(&heat_flags)
1703                     .arg("-cg")
1704                     .arg("AnalysisGroup")
1705                     .arg("-dr")
1706                     .arg("Analysis")
1707                     .arg("-var")
1708                     .arg("var.AnalysisDir")
1709                     .arg("-out")
1710                     .arg(exe.join("AnalysisGroup.wxs"))
1711                     .arg("-t")
1712                     .arg(etc.join("msi/remove-duplicates.xsl")),
1713             );
1714             if target.ends_with("windows-gnu") {
1715                 builder.run(
1716                     Command::new(&heat)
1717                         .current_dir(&exe)
1718                         .arg("dir")
1719                         .arg("rust-mingw")
1720                         .args(&heat_flags)
1721                         .arg("-cg")
1722                         .arg("GccGroup")
1723                         .arg("-dr")
1724                         .arg("Gcc")
1725                         .arg("-var")
1726                         .arg("var.GccDir")
1727                         .arg("-out")
1728                         .arg(exe.join("GccGroup.wxs")),
1729                 );
1730             }
1731
1732             let candle = |input: &Path| {
1733                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1734                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1735                 let mut cmd = Command::new(&candle);
1736                 cmd.current_dir(&exe)
1737                     .arg("-nologo")
1738                     .arg("-dRustcDir=rustc")
1739                     .arg("-dDocsDir=rust-docs")
1740                     .arg("-dCargoDir=cargo")
1741                     .arg("-dStdDir=rust-std")
1742                     .arg("-dAnalysisDir=rust-analysis")
1743                     .arg("-dClippyDir=clippy")
1744                     .arg("-arch")
1745                     .arg(&arch)
1746                     .arg("-out")
1747                     .arg(&output)
1748                     .arg(&input);
1749                 add_env(builder, &mut cmd, target);
1750
1751                 if built_tools.contains("rust-demangler") {
1752                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1753                 }
1754                 if built_tools.contains("rls") {
1755                     cmd.arg("-dRlsDir=rls");
1756                 }
1757                 if built_tools.contains("rust-analyzer") {
1758                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1759                 }
1760                 if built_tools.contains("miri") {
1761                     cmd.arg("-dMiriDir=miri");
1762                 }
1763                 if target.ends_with("windows-gnu") {
1764                     cmd.arg("-dGccDir=rust-mingw");
1765                 }
1766                 builder.run(&mut cmd);
1767             };
1768             candle(&xform(&etc.join("msi/rust.wxs")));
1769             candle(&etc.join("msi/ui.wxs"));
1770             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1771             candle("RustcGroup.wxs".as_ref());
1772             candle("DocsGroup.wxs".as_ref());
1773             candle("CargoGroup.wxs".as_ref());
1774             candle("StdGroup.wxs".as_ref());
1775             candle("ClippyGroup.wxs".as_ref());
1776             if built_tools.contains("rust-demangler") {
1777                 candle("RustDemanglerGroup.wxs".as_ref());
1778             }
1779             if built_tools.contains("rls") {
1780                 candle("RlsGroup.wxs".as_ref());
1781             }
1782             if built_tools.contains("rust-analyzer") {
1783                 candle("RustAnalyzerGroup.wxs".as_ref());
1784             }
1785             if built_tools.contains("miri") {
1786                 candle("MiriGroup.wxs".as_ref());
1787             }
1788             candle("AnalysisGroup.wxs".as_ref());
1789
1790             if target.ends_with("windows-gnu") {
1791                 candle("GccGroup.wxs".as_ref());
1792             }
1793
1794             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1795             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1796             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1797
1798             builder.info(&format!("building `msi` installer with {:?}", light));
1799             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1800             let mut cmd = Command::new(&light);
1801             cmd.arg("-nologo")
1802                 .arg("-ext")
1803                 .arg("WixUIExtension")
1804                 .arg("-ext")
1805                 .arg("WixUtilExtension")
1806                 .arg("-out")
1807                 .arg(exe.join(&filename))
1808                 .arg("rust.wixobj")
1809                 .arg("ui.wixobj")
1810                 .arg("rustwelcomedlg.wixobj")
1811                 .arg("RustcGroup.wixobj")
1812                 .arg("DocsGroup.wixobj")
1813                 .arg("CargoGroup.wixobj")
1814                 .arg("StdGroup.wixobj")
1815                 .arg("AnalysisGroup.wixobj")
1816                 .arg("ClippyGroup.wixobj")
1817                 .current_dir(&exe);
1818
1819             if built_tools.contains("rls") {
1820                 cmd.arg("RlsGroup.wixobj");
1821             }
1822             if built_tools.contains("rust-analyzer") {
1823                 cmd.arg("RustAnalyzerGroup.wixobj");
1824             }
1825             if built_tools.contains("rust-demangler") {
1826                 cmd.arg("RustDemanglerGroup.wixobj");
1827             }
1828             if built_tools.contains("miri") {
1829                 cmd.arg("MiriGroup.wixobj");
1830             }
1831
1832             if target.ends_with("windows-gnu") {
1833                 cmd.arg("GccGroup.wixobj");
1834             }
1835             // ICE57 wrongly complains about the shortcuts
1836             cmd.arg("-sice:ICE57");
1837
1838             let _time = timeit(builder);
1839             builder.run(&mut cmd);
1840
1841             if !builder.config.dry_run {
1842                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1843             }
1844         }
1845     }
1846 }
1847
1848 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1849     let mut parts = builder.version.split('.');
1850     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1851         .env("CFG_RELEASE_NUM", &builder.version)
1852         .env("CFG_RELEASE", builder.rust_release())
1853         .env("CFG_VER_MAJOR", parts.next().unwrap())
1854         .env("CFG_VER_MINOR", parts.next().unwrap())
1855         .env("CFG_VER_PATCH", parts.next().unwrap())
1856         .env("CFG_VER_BUILD", "0") // just needed to build
1857         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1858         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1859         .env("CFG_BUILD", target.triple)
1860         .env("CFG_CHANNEL", &builder.config.channel);
1861
1862     if target.contains("windows-gnullvm") {
1863         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1864     } else if target.contains("windows-gnu") {
1865         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1866     } else {
1867         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1868     }
1869 }
1870
1871 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1872 ///
1873
1874 /// Returns whether the files were actually copied.
1875 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1876     if let Some(config) = builder.config.target_config.get(&target) {
1877         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1878             // If the LLVM was externally provided, then we don't currently copy
1879             // artifacts into the sysroot. This is not necessarily the right
1880             // choice (in particular, it will require the LLVM dylib to be in
1881             // the linker's load path at runtime), but the common use case for
1882             // external LLVMs is distribution provided LLVMs, and in that case
1883             // they're usually in the standard search path (e.g., /usr/lib) and
1884             // copying them here is going to cause problems as we may end up
1885             // with the wrong files and isn't what distributions want.
1886             //
1887             // This behavior may be revisited in the future though.
1888             //
1889             // If the LLVM is coming from ourselves (just from CI) though, we
1890             // still want to install it, as it otherwise won't be available.
1891             return false;
1892         }
1893     }
1894
1895     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1896     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1897     // clear why this is the case, though. llvm-config will emit the versioned
1898     // paths and we don't want those in the sysroot (as we're expecting
1899     // unversioned paths).
1900     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1901         let src_libdir = builder.llvm_out(target).join("lib");
1902         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1903         if llvm_dylib_path.exists() {
1904             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1905         }
1906         !builder.config.dry_run
1907     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1908         let mut cmd = Command::new(llvm_config);
1909         cmd.arg("--libfiles");
1910         builder.verbose(&format!("running {:?}", cmd));
1911         let files = output(&mut cmd);
1912         let build_llvm_out = &builder.llvm_out(builder.config.build);
1913         let target_llvm_out = &builder.llvm_out(target);
1914         for file in files.trim_end().split(' ') {
1915             // If we're not using a custom LLVM, make sure we package for the target.
1916             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1917                 target_llvm_out.join(relative_path)
1918             } else {
1919                 PathBuf::from(file)
1920             };
1921             builder.install(&file, dst_libdir, 0o644);
1922         }
1923         !builder.config.dry_run
1924     } else {
1925         false
1926     }
1927 }
1928
1929 /// Maybe add libLLVM.so to the target lib-dir for linking.
1930 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1931     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1932     // We do not need to copy LLVM files into the sysroot if it is not
1933     // dynamically linked; it is already included into librustc_llvm
1934     // statically.
1935     if builder.llvm_link_shared() {
1936         maybe_install_llvm(builder, target, &dst_libdir);
1937     }
1938 }
1939
1940 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1941 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1942     let dst_libdir =
1943         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1944     // We do not need to copy LLVM files into the sysroot if it is not
1945     // dynamically linked; it is already included into librustc_llvm
1946     // statically.
1947     if builder.llvm_link_shared() {
1948         maybe_install_llvm(builder, target, &dst_libdir);
1949     }
1950 }
1951
1952 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1953 pub struct LlvmTools {
1954     pub target: TargetSelection,
1955 }
1956
1957 impl Step for LlvmTools {
1958     type Output = Option<GeneratedTarball>;
1959     const ONLY_HOSTS: bool = true;
1960     const DEFAULT: bool = true;
1961
1962     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1963         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1964         // FIXME: allow using the names of the tools themselves?
1965         run.alias("llvm-tools").default_condition(default)
1966     }
1967
1968     fn make_run(run: RunConfig<'_>) {
1969         run.builder.ensure(LlvmTools { target: run.target });
1970     }
1971
1972     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1973         let target = self.target;
1974
1975         /* run only if llvm-config isn't used */
1976         if let Some(config) = builder.config.target_config.get(&target) {
1977             if let Some(ref _s) = config.llvm_config {
1978                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1979                 return None;
1980             }
1981         }
1982
1983         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1984         tarball.set_overlay(OverlayKind::LLVM);
1985         tarball.is_preview(true);
1986
1987         // Prepare the image directory
1988         let src_bindir = builder.llvm_out(target).join("bin");
1989         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1990         for tool in LLVM_TOOLS {
1991             let exe = src_bindir.join(exe(tool, target));
1992             tarball.add_file(&exe, &dst_bindir, 0o755);
1993         }
1994
1995         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1996         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1997         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1998         // compiler libraries.
1999         maybe_install_llvm_target(builder, target, tarball.image_dir());
2000
2001         Some(tarball.generate())
2002     }
2003 }
2004
2005 // Tarball intended for internal consumption to ease rustc/std development.
2006 //
2007 // Should not be considered stable by end users.
2008 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2009 pub struct RustDev {
2010     pub target: TargetSelection,
2011 }
2012
2013 impl Step for RustDev {
2014     type Output = Option<GeneratedTarball>;
2015     const DEFAULT: bool = true;
2016     const ONLY_HOSTS: bool = true;
2017
2018     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2019         run.alias("rust-dev")
2020     }
2021
2022     fn make_run(run: RunConfig<'_>) {
2023         run.builder.ensure(RustDev { target: run.target });
2024     }
2025
2026     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2027         let target = self.target;
2028
2029         /* run only if llvm-config isn't used */
2030         if let Some(config) = builder.config.target_config.get(&target) {
2031             if let Some(ref _s) = config.llvm_config {
2032                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2033                 return None;
2034             }
2035         }
2036
2037         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2038         tarball.set_overlay(OverlayKind::LLVM);
2039
2040         let src_bindir = builder.llvm_out(target).join("bin");
2041         // If updating this list, you likely want to change
2042         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2043         // will not pick up the extra file until LLVM gets bumped.
2044         for bin in &[
2045             "llvm-config",
2046             "llvm-ar",
2047             "llvm-objdump",
2048             "llvm-profdata",
2049             "llvm-bcanalyzer",
2050             "llvm-cov",
2051             "llvm-dwp",
2052             "llvm-nm",
2053             "llvm-dwarfdump",
2054         ] {
2055             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2056         }
2057
2058         // We don't build LLD on some platforms, so only add it if it exists
2059         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2060         if lld_path.exists() {
2061             tarball.add_file(lld_path, "bin", 0o755);
2062         }
2063
2064         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2065
2066         // Copy the include directory as well; needed mostly to build
2067         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2068         // just broadly useful to be able to link against the bundled LLVM.
2069         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2070
2071         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2072         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2073         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2074         // compiler libraries.
2075         let dst_libdir = tarball.image_dir().join("lib");
2076         maybe_install_llvm(builder, target, &dst_libdir);
2077         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2078         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2079
2080         Some(tarball.generate())
2081     }
2082 }
2083
2084 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2085 /// release process to avoid cloning the monorepo and building stuff.
2086 ///
2087 /// Should not be considered stable by end users.
2088 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2089 pub struct BuildManifest {
2090     pub target: TargetSelection,
2091 }
2092
2093 impl Step for BuildManifest {
2094     type Output = GeneratedTarball;
2095     const DEFAULT: bool = false;
2096     const ONLY_HOSTS: bool = true;
2097
2098     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2099         run.alias("build-manifest")
2100     }
2101
2102     fn make_run(run: RunConfig<'_>) {
2103         run.builder.ensure(BuildManifest { target: run.target });
2104     }
2105
2106     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2107         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2108
2109         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2110         tarball.add_file(&build_manifest, "bin", 0o755);
2111         tarball.generate()
2112     }
2113 }
2114
2115 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2116 ///
2117 /// Currently this is the PGO profile data.
2118 ///
2119 /// Should not be considered stable by end users.
2120 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2121 pub struct ReproducibleArtifacts {
2122     pub target: TargetSelection,
2123 }
2124
2125 impl Step for ReproducibleArtifacts {
2126     type Output = Option<GeneratedTarball>;
2127     const DEFAULT: bool = true;
2128     const ONLY_HOSTS: bool = true;
2129
2130     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2131         run.alias("reproducible-artifacts")
2132     }
2133
2134     fn make_run(run: RunConfig<'_>) {
2135         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2136     }
2137
2138     fn run(self, builder: &Builder<'_>) -> Self::Output {
2139         let mut added_anything = false;
2140         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2141         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2142             tarball.add_file(path, ".", 0o644);
2143             added_anything = true;
2144         }
2145         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2146             tarball.add_file(path, ".", 0o644);
2147             added_anything = true;
2148         }
2149         if added_anything { Some(tarball.generate()) } else { None }
2150     }
2151 }