src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::build_helper::{output, t};
  18 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  19 use crate::cache::{Interned, INTERNER};
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  23 use crate::tool::{self, Tool};
  24 use crate::util::{exe, is_dylib, timeit};
  25 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  26
  27 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  28     format!("{}-{}", component, builder.rust_package_vers())
  29 }
  30
  31 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  32     builder.out.join("dist")
  33 }
  34
  35 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  36     builder.out.join("tmp/dist")
  37 }
  38
  39 fn missing_tool(tool_name: &str, skip: bool) {
  40     if skip {
  41         println!("Unable to build {}, skipping dist", tool_name)
  42     } else {
  43         panic!("Unable to build {}", tool_name)
  44     }
  45 }
  46
  47 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  48     if !builder.config.extended {
  49         return false;
  50     }
  51     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  52 }
  53
  54 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  55 pub struct Docs {
  56     pub host: TargetSelection,
  57 }
  58
  59 impl Step for Docs {
  60     type Output = Option<GeneratedTarball>;
  61     const DEFAULT: bool = true;
  62
  63     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  64         let default = run.builder.config.docs;
  65         run.path("rust-docs").default_condition(default)
  66     }
  67
  68     fn make_run(run: RunConfig<'_>) {
  69         run.builder.ensure(Docs { host: run.target });
  70     }
  71
  72     /// Builds the `rust-docs` installer component.
  73     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  74         let host = self.host;
  75         builder.default_doc(&[]);
  76
  77         let dest = "share/doc/rust/html";
  78
  79         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  80         tarball.set_product_name("Rust Documentation");
  81         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  82         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  83         Some(tarball.generate())
  84     }
  85 }
  86
  87 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  88 pub struct RustcDocs {
  89     pub host: TargetSelection,
  90 }
  91
  92 impl Step for RustcDocs {
  93     type Output = Option<GeneratedTarball>;
  94     const DEFAULT: bool = true;
  95
  96     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  97         let builder = run.builder;
  98         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
  99     }
 100
 101     fn make_run(run: RunConfig<'_>) {
 102         run.builder.ensure(RustcDocs { host: run.target });
 103     }
 104
 105     /// Builds the `rustc-docs` installer component.
 106     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 107         let host = self.host;
 108         builder.default_doc(&[]);
 109
 110         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 111         tarball.set_product_name("Rustc Documentation");
 112         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 113         Some(tarball.generate())
 114     }
 115 }
 116
 117 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 118     let mut found = Vec::with_capacity(files.len());
 119
 120     for file in files {
 121         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 122
 123         if let Some(file_path) = file_path {
 124             found.push(file_path);
 125         } else {
 126             panic!("Could not find '{}' in {:?}", file, path);
 127         }
 128     }
 129
 130     found
 131 }
 132
 133 fn make_win_dist(
 134     rust_root: &Path,
 135     plat_root: &Path,
 136     target: TargetSelection,
 137     builder: &Builder<'_>,
 138 ) {
 139     //Ask gcc where it keeps its stuff
 140     let mut cmd = Command::new(builder.cc(target));
 141     cmd.arg("-print-search-dirs");
 142     let gcc_out = output(&mut cmd);
 143
 144     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 145     let mut lib_path = Vec::new();
 146
 147     for line in gcc_out.lines() {
 148         let idx = line.find(':').unwrap();
 149         let key = &line[..idx];
 150         let trim_chars: &[_] = &[' ', '='];
 151         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 152
 153         if key == "programs" {
 154             bin_path.extend(value);
 155         } else if key == "libraries" {
 156             lib_path.extend(value);
 157         }
 158     }
 159
 160     let compiler = if target == "i686-pc-windows-gnu" {
 161         "i686-w64-mingw32-gcc.exe"
 162     } else if target == "x86_64-pc-windows-gnu" {
 163         "x86_64-w64-mingw32-gcc.exe"
 164     } else {
 165         "gcc.exe"
 166     };
 167     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 168     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 169     if target.starts_with("i686-") {
 170         rustc_dlls.push("libgcc_s_dw2-1.dll");
 171     } else {
 172         rustc_dlls.push("libgcc_s_seh-1.dll");
 173     }
 174
 175     let target_libs = [
 176         //MinGW libs
 177         "libgcc.a",
 178         "libgcc_eh.a",
 179         "libgcc_s.a",
 180         "libm.a",
 181         "libmingw32.a",
 182         "libmingwex.a",
 183         "libstdc++.a",
 184         "libiconv.a",
 185         "libmoldname.a",
 186         "libpthread.a",
 187         //Windows import libs
 188         "libadvapi32.a",
 189         "libbcrypt.a",
 190         "libcomctl32.a",
 191         "libcomdlg32.a",
 192         "libcredui.a",
 193         "libcrypt32.a",
 194         "libdbghelp.a",
 195         "libgdi32.a",
 196         "libimagehlp.a",
 197         "libiphlpapi.a",
 198         "libkernel32.a",
 199         "libmsimg32.a",
 200         "libmsvcrt.a",
 201         "libodbc32.a",
 202         "libole32.a",
 203         "liboleaut32.a",
 204         "libopengl32.a",
 205         "libpsapi.a",
 206         "librpcrt4.a",
 207         "libsecur32.a",
 208         "libsetupapi.a",
 209         "libshell32.a",
 210         "libsynchronization.a",
 211         "libuser32.a",
 212         "libuserenv.a",
 213         "libuuid.a",
 214         "libwinhttp.a",
 215         "libwinmm.a",
 216         "libwinspool.a",
 217         "libws2_32.a",
 218         "libwsock32.a",
 219     ];
 220
 221     //Find mingw artifacts we want to bundle
 222     let target_tools = find_files(&target_tools, &bin_path);
 223     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 224     let target_libs = find_files(&target_libs, &lib_path);
 225
 226     // Copy runtime dlls next to rustc.exe
 227     let dist_bin_dir = rust_root.join("bin/");
 228     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 229     for src in rustc_dlls {
 230         builder.copy_to_folder(&src, &dist_bin_dir);
 231     }
 232
 233     //Copy platform tools to platform-specific bin directory
 234     let target_bin_dir = plat_root
 235         .join("lib")
 236         .join("rustlib")
 237         .join(target.triple)
 238         .join("bin")
 239         .join("self-contained");
 240     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 241     for src in target_tools {
 242         builder.copy_to_folder(&src, &target_bin_dir);
 243     }
 244
 245     // Warn windows-gnu users that the bundled GCC cannot compile C files
 246     builder.create(
 247         &target_bin_dir.join("GCC-WARNING.txt"),
 248         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 249          used as a linker. In order to be able to compile projects containing C code use \
 250          the GCC provided by MinGW or Cygwin.",
 251     );
 252
 253     //Copy platform libs to platform-specific lib directory
 254     let target_lib_dir = plat_root
 255         .join("lib")
 256         .join("rustlib")
 257         .join(target.triple)
 258         .join("lib")
 259         .join("self-contained");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         builder.copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     pub host: TargetSelection,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<GeneratedTarball>;
 273     const DEFAULT: bool = true;
 274
 275     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 276         run.path("rust-mingw")
 277     }
 278
 279     fn make_run(run: RunConfig<'_>) {
 280         run.builder.ensure(Mingw { host: run.target });
 281     }
 282
 283     /// Builds the `rust-mingw` installer component.
 284     ///
 285     /// This contains all the bits and pieces to run the MinGW Windows targets
 286     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 287     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 288         let host = self.host;
 289         if !host.contains("pc-windows-gnu") {
 290             return None;
 291         }
 292
 293         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 294         tarball.set_product_name("Rust MinGW");
 295
 296         // The first argument is a "temporary directory" which is just
 297         // thrown away (this contains the runtime DLLs included in the rustc package
 298         // above) and the second argument is where to place all the MinGW components
 299         // (which is what we want).
 300         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 301
 302         Some(tarball.generate())
 303     }
 304 }
 305
 306 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 307 pub struct Rustc {
 308     pub compiler: Compiler,
 309 }
 310
 311 impl Step for Rustc {
 312     type Output = GeneratedTarball;
 313     const DEFAULT: bool = true;
 314     const ONLY_HOSTS: bool = true;
 315
 316     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 317         run.path("rustc")
 318     }
 319
 320     fn make_run(run: RunConfig<'_>) {
 321         run.builder
 322             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 323     }
 324
 325     /// Creates the `rustc` installer component.
 326     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 327         let compiler = self.compiler;
 328         let host = self.compiler.host;
 329
 330         let tarball = Tarball::new(builder, "rustc", &host.triple);
 331
 332         // Prepare the rustc "image", what will actually end up getting installed
 333         prepare_image(builder, compiler, tarball.image_dir());
 334
 335         // On MinGW we've got a few runtime DLL dependencies that we need to
 336         // include. The first argument to this script is where to put these DLLs
 337         // (the image we're creating), and the second argument is a junk directory
 338         // to ignore all other MinGW stuff the script creates.
 339         //
 340         // On 32-bit MinGW we're always including a DLL which needs some extra
 341         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 342         // anything requiring us to distribute a license, but it's likely the
 343         // install will *also* include the rust-mingw package, which also needs
 344         // licenses, so to be safe we just include it here in all MinGW packages.
 345         if host.contains("pc-windows-gnu") {
 346             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 347             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 348         }
 349
 350         return tarball.generate();
 351
 352         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 353             let host = compiler.host;
 354             let src = builder.sysroot(compiler);
 355
 356             // Copy rustc/rustdoc binaries
 357             t!(fs::create_dir_all(image.join("bin")));
 358             builder.cp_r(&src.join("bin"), &image.join("bin"));
 359
 360             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 361
 362             let libdir_relative = builder.libdir_relative(compiler);
 363
 364             // Copy runtime DLLs needed by the compiler
 365             if libdir_relative.to_str() != Some("bin") {
 366                 let libdir = builder.rustc_libdir(compiler);
 367                 for entry in builder.read_dir(&libdir) {
 368                     let name = entry.file_name();
 369                     if let Some(s) = name.to_str() {
 370                         if is_dylib(s) {
 371                             // Don't use custom libdir here because ^lib/ will be resolved again
 372                             // with installer
 373                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 374                         }
 375                     }
 376                 }
 377             }
 378
 379             // Copy over the codegen backends
 380             let backends_src = builder.sysroot_codegen_backends(compiler);
 381             let backends_rel = backends_src
 382                 .strip_prefix(&src)
 383                 .unwrap()
 384                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 385                 .unwrap();
 386             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 387             let backends_dst = image.join("lib").join(&backends_rel);
 388
 389             t!(fs::create_dir_all(&backends_dst));
 390             builder.cp_r(&backends_src, &backends_dst);
 391
 392             // Copy libLLVM.so to the lib dir as well, if needed. While not
 393             // technically needed by rustc itself it's needed by lots of other
 394             // components like the llvm tools and LLD. LLD is included below and
 395             // tools/LLDB come later, so let's just throw it in the rustc
 396             // component for now.
 397             maybe_install_llvm_runtime(builder, host, image);
 398
 399             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 400             t!(fs::create_dir_all(&dst_dir));
 401
 402             // Copy over lld if it's there
 403             if builder.config.lld_enabled {
 404                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 405                 let rust_lld = exe("rust-lld", compiler.host);
 406                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 407                 // for `-Z gcc-ld=lld`
 408                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 409                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 410                 t!(fs::create_dir(&gcc_lld_dst_dir));
 411                 for flavor in ["ld", "ld64"] {
 412                     let exe_name = exe(flavor, compiler.host);
 413                     builder
 414                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 415                 }
 416             }
 417
 418             // Man pages
 419             t!(fs::create_dir_all(image.join("share/man/man1")));
 420             let man_src = builder.src.join("src/doc/man");
 421             let man_dst = image.join("share/man/man1");
 422
 423             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 424             // to hardlink, and we don't want to edit the source templates
 425             for file_entry in builder.read_dir(&man_src) {
 426                 let page_src = file_entry.path();
 427                 let page_dst = man_dst.join(file_entry.file_name());
 428                 let src_text = t!(std::fs::read_to_string(&page_src));
 429                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 430                 t!(std::fs::write(&page_dst, &new_text));
 431                 t!(fs::copy(&page_src, &page_dst));
 432             }
 433
 434             // Debugger scripts
 435             builder
 436                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 437
 438             // Misc license info
 439             let cp = |file: &str| {
 440                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 441             };
 442             cp("COPYRIGHT");
 443             cp("LICENSE-APACHE");
 444             cp("LICENSE-MIT");
 445             cp("README.md");
 446         }
 447     }
 448 }
 449
 450 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 451 pub struct DebuggerScripts {
 452     pub sysroot: Interned<PathBuf>,
 453     pub host: TargetSelection,
 454 }
 455
 456 impl Step for DebuggerScripts {
 457     type Output = ();
 458
 459     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 460         run.path("src/lldb_batchmode.py")
 461     }
 462
 463     fn make_run(run: RunConfig<'_>) {
 464         run.builder.ensure(DebuggerScripts {
 465             sysroot: run
 466                 .builder
 467                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 468             host: run.target,
 469         });
 470     }
 471
 472     /// Copies debugger scripts for `target` into the `sysroot` specified.
 473     fn run(self, builder: &Builder<'_>) {
 474         let host = self.host;
 475         let sysroot = self.sysroot;
 476         let dst = sysroot.join("lib/rustlib/etc");
 477         t!(fs::create_dir_all(&dst));
 478         let cp_debugger_script = |file: &str| {
 479             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 480         };
 481         if host.contains("windows-msvc") {
 482             // windbg debugger scripts
 483             builder.install(
 484                 &builder.src.join("src/etc/rust-windbg.cmd"),
 485                 &sysroot.join("bin"),
 486                 0o755,
 487             );
 488
 489             cp_debugger_script("natvis/intrinsic.natvis");
 490             cp_debugger_script("natvis/liballoc.natvis");
 491             cp_debugger_script("natvis/libcore.natvis");
 492             cp_debugger_script("natvis/libstd.natvis");
 493         } else {
 494             cp_debugger_script("rust_types.py");
 495
 496             // gdb debugger scripts
 497             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 498             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 499
 500             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 501             cp_debugger_script("gdb_lookup.py");
 502             cp_debugger_script("gdb_providers.py");
 503
 504             // lldb debugger scripts
 505             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 506
 507             cp_debugger_script("lldb_lookup.py");
 508             cp_debugger_script("lldb_providers.py");
 509             cp_debugger_script("lldb_commands")
 510         }
 511     }
 512 }
 513
 514 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 515     // The only true set of target libraries came from the build triple, so
 516     // let's reduce redundant work by only producing archives from that host.
 517     if compiler.host != builder.config.build {
 518         builder.info("\tskipping, not a build host");
 519         true
 520     } else {
 521         false
 522     }
 523 }
 524
 525 /// Copy stamped files into an image's `target/lib` directory.
 526 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 527     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 528     let self_contained_dst = dst.join("self-contained");
 529     t!(fs::create_dir_all(&dst));
 530     t!(fs::create_dir_all(&self_contained_dst));
 531     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 532         if dependency_type == DependencyType::TargetSelfContained {
 533             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 534         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 535             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 536         }
 537     }
 538 }
 539
 540 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 541 pub struct Std {
 542     pub compiler: Compiler,
 543     pub target: TargetSelection,
 544 }
 545
 546 impl Step for Std {
 547     type Output = Option<GeneratedTarball>;
 548     const DEFAULT: bool = true;
 549
 550     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 551         run.path("rust-std")
 552     }
 553
 554     fn make_run(run: RunConfig<'_>) {
 555         run.builder.ensure(Std {
 556             compiler: run.builder.compiler_for(
 557                 run.builder.top_stage,
 558                 run.builder.config.build,
 559                 run.target,
 560             ),
 561             target: run.target,
 562         });
 563     }
 564
 565     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 566         let compiler = self.compiler;
 567         let target = self.target;
 568
 569         if skip_host_target_lib(builder, compiler) {
 570             return None;
 571         }
 572
 573         builder.ensure(compile::Std { compiler, target });
 574
 575         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 576         tarball.include_target_in_component_name(true);
 577
 578         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 579         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 580         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 581
 582         Some(tarball.generate())
 583     }
 584 }
 585
 586 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 587 pub struct RustcDev {
 588     pub compiler: Compiler,
 589     pub target: TargetSelection,
 590 }
 591
 592 impl Step for RustcDev {
 593     type Output = Option<GeneratedTarball>;
 594     const DEFAULT: bool = true;
 595     const ONLY_HOSTS: bool = true;
 596
 597     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 598         run.path("rustc-dev")
 599     }
 600
 601     fn make_run(run: RunConfig<'_>) {
 602         run.builder.ensure(RustcDev {
 603             compiler: run.builder.compiler_for(
 604                 run.builder.top_stage,
 605                 run.builder.config.build,
 606                 run.target,
 607             ),
 608             target: run.target,
 609         });
 610     }
 611
 612     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 613         let compiler = self.compiler;
 614         let target = self.target;
 615         if skip_host_target_lib(builder, compiler) {
 616             return None;
 617         }
 618
 619         builder.ensure(compile::Rustc { compiler, target });
 620
 621         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 622
 623         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 624         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 625         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 626
 627         let src_files = &["Cargo.lock"];
 628         // This is the reduced set of paths which will become the rustc-dev component
 629         // (essentially the compiler crates and all of their path dependencies).
 630         copy_src_dirs(
 631             builder,
 632             &builder.src,
 633             &["compiler"],
 634             &[],
 635             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 636         );
 637         for file in src_files {
 638             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 639         }
 640
 641         Some(tarball.generate())
 642     }
 643 }
 644
 645 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 646 pub struct Analysis {
 647     pub compiler: Compiler,
 648     pub target: TargetSelection,
 649 }
 650
 651 impl Step for Analysis {
 652     type Output = Option<GeneratedTarball>;
 653     const DEFAULT: bool = true;
 654
 655     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 656         let default = should_build_extended_tool(&run.builder, "analysis");
 657         run.path("rust-analysis").default_condition(default)
 658     }
 659
 660     fn make_run(run: RunConfig<'_>) {
 661         run.builder.ensure(Analysis {
 662             // Find the actual compiler (handling the full bootstrap option) which
 663             // produced the save-analysis data because that data isn't copied
 664             // through the sysroot uplifting.
 665             compiler: run.builder.compiler_for(
 666                 run.builder.top_stage,
 667                 run.builder.config.build,
 668                 run.target,
 669             ),
 670             target: run.target,
 671         });
 672     }
 673
 674     /// Creates a tarball of save-analysis metadata, if available.
 675     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 676         let compiler = self.compiler;
 677         let target = self.target;
 678         if compiler.host != builder.config.build {
 679             return None;
 680         }
 681
 682         builder.ensure(compile::Std { compiler, target });
 683         let src = builder
 684             .stage_out(compiler, Mode::Std)
 685             .join(target.triple)
 686             .join(builder.cargo_dir())
 687             .join("deps")
 688             .join("save-analysis");
 689
 690         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 691         tarball.include_target_in_component_name(true);
 692         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 693         Some(tarball.generate())
 694     }
 695 }
 696
 697 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 698 /// `dst_dir`.
 699 fn copy_src_dirs(
 700     builder: &Builder<'_>,
 701     base: &Path,
 702     src_dirs: &[&str],
 703     exclude_dirs: &[&str],
 704     dst_dir: &Path,
 705 ) {
 706     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 707         let spath = match path.to_str() {
 708             Some(path) => path,
 709             None => return false,
 710         };
 711         if spath.ends_with('~') || spath.ends_with(".pyc") {
 712             return false;
 713         }
 714
 715         const LLVM_PROJECTS: &[&str] = &[
 716             "llvm-project/clang",
 717             "llvm-project\\clang",
 718             "llvm-project/libunwind",
 719             "llvm-project\\libunwind",
 720             "llvm-project/lld",
 721             "llvm-project\\lld",
 722             "llvm-project/lldb",
 723             "llvm-project\\lldb",
 724             "llvm-project/llvm",
 725             "llvm-project\\llvm",
 726             "llvm-project/compiler-rt",
 727             "llvm-project\\compiler-rt",
 728             "llvm-project/cmake",
 729             "llvm-project\\cmake",
 730         ];
 731         if spath.contains("llvm-project")
 732             && !spath.ends_with("llvm-project")
 733             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 734         {
 735             return false;
 736         }
 737
 738         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 739         if LLVM_TEST.iter().any(|path| spath.contains(path))
 740             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 741         {
 742             return false;
 743         }
 744
 745         let full_path = Path::new(dir).join(path);
 746         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 747             return false;
 748         }
 749
 750         let excludes = [
 751             "CVS",
 752             "RCS",
 753             "SCCS",
 754             ".git",
 755             ".gitignore",
 756             ".gitmodules",
 757             ".gitattributes",
 758             ".cvsignore",
 759             ".svn",
 760             ".arch-ids",
 761             "{arch}",
 762             "=RELEASE-ID",
 763             "=meta-update",
 764             "=update",
 765             ".bzr",
 766             ".bzrignore",
 767             ".bzrtags",
 768             ".hg",
 769             ".hgignore",
 770             ".hgrags",
 771             "_darcs",
 772         ];
 773         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 774     }
 775
 776     // Copy the directories using our filter
 777     for item in src_dirs {
 778         let dst = &dst_dir.join(item);
 779         t!(fs::create_dir_all(dst));
 780         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 781     }
 782 }
 783
 784 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 785 pub struct Src;
 786
 787 impl Step for Src {
 788     /// The output path of the src installer tarball
 789     type Output = GeneratedTarball;
 790     const DEFAULT: bool = true;
 791     const ONLY_HOSTS: bool = true;
 792
 793     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 794         run.path("rust-src")
 795     }
 796
 797     fn make_run(run: RunConfig<'_>) {
 798         run.builder.ensure(Src);
 799     }
 800
 801     /// Creates the `rust-src` installer component
 802     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 803         let tarball = Tarball::new_targetless(builder, "rust-src");
 804
 805         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 806         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 807         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 808         // and fix them...
 809         //
 810         // NOTE: if you update the paths here, you also should update the "virtual" path
 811         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 812         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 813
 814         let src_files = ["Cargo.lock"];
 815         // This is the reduced set of paths which will become the rust-src component
 816         // (essentially libstd and all of its path dependencies).
 817         copy_src_dirs(
 818             builder,
 819             &builder.src,
 820             &["library", "src/llvm-project/libunwind"],
 821             &[
 822                 // not needed and contains symlinks which rustup currently
 823                 // chokes on when unpacking.
 824                 "library/backtrace/crates",
 825             ],
 826             &dst_src,
 827         );
 828         for file in src_files.iter() {
 829             builder.copy(&builder.src.join(file), &dst_src.join(file));
 830         }
 831
 832         tarball.generate()
 833     }
 834 }
 835
 836 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 837 pub struct PlainSourceTarball;
 838
 839 impl Step for PlainSourceTarball {
 840     /// Produces the location of the tarball generated
 841     type Output = GeneratedTarball;
 842     const DEFAULT: bool = true;
 843     const ONLY_HOSTS: bool = true;
 844
 845     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 846         let builder = run.builder;
 847         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 848     }
 849
 850     fn make_run(run: RunConfig<'_>) {
 851         run.builder.ensure(PlainSourceTarball);
 852     }
 853
 854     /// Creates the plain source tarball
 855     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 856         let tarball = Tarball::new(builder, "rustc", "src");
 857         let plain_dst_src = tarball.image_dir();
 858
 859         // This is the set of root paths which will become part of the source package
 860         let src_files = [
 861             "COPYRIGHT",
 862             "LICENSE-APACHE",
 863             "LICENSE-MIT",
 864             "CONTRIBUTING.md",
 865             "README.md",
 866             "RELEASES.md",
 867             "configure",
 868             "x.py",
 869             "config.toml.example",
 870             "Cargo.toml",
 871             "Cargo.lock",
 872         ];
 873         let src_dirs = ["src", "compiler", "library"];
 874
 875         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 876
 877         // Copy the files normally
 878         for item in &src_files {
 879             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 880         }
 881
 882         // Create the version file
 883         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 884         if let Some(sha) = builder.rust_sha() {
 885             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 886         }
 887
 888         // If we're building from git sources, we need to vendor a complete distribution.
 889         if builder.rust_info.is_git() {
 890             // Vendor all Cargo dependencies
 891             let mut cmd = Command::new(&builder.initial_cargo);
 892             cmd.arg("vendor")
 893                 .arg("--sync")
 894                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 895                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 896                 .current_dir(&plain_dst_src);
 897             builder.run(&mut cmd);
 898         }
 899
 900         tarball.bare()
 901     }
 902 }
 903
 904 // We have to run a few shell scripts, which choke quite a bit on both `\`
 905 // characters and on `C:\` paths, so normalize both of them away.
 906 pub fn sanitize_sh(path: &Path) -> String {
 907     let path = path.to_str().unwrap().replace("\\", "/");
 908     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 909
 910     fn unc_to_lfs(s: &str) -> &str {
 911         s.strip_prefix("//?/").unwrap_or(s)
 912     }
 913
 914     fn change_drive(s: &str) -> Option<String> {
 915         let mut ch = s.chars();
 916         let drive = ch.next().unwrap_or('C');
 917         if ch.next() != Some(':') {
 918             return None;
 919         }
 920         if ch.next() != Some('/') {
 921             return None;
 922         }
 923         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 924     }
 925 }
 926
 927 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 928 pub struct Cargo {
 929     pub compiler: Compiler,
 930     pub target: TargetSelection,
 931 }
 932
 933 impl Step for Cargo {
 934     type Output = Option<GeneratedTarball>;
 935     const DEFAULT: bool = true;
 936     const ONLY_HOSTS: bool = true;
 937
 938     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 939         let default = should_build_extended_tool(&run.builder, "cargo");
 940         run.path("cargo").default_condition(default)
 941     }
 942
 943     fn make_run(run: RunConfig<'_>) {
 944         run.builder.ensure(Cargo {
 945             compiler: run.builder.compiler_for(
 946                 run.builder.top_stage,
 947                 run.builder.config.build,
 948                 run.target,
 949             ),
 950             target: run.target,
 951         });
 952     }
 953
 954     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 955         let compiler = self.compiler;
 956         let target = self.target;
 957
 958         let cargo = builder.ensure(tool::Cargo { compiler, target });
 959         let src = builder.src.join("src/tools/cargo");
 960         let etc = src.join("src/etc");
 961
 962         // Prepare the image directory
 963         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 964         tarball.set_overlay(OverlayKind::Cargo);
 965
 966         tarball.add_file(&cargo, "bin", 0o755);
 967         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 968         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 969         tarball.add_dir(etc.join("man"), "share/man/man1");
 970         tarball.add_legal_and_readme_to("share/doc/cargo");
 971
 972         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 973             let dirent = dirent.expect("read dir entry");
 974             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 975                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 976             }
 977         }
 978
 979         Some(tarball.generate())
 980     }
 981 }
 982
 983 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 984 pub struct Rls {
 985     pub compiler: Compiler,
 986     pub target: TargetSelection,
 987 }
 988
 989 impl Step for Rls {
 990     type Output = Option<GeneratedTarball>;
 991     const ONLY_HOSTS: bool = true;
 992     const DEFAULT: bool = true;
 993
 994     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 995         let default = should_build_extended_tool(&run.builder, "rls");
 996         run.path("rls").default_condition(default)
 997     }
 998
 999     fn make_run(run: RunConfig<'_>) {
1000         run.builder.ensure(Rls {
1001             compiler: run.builder.compiler_for(
1002                 run.builder.top_stage,
1003                 run.builder.config.build,
1004                 run.target,
1005             ),
1006             target: run.target,
1007         });
1008     }
1009
1010     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1011         let compiler = self.compiler;
1012         let target = self.target;
1013
1014         let rls = builder
1015             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1016             .or_else(|| {
1017                 missing_tool("RLS", builder.build.config.missing_tools);
1018                 None
1019             })?;
1020
1021         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1022         tarball.set_overlay(OverlayKind::RLS);
1023         tarball.is_preview(true);
1024         tarball.add_file(rls, "bin", 0o755);
1025         tarball.add_legal_and_readme_to("share/doc/rls");
1026         Some(tarball.generate())
1027     }
1028 }
1029
1030 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1031 pub struct RustAnalyzer {
1032     pub compiler: Compiler,
1033     pub target: TargetSelection,
1034 }
1035
1036 impl Step for RustAnalyzer {
1037     type Output = Option<GeneratedTarball>;
1038     const DEFAULT: bool = true;
1039     const ONLY_HOSTS: bool = true;
1040
1041     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1042         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1043         run.path("rust-analyzer").default_condition(default)
1044     }
1045
1046     fn make_run(run: RunConfig<'_>) {
1047         run.builder.ensure(RustAnalyzer {
1048             compiler: run.builder.compiler_for(
1049                 run.builder.top_stage,
1050                 run.builder.config.build,
1051                 run.target,
1052             ),
1053             target: run.target,
1054         });
1055     }
1056
1057     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1058         // This prevents rust-analyzer from being built for "dist" or "install"
1059         // on the stable/beta channels. It is a nightly-only tool and should
1060         // not be included.
1061         if !builder.build.unstable_features() {
1062             return None;
1063         }
1064         let compiler = self.compiler;
1065         let target = self.target;
1066
1067         if target.contains("riscv64") {
1068             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1069             // to build. See #74813 for details.
1070             return None;
1071         }
1072
1073         let rust_analyzer = builder
1074             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1075             .expect("rust-analyzer always builds");
1076
1077         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1078         tarball.set_overlay(OverlayKind::RustAnalyzer);
1079         tarball.is_preview(true);
1080         tarball.add_file(rust_analyzer, "bin", 0o755);
1081         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1082         Some(tarball.generate())
1083     }
1084 }
1085
1086 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1087 pub struct Clippy {
1088     pub compiler: Compiler,
1089     pub target: TargetSelection,
1090 }
1091
1092 impl Step for Clippy {
1093     type Output = Option<GeneratedTarball>;
1094     const DEFAULT: bool = true;
1095     const ONLY_HOSTS: bool = true;
1096
1097     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1098         let default = should_build_extended_tool(&run.builder, "clippy");
1099         run.path("clippy").default_condition(default)
1100     }
1101
1102     fn make_run(run: RunConfig<'_>) {
1103         run.builder.ensure(Clippy {
1104             compiler: run.builder.compiler_for(
1105                 run.builder.top_stage,
1106                 run.builder.config.build,
1107                 run.target,
1108             ),
1109             target: run.target,
1110         });
1111     }
1112
1113     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1114         let compiler = self.compiler;
1115         let target = self.target;
1116
1117         // Prepare the image directory
1118         // We expect clippy to build, because we've exited this step above if tool
1119         // state for clippy isn't testing.
1120         let clippy = builder
1121             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1122             .expect("clippy expected to build - essential tool");
1123         let cargoclippy = builder
1124             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1125             .expect("clippy expected to build - essential tool");
1126
1127         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1128         tarball.set_overlay(OverlayKind::Clippy);
1129         tarball.is_preview(true);
1130         tarball.add_file(clippy, "bin", 0o755);
1131         tarball.add_file(cargoclippy, "bin", 0o755);
1132         tarball.add_legal_and_readme_to("share/doc/clippy");
1133         Some(tarball.generate())
1134     }
1135 }
1136
1137 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1138 pub struct Miri {
1139     pub compiler: Compiler,
1140     pub target: TargetSelection,
1141 }
1142
1143 impl Step for Miri {
1144     type Output = Option<GeneratedTarball>;
1145     const DEFAULT: bool = true;
1146     const ONLY_HOSTS: bool = true;
1147
1148     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1149         let default = should_build_extended_tool(&run.builder, "miri");
1150         run.path("miri").default_condition(default)
1151     }
1152
1153     fn make_run(run: RunConfig<'_>) {
1154         run.builder.ensure(Miri {
1155             compiler: run.builder.compiler_for(
1156                 run.builder.top_stage,
1157                 run.builder.config.build,
1158                 run.target,
1159             ),
1160             target: run.target,
1161         });
1162     }
1163
1164     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1165         // This prevents miri from being built for "dist" or "install"
1166         // on the stable/beta channels. It is a nightly-only tool and should
1167         // not be included.
1168         if !builder.build.unstable_features() {
1169             return None;
1170         }
1171         let compiler = self.compiler;
1172         let target = self.target;
1173
1174         let miri = builder
1175             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1176             .or_else(|| {
1177                 missing_tool("miri", builder.build.config.missing_tools);
1178                 None
1179             })?;
1180         let cargomiri = builder
1181             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1182             .or_else(|| {
1183                 missing_tool("cargo miri", builder.build.config.missing_tools);
1184                 None
1185             })?;
1186
1187         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1188         tarball.set_overlay(OverlayKind::Miri);
1189         tarball.is_preview(true);
1190         tarball.add_file(miri, "bin", 0o755);
1191         tarball.add_file(cargomiri, "bin", 0o755);
1192         tarball.add_legal_and_readme_to("share/doc/miri");
1193         Some(tarball.generate())
1194     }
1195 }
1196
1197 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1198 pub struct Rustfmt {
1199     pub compiler: Compiler,
1200     pub target: TargetSelection,
1201 }
1202
1203 impl Step for Rustfmt {
1204     type Output = Option<GeneratedTarball>;
1205     const DEFAULT: bool = true;
1206     const ONLY_HOSTS: bool = true;
1207
1208     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1209         let default = should_build_extended_tool(&run.builder, "rustfmt");
1210         run.path("rustfmt").default_condition(default)
1211     }
1212
1213     fn make_run(run: RunConfig<'_>) {
1214         run.builder.ensure(Rustfmt {
1215             compiler: run.builder.compiler_for(
1216                 run.builder.top_stage,
1217                 run.builder.config.build,
1218                 run.target,
1219             ),
1220             target: run.target,
1221         });
1222     }
1223
1224     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1225         let compiler = self.compiler;
1226         let target = self.target;
1227
1228         let rustfmt = builder
1229             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1230             .or_else(|| {
1231                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1232                 None
1233             })?;
1234         let cargofmt = builder
1235             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1236             .or_else(|| {
1237                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1238                 None
1239             })?;
1240
1241         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1242         tarball.set_overlay(OverlayKind::Rustfmt);
1243         tarball.is_preview(true);
1244         tarball.add_file(rustfmt, "bin", 0o755);
1245         tarball.add_file(cargofmt, "bin", 0o755);
1246         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1247         Some(tarball.generate())
1248     }
1249 }
1250
1251 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1252 pub struct RustDemangler {
1253     pub compiler: Compiler,
1254     pub target: TargetSelection,
1255 }
1256
1257 impl Step for RustDemangler {
1258     type Output = Option<GeneratedTarball>;
1259     const DEFAULT: bool = true;
1260     const ONLY_HOSTS: bool = true;
1261
1262     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1263         // While other tools use `should_build_extended_tool` to decide whether to be run by
1264         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1265         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1266         // we run the step by default when only `extended = true`, and decide whether to actually
1267         // run it or not later.
1268         let default = run.builder.config.extended;
1269         run.path("rust-demangler").default_condition(default)
1270     }
1271
1272     fn make_run(run: RunConfig<'_>) {
1273         run.builder.ensure(RustDemangler {
1274             compiler: run.builder.compiler_for(
1275                 run.builder.top_stage,
1276                 run.builder.config.build,
1277                 run.target,
1278             ),
1279             target: run.target,
1280         });
1281     }
1282
1283     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1284         let compiler = self.compiler;
1285         let target = self.target;
1286
1287         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1288         let condition = should_build_extended_tool(builder, "rust-demangler")
1289             || builder.config.profiler_enabled(target);
1290         if builder.config.extended && !condition {
1291             return None;
1292         }
1293
1294         let rust_demangler = builder
1295             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1296             .expect("rust-demangler expected to build - in-tree tool");
1297
1298         // Prepare the image directory
1299         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1300         tarball.set_overlay(OverlayKind::RustDemangler);
1301         tarball.is_preview(true);
1302         tarball.add_file(&rust_demangler, "bin", 0o755);
1303         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1304         Some(tarball.generate())
1305     }
1306 }
1307
1308 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1309 pub struct Extended {
1310     stage: u32,
1311     host: TargetSelection,
1312     target: TargetSelection,
1313 }
1314
1315 impl Step for Extended {
1316     type Output = ();
1317     const DEFAULT: bool = true;
1318     const ONLY_HOSTS: bool = true;
1319
1320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1321         let builder = run.builder;
1322         run.path("extended").default_condition(builder.config.extended)
1323     }
1324
1325     fn make_run(run: RunConfig<'_>) {
1326         run.builder.ensure(Extended {
1327             stage: run.builder.top_stage,
1328             host: run.builder.config.build,
1329             target: run.target,
1330         });
1331     }
1332
1333     /// Creates a combined installer for the specified target in the provided stage.
1334     fn run(self, builder: &Builder<'_>) {
1335         let target = self.target;
1336         let stage = self.stage;
1337         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1338
1339         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1340
1341         let mut tarballs = Vec::new();
1342         let mut built_tools = HashSet::new();
1343         macro_rules! add_component {
1344             ($name:expr => $step:expr) => {
1345                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1346                     tarballs.push(tarball);
1347                     built_tools.insert($name);
1348                 }
1349             };
1350         }
1351
1352         // When rust-std package split from rustc, we needed to ensure that during
1353         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1354         // the std files during uninstall. To do this ensure that rustc comes
1355         // before rust-std in the list below.
1356         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1357         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1358
1359         if target.contains("windows-gnu") {
1360             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1361         }
1362
1363         add_component!("rust-docs" => Docs { host: target });
1364         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1365         add_component!("cargo" => Cargo { compiler, target });
1366         add_component!("rustfmt" => Rustfmt { compiler, target });
1367         add_component!("rls" => Rls { compiler, target });
1368         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1369         add_component!("llvm-components" => LlvmTools { target });
1370         add_component!("clippy" => Clippy { compiler, target });
1371         add_component!("miri" => Miri { compiler, target });
1372         add_component!("analysis" => Analysis { compiler, target });
1373
1374         let etc = builder.src.join("src/etc/installer");
1375
1376         // Avoid producing tarballs during a dry run.
1377         if builder.config.dry_run {
1378             return;
1379         }
1380
1381         let tarball = Tarball::new(builder, "rust", &target.triple);
1382         let generated = tarball.combine(&tarballs);
1383
1384         let tmp = tmpdir(builder).join("combined-tarball");
1385         let work = generated.work_dir();
1386
1387         let mut license = String::new();
1388         license += &builder.read(&builder.src.join("COPYRIGHT"));
1389         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1390         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1391         license.push('\n');
1392         license.push('\n');
1393
1394         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1395         let mut rtf = rtf.to_string();
1396         rtf.push('\n');
1397         for line in license.lines() {
1398             rtf.push_str(line);
1399             rtf.push_str("\\line ");
1400         }
1401         rtf.push('}');
1402
1403         fn filter(contents: &str, marker: &str) -> String {
1404             let start = format!("tool-{}-start", marker);
1405             let end = format!("tool-{}-end", marker);
1406             let mut lines = Vec::new();
1407             let mut omitted = false;
1408             for line in contents.lines() {
1409                 if line.contains(&start) {
1410                     omitted = true;
1411                 } else if line.contains(&end) {
1412                     omitted = false;
1413                 } else if !omitted {
1414                     lines.push(line);
1415                 }
1416             }
1417
1418             lines.join("\n")
1419         }
1420
1421         let xform = |p: &Path| {
1422             let mut contents = t!(fs::read_to_string(p));
1423             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1424                 if !built_tools.contains(tool) {
1425                     contents = filter(&contents, tool);
1426                 }
1427             }
1428             let ret = tmp.join(p.file_name().unwrap());
1429             t!(fs::write(&ret, &contents));
1430             ret
1431         };
1432
1433         if target.contains("apple-darwin") {
1434             builder.info("building pkg installer");
1435             let pkg = tmp.join("pkg");
1436             let _ = fs::remove_dir_all(&pkg);
1437
1438             let pkgbuild = |component: &str| {
1439                 let mut cmd = Command::new("pkgbuild");
1440                 cmd.arg("--identifier")
1441                     .arg(format!("org.rust-lang.{}", component))
1442                     .arg("--scripts")
1443                     .arg(pkg.join(component))
1444                     .arg("--nopayload")
1445                     .arg(pkg.join(component).with_extension("pkg"));
1446                 builder.run(&mut cmd);
1447             };
1448
1449             let prepare = |name: &str| {
1450                 builder.create_dir(&pkg.join(name));
1451                 builder.cp_r(
1452                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1453                     &pkg.join(name),
1454                 );
1455                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1456                 pkgbuild(name);
1457             };
1458             prepare("rustc");
1459             prepare("cargo");
1460             prepare("rust-std");
1461             prepare("rust-analysis");
1462             prepare("clippy");
1463             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1464                 if built_tools.contains(tool) {
1465                     prepare(tool);
1466                 }
1467             }
1468             // create an 'uninstall' package
1469             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1470             pkgbuild("uninstall");
1471
1472             builder.create_dir(&pkg.join("res"));
1473             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1474             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1475             let mut cmd = Command::new("productbuild");
1476             cmd.arg("--distribution")
1477                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1478                 .arg("--resources")
1479                 .arg(pkg.join("res"))
1480                 .arg(distdir(builder).join(format!(
1481                     "{}-{}.pkg",
1482                     pkgname(builder, "rust"),
1483                     target.triple
1484                 )))
1485                 .arg("--package-path")
1486                 .arg(&pkg);
1487             let _time = timeit(builder);
1488             builder.run(&mut cmd);
1489         }
1490
1491         if target.contains("windows") {
1492             let exe = tmp.join("exe");
1493             let _ = fs::remove_dir_all(&exe);
1494
1495             let prepare = |name: &str| {
1496                 builder.create_dir(&exe.join(name));
1497                 let dir = if name == "rust-std" || name == "rust-analysis" {
1498                     format!("{}-{}", name, target.triple)
1499                 } else if name == "rls" {
1500                     "rls-preview".to_string()
1501                 } else if name == "rust-analyzer" {
1502                     "rust-analyzer-preview".to_string()
1503                 } else if name == "clippy" {
1504                     "clippy-preview".to_string()
1505                 } else if name == "rust-demangler" {
1506                     "rust-demangler-preview".to_string()
1507                 } else if name == "miri" {
1508                     "miri-preview".to_string()
1509                 } else {
1510                     name.to_string()
1511                 };
1512                 builder.cp_r(
1513                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1514                     &exe.join(name),
1515                 );
1516                 builder.remove(&exe.join(name).join("manifest.in"));
1517             };
1518             prepare("rustc");
1519             prepare("cargo");
1520             prepare("rust-analysis");
1521             prepare("rust-docs");
1522             prepare("rust-std");
1523             prepare("clippy");
1524             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1525                 if built_tools.contains(tool) {
1526                     prepare(tool);
1527                 }
1528             }
1529             if target.contains("windows-gnu") {
1530                 prepare("rust-mingw");
1531             }
1532
1533             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1534
1535             // Generate msi installer
1536             let wix_path = env::var_os("WIX")
1537                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1538             let wix = PathBuf::from(wix_path);
1539             let heat = wix.join("bin/heat.exe");
1540             let candle = wix.join("bin/candle.exe");
1541             let light = wix.join("bin/light.exe");
1542
1543             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1544             builder.run(
1545                 Command::new(&heat)
1546                     .current_dir(&exe)
1547                     .arg("dir")
1548                     .arg("rustc")
1549                     .args(&heat_flags)
1550                     .arg("-cg")
1551                     .arg("RustcGroup")
1552                     .arg("-dr")
1553                     .arg("Rustc")
1554                     .arg("-var")
1555                     .arg("var.RustcDir")
1556                     .arg("-out")
1557                     .arg(exe.join("RustcGroup.wxs")),
1558             );
1559             builder.run(
1560                 Command::new(&heat)
1561                     .current_dir(&exe)
1562                     .arg("dir")
1563                     .arg("rust-docs")
1564                     .args(&heat_flags)
1565                     .arg("-cg")
1566                     .arg("DocsGroup")
1567                     .arg("-dr")
1568                     .arg("Docs")
1569                     .arg("-var")
1570                     .arg("var.DocsDir")
1571                     .arg("-out")
1572                     .arg(exe.join("DocsGroup.wxs"))
1573                     .arg("-t")
1574                     .arg(etc.join("msi/squash-components.xsl")),
1575             );
1576             builder.run(
1577                 Command::new(&heat)
1578                     .current_dir(&exe)
1579                     .arg("dir")
1580                     .arg("cargo")
1581                     .args(&heat_flags)
1582                     .arg("-cg")
1583                     .arg("CargoGroup")
1584                     .arg("-dr")
1585                     .arg("Cargo")
1586                     .arg("-var")
1587                     .arg("var.CargoDir")
1588                     .arg("-out")
1589                     .arg(exe.join("CargoGroup.wxs"))
1590                     .arg("-t")
1591                     .arg(etc.join("msi/remove-duplicates.xsl")),
1592             );
1593             builder.run(
1594                 Command::new(&heat)
1595                     .current_dir(&exe)
1596                     .arg("dir")
1597                     .arg("rust-std")
1598                     .args(&heat_flags)
1599                     .arg("-cg")
1600                     .arg("StdGroup")
1601                     .arg("-dr")
1602                     .arg("Std")
1603                     .arg("-var")
1604                     .arg("var.StdDir")
1605                     .arg("-out")
1606                     .arg(exe.join("StdGroup.wxs")),
1607             );
1608             if built_tools.contains("rls") {
1609                 builder.run(
1610                     Command::new(&heat)
1611                         .current_dir(&exe)
1612                         .arg("dir")
1613                         .arg("rls")
1614                         .args(&heat_flags)
1615                         .arg("-cg")
1616                         .arg("RlsGroup")
1617                         .arg("-dr")
1618                         .arg("Rls")
1619                         .arg("-var")
1620                         .arg("var.RlsDir")
1621                         .arg("-out")
1622                         .arg(exe.join("RlsGroup.wxs"))
1623                         .arg("-t")
1624                         .arg(etc.join("msi/remove-duplicates.xsl")),
1625                 );
1626             }
1627             if built_tools.contains("rust-analyzer") {
1628                 builder.run(
1629                     Command::new(&heat)
1630                         .current_dir(&exe)
1631                         .arg("dir")
1632                         .arg("rust-analyzer")
1633                         .args(&heat_flags)
1634                         .arg("-cg")
1635                         .arg("RustAnalyzerGroup")
1636                         .arg("-dr")
1637                         .arg("RustAnalyzer")
1638                         .arg("-var")
1639                         .arg("var.RustAnalyzerDir")
1640                         .arg("-out")
1641                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1642                         .arg("-t")
1643                         .arg(etc.join("msi/remove-duplicates.xsl")),
1644                 );
1645             }
1646             builder.run(
1647                 Command::new(&heat)
1648                     .current_dir(&exe)
1649                     .arg("dir")
1650                     .arg("clippy")
1651                     .args(&heat_flags)
1652                     .arg("-cg")
1653                     .arg("ClippyGroup")
1654                     .arg("-dr")
1655                     .arg("Clippy")
1656                     .arg("-var")
1657                     .arg("var.ClippyDir")
1658                     .arg("-out")
1659                     .arg(exe.join("ClippyGroup.wxs"))
1660                     .arg("-t")
1661                     .arg(etc.join("msi/remove-duplicates.xsl")),
1662             );
1663             if built_tools.contains("rust-demangler") {
1664                 builder.run(
1665                     Command::new(&heat)
1666                         .current_dir(&exe)
1667                         .arg("dir")
1668                         .arg("rust-demangler")
1669                         .args(&heat_flags)
1670                         .arg("-cg")
1671                         .arg("RustDemanglerGroup")
1672                         .arg("-dr")
1673                         .arg("RustDemangler")
1674                         .arg("-var")
1675                         .arg("var.RustDemanglerDir")
1676                         .arg("-out")
1677                         .arg(exe.join("RustDemanglerGroup.wxs"))
1678                         .arg("-t")
1679                         .arg(etc.join("msi/remove-duplicates.xsl")),
1680                 );
1681             }
1682             if built_tools.contains("miri") {
1683                 builder.run(
1684                     Command::new(&heat)
1685                         .current_dir(&exe)
1686                         .arg("dir")
1687                         .arg("miri")
1688                         .args(&heat_flags)
1689                         .arg("-cg")
1690                         .arg("MiriGroup")
1691                         .arg("-dr")
1692                         .arg("Miri")
1693                         .arg("-var")
1694                         .arg("var.MiriDir")
1695                         .arg("-out")
1696                         .arg(exe.join("MiriGroup.wxs"))
1697                         .arg("-t")
1698                         .arg(etc.join("msi/remove-duplicates.xsl")),
1699                 );
1700             }
1701             builder.run(
1702                 Command::new(&heat)
1703                     .current_dir(&exe)
1704                     .arg("dir")
1705                     .arg("rust-analysis")
1706                     .args(&heat_flags)
1707                     .arg("-cg")
1708                     .arg("AnalysisGroup")
1709                     .arg("-dr")
1710                     .arg("Analysis")
1711                     .arg("-var")
1712                     .arg("var.AnalysisDir")
1713                     .arg("-out")
1714                     .arg(exe.join("AnalysisGroup.wxs"))
1715                     .arg("-t")
1716                     .arg(etc.join("msi/remove-duplicates.xsl")),
1717             );
1718             if target.contains("windows-gnu") {
1719                 builder.run(
1720                     Command::new(&heat)
1721                         .current_dir(&exe)
1722                         .arg("dir")
1723                         .arg("rust-mingw")
1724                         .args(&heat_flags)
1725                         .arg("-cg")
1726                         .arg("GccGroup")
1727                         .arg("-dr")
1728                         .arg("Gcc")
1729                         .arg("-var")
1730                         .arg("var.GccDir")
1731                         .arg("-out")
1732                         .arg(exe.join("GccGroup.wxs")),
1733                 );
1734             }
1735
1736             let candle = |input: &Path| {
1737                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1738                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1739                 let mut cmd = Command::new(&candle);
1740                 cmd.current_dir(&exe)
1741                     .arg("-nologo")
1742                     .arg("-dRustcDir=rustc")
1743                     .arg("-dDocsDir=rust-docs")
1744                     .arg("-dCargoDir=cargo")
1745                     .arg("-dStdDir=rust-std")
1746                     .arg("-dAnalysisDir=rust-analysis")
1747                     .arg("-dClippyDir=clippy")
1748                     .arg("-arch")
1749                     .arg(&arch)
1750                     .arg("-out")
1751                     .arg(&output)
1752                     .arg(&input);
1753                 add_env(builder, &mut cmd, target);
1754
1755                 if built_tools.contains("rust-demangler") {
1756                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1757                 }
1758                 if built_tools.contains("rls") {
1759                     cmd.arg("-dRlsDir=rls");
1760                 }
1761                 if built_tools.contains("rust-analyzer") {
1762                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1763                 }
1764                 if built_tools.contains("miri") {
1765                     cmd.arg("-dMiriDir=miri");
1766                 }
1767                 if target.contains("windows-gnu") {
1768                     cmd.arg("-dGccDir=rust-mingw");
1769                 }
1770                 builder.run(&mut cmd);
1771             };
1772             candle(&xform(&etc.join("msi/rust.wxs")));
1773             candle(&etc.join("msi/ui.wxs"));
1774             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1775             candle("RustcGroup.wxs".as_ref());
1776             candle("DocsGroup.wxs".as_ref());
1777             candle("CargoGroup.wxs".as_ref());
1778             candle("StdGroup.wxs".as_ref());
1779             candle("ClippyGroup.wxs".as_ref());
1780             if built_tools.contains("rust-demangler") {
1781                 candle("RustDemanglerGroup.wxs".as_ref());
1782             }
1783             if built_tools.contains("rls") {
1784                 candle("RlsGroup.wxs".as_ref());
1785             }
1786             if built_tools.contains("rust-analyzer") {
1787                 candle("RustAnalyzerGroup.wxs".as_ref());
1788             }
1789             if built_tools.contains("miri") {
1790                 candle("MiriGroup.wxs".as_ref());
1791             }
1792             candle("AnalysisGroup.wxs".as_ref());
1793
1794             if target.contains("windows-gnu") {
1795                 candle("GccGroup.wxs".as_ref());
1796             }
1797
1798             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1799             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1800             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1801
1802             builder.info(&format!("building `msi` installer with {:?}", light));
1803             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1804             let mut cmd = Command::new(&light);
1805             cmd.arg("-nologo")
1806                 .arg("-ext")
1807                 .arg("WixUIExtension")
1808                 .arg("-ext")
1809                 .arg("WixUtilExtension")
1810                 .arg("-out")
1811                 .arg(exe.join(&filename))
1812                 .arg("rust.wixobj")
1813                 .arg("ui.wixobj")
1814                 .arg("rustwelcomedlg.wixobj")
1815                 .arg("RustcGroup.wixobj")
1816                 .arg("DocsGroup.wixobj")
1817                 .arg("CargoGroup.wixobj")
1818                 .arg("StdGroup.wixobj")
1819                 .arg("AnalysisGroup.wixobj")
1820                 .arg("ClippyGroup.wixobj")
1821                 .current_dir(&exe);
1822
1823             if built_tools.contains("rls") {
1824                 cmd.arg("RlsGroup.wixobj");
1825             }
1826             if built_tools.contains("rust-analyzer") {
1827                 cmd.arg("RustAnalyzerGroup.wixobj");
1828             }
1829             if built_tools.contains("rust-demangler") {
1830                 cmd.arg("RustDemanglerGroup.wixobj");
1831             }
1832             if built_tools.contains("miri") {
1833                 cmd.arg("MiriGroup.wixobj");
1834             }
1835
1836             if target.contains("windows-gnu") {
1837                 cmd.arg("GccGroup.wixobj");
1838             }
1839             // ICE57 wrongly complains about the shortcuts
1840             cmd.arg("-sice:ICE57");
1841
1842             let _time = timeit(builder);
1843             builder.run(&mut cmd);
1844
1845             if !builder.config.dry_run {
1846                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1847             }
1848         }
1849     }
1850 }
1851
1852 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1853     let mut parts = builder.version.split('.');
1854     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1855         .env("CFG_RELEASE_NUM", &builder.version)
1856         .env("CFG_RELEASE", builder.rust_release())
1857         .env("CFG_VER_MAJOR", parts.next().unwrap())
1858         .env("CFG_VER_MINOR", parts.next().unwrap())
1859         .env("CFG_VER_PATCH", parts.next().unwrap())
1860         .env("CFG_VER_BUILD", "0") // just needed to build
1861         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1862         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1863         .env("CFG_BUILD", target.triple)
1864         .env("CFG_CHANNEL", &builder.config.channel);
1865
1866     if target.contains("windows-gnu") {
1867         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1868     } else {
1869         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1870     }
1871 }
1872
1873 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1874 ///
1875
1876 /// Returns whether the files were actually copied.
1877 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1878     if let Some(config) = builder.config.target_config.get(&target) {
1879         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1880             // If the LLVM was externally provided, then we don't currently copy
1881             // artifacts into the sysroot. This is not necessarily the right
1882             // choice (in particular, it will require the LLVM dylib to be in
1883             // the linker's load path at runtime), but the common use case for
1884             // external LLVMs is distribution provided LLVMs, and in that case
1885             // they're usually in the standard search path (e.g., /usr/lib) and
1886             // copying them here is going to cause problems as we may end up
1887             // with the wrong files and isn't what distributions want.
1888             //
1889             // This behavior may be revisited in the future though.
1890             //
1891             // If the LLVM is coming from ourselves (just from CI) though, we
1892             // still want to install it, as it otherwise won't be available.
1893             return false;
1894         }
1895     }
1896
1897     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1898     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1899     // clear why this is the case, though. llvm-config will emit the versioned
1900     // paths and we don't want those in the sysroot (as we're expecting
1901     // unversioned paths).
1902     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1903         let src_libdir = builder.llvm_out(target).join("lib");
1904         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1905         if llvm_dylib_path.exists() {
1906             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1907         }
1908         !builder.config.dry_run
1909     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1910         let mut cmd = Command::new(llvm_config);
1911         cmd.arg("--libfiles");
1912         builder.verbose(&format!("running {:?}", cmd));
1913         let files = output(&mut cmd);
1914         let build_llvm_out = &builder.llvm_out(builder.config.build);
1915         let target_llvm_out = &builder.llvm_out(target);
1916         for file in files.trim_end().split(' ') {
1917             // If we're not using a custom LLVM, make sure we package for the target.
1918             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1919                 target_llvm_out.join(relative_path)
1920             } else {
1921                 PathBuf::from(file)
1922             };
1923             builder.install(&file, dst_libdir, 0o644);
1924         }
1925         !builder.config.dry_run
1926     } else {
1927         false
1928     }
1929 }
1930
1931 /// Maybe add libLLVM.so to the target lib-dir for linking.
1932 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1933     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1934     // We do not need to copy LLVM files into the sysroot if it is not
1935     // dynamically linked; it is already included into librustc_llvm
1936     // statically.
1937     if builder.config.llvm_link_shared {
1938         maybe_install_llvm(builder, target, &dst_libdir);
1939     }
1940 }
1941
1942 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1943 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1944     let dst_libdir =
1945         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1946     // We do not need to copy LLVM files into the sysroot if it is not
1947     // dynamically linked; it is already included into librustc_llvm
1948     // statically.
1949     if builder.config.llvm_link_shared {
1950         maybe_install_llvm(builder, target, &dst_libdir);
1951     }
1952 }
1953
1954 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1955 pub struct LlvmTools {
1956     pub target: TargetSelection,
1957 }
1958
1959 impl Step for LlvmTools {
1960     type Output = Option<GeneratedTarball>;
1961     const ONLY_HOSTS: bool = true;
1962     const DEFAULT: bool = true;
1963
1964     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1965         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1966         run.path("llvm-tools").default_condition(default)
1967     }
1968
1969     fn make_run(run: RunConfig<'_>) {
1970         run.builder.ensure(LlvmTools { target: run.target });
1971     }
1972
1973     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1974         let target = self.target;
1975
1976         /* run only if llvm-config isn't used */
1977         if let Some(config) = builder.config.target_config.get(&target) {
1978             if let Some(ref _s) = config.llvm_config {
1979                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1980                 return None;
1981             }
1982         }
1983
1984         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1985         tarball.set_overlay(OverlayKind::LLVM);
1986         tarball.is_preview(true);
1987
1988         // Prepare the image directory
1989         let src_bindir = builder.llvm_out(target).join("bin");
1990         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1991         for tool in LLVM_TOOLS {
1992             let exe = src_bindir.join(exe(tool, target));
1993             tarball.add_file(&exe, &dst_bindir, 0o755);
1994         }
1995
1996         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1997         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1998         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1999         // compiler libraries.
2000         maybe_install_llvm_target(builder, target, tarball.image_dir());
2001
2002         Some(tarball.generate())
2003     }
2004 }
2005
2006 // Tarball intended for internal consumption to ease rustc/std development.
2007 //
2008 // Should not be considered stable by end users.
2009 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2010 pub struct RustDev {
2011     pub target: TargetSelection,
2012 }
2013
2014 impl Step for RustDev {
2015     type Output = Option<GeneratedTarball>;
2016     const DEFAULT: bool = true;
2017     const ONLY_HOSTS: bool = true;
2018
2019     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2020         run.path("rust-dev")
2021     }
2022
2023     fn make_run(run: RunConfig<'_>) {
2024         run.builder.ensure(RustDev { target: run.target });
2025     }
2026
2027     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2028         let target = self.target;
2029
2030         /* run only if llvm-config isn't used */
2031         if let Some(config) = builder.config.target_config.get(&target) {
2032             if let Some(ref _s) = config.llvm_config {
2033                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2034                 return None;
2035             }
2036         }
2037
2038         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2039         tarball.set_overlay(OverlayKind::LLVM);
2040
2041         let src_bindir = builder.llvm_out(target).join("bin");
2042         for bin in &[
2043             "llvm-config",
2044             "llvm-ar",
2045             "llvm-objdump",
2046             "llvm-profdata",
2047             "llvm-bcanalyzer",
2048             "llvm-cov",
2049             "llvm-dwp",
2050             "llvm-nm",
2051         ] {
2052             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2053         }
2054
2055         // We don't build LLD on some platforms, so only add it if it exists
2056         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2057         if lld_path.exists() {
2058             tarball.add_file(lld_path, "bin", 0o755);
2059         }
2060
2061         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2062
2063         // Copy the include directory as well; needed mostly to build
2064         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2065         // just broadly useful to be able to link against the bundled LLVM.
2066         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2067
2068         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2069         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2070         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2071         // compiler libraries.
2072         let dst_libdir = tarball.image_dir().join("lib");
2073         maybe_install_llvm(builder, target, &dst_libdir);
2074         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2075         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2076
2077         Some(tarball.generate())
2078     }
2079 }
2080
2081 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2082 /// release process to avoid cloning the monorepo and building stuff.
2083 ///
2084 /// Should not be considered stable by end users.
2085 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2086 pub struct BuildManifest {
2087     pub target: TargetSelection,
2088 }
2089
2090 impl Step for BuildManifest {
2091     type Output = GeneratedTarball;
2092     const DEFAULT: bool = false;
2093     const ONLY_HOSTS: bool = true;
2094
2095     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2096         run.path("build-manifest")
2097     }
2098
2099     fn make_run(run: RunConfig<'_>) {
2100         run.builder.ensure(BuildManifest { target: run.target });
2101     }
2102
2103     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2104         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2105
2106         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2107         tarball.add_file(&build_manifest, "bin", 0o755);
2108         tarball.generate()
2109     }
2110 }
2111
2112 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2113 ///
2114 /// Currently this is the PGO profile data.
2115 ///
2116 /// Should not be considered stable by end users.
2117 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2118 pub struct ReproducibleArtifacts {
2119     pub target: TargetSelection,
2120 }
2121
2122 impl Step for ReproducibleArtifacts {
2123     type Output = Option<GeneratedTarball>;
2124     const DEFAULT: bool = true;
2125     const ONLY_HOSTS: bool = true;
2126
2127     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2128         run.path("reproducible-artifacts")
2129     }
2130
2131     fn make_run(run: RunConfig<'_>) {
2132         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2133     }
2134
2135     fn run(self, builder: &Builder<'_>) -> Self::Output {
2136         let mut added_anything = false;
2137         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2138         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2139             tarball.add_file(path, ".", 0o644);
2140             added_anything = true;
2141         }
2142         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2143             tarball.add_file(path, ".", 0o644);
2144             added_anything = true;
2145         }
2146         if added_anything { Some(tarball.generate()) } else { None }
2147     }
2148 }