src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("rust-docs").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         let builder = run.builder;
 100         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
 101     }
 102
 103     fn make_run(run: RunConfig<'_>) {
 104         run.builder.ensure(RustcDocs { host: run.target });
 105     }
 106
 107     /// Builds the `rustc-docs` installer component.
 108     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 109         let host = self.host;
 110         builder.default_doc(&[]);
 111
 112         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 113         tarball.set_product_name("Rustc Documentation");
 114         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 115         Some(tarball.generate())
 116     }
 117 }
 118
 119 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 120     let mut found = Vec::with_capacity(files.len());
 121
 122     for file in files {
 123         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 124
 125         if let Some(file_path) = file_path {
 126             found.push(file_path);
 127         } else {
 128             panic!("Could not find '{}' in {:?}", file, path);
 129         }
 130     }
 131
 132     found
 133 }
 134
 135 fn make_win_dist(
 136     rust_root: &Path,
 137     plat_root: &Path,
 138     target: TargetSelection,
 139     builder: &Builder<'_>,
 140 ) {
 141     //Ask gcc where it keeps its stuff
 142     let mut cmd = Command::new(builder.cc(target));
 143     cmd.arg("-print-search-dirs");
 144     let gcc_out = output(&mut cmd);
 145
 146     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 147     let mut lib_path = Vec::new();
 148
 149     for line in gcc_out.lines() {
 150         let idx = line.find(':').unwrap();
 151         let key = &line[..idx];
 152         let trim_chars: &[_] = &[' ', '='];
 153         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 154
 155         if key == "programs" {
 156             bin_path.extend(value);
 157         } else if key == "libraries" {
 158             lib_path.extend(value);
 159         }
 160     }
 161
 162     let compiler = if target == "i686-pc-windows-gnu" {
 163         "i686-w64-mingw32-gcc.exe"
 164     } else if target == "x86_64-pc-windows-gnu" {
 165         "x86_64-w64-mingw32-gcc.exe"
 166     } else {
 167         "gcc.exe"
 168     };
 169     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 170     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 171     if target.starts_with("i686-") {
 172         rustc_dlls.push("libgcc_s_dw2-1.dll");
 173     } else {
 174         rustc_dlls.push("libgcc_s_seh-1.dll");
 175     }
 176
 177     let target_libs = [
 178         //MinGW libs
 179         "libgcc.a",
 180         "libgcc_eh.a",
 181         "libgcc_s.a",
 182         "libm.a",
 183         "libmingw32.a",
 184         "libmingwex.a",
 185         "libstdc++.a",
 186         "libiconv.a",
 187         "libmoldname.a",
 188         "libpthread.a",
 189         //Windows import libs
 190         "libadvapi32.a",
 191         "libbcrypt.a",
 192         "libcomctl32.a",
 193         "libcomdlg32.a",
 194         "libcredui.a",
 195         "libcrypt32.a",
 196         "libdbghelp.a",
 197         "libgdi32.a",
 198         "libimagehlp.a",
 199         "libiphlpapi.a",
 200         "libkernel32.a",
 201         "libmsimg32.a",
 202         "libmsvcrt.a",
 203         "libodbc32.a",
 204         "libole32.a",
 205         "liboleaut32.a",
 206         "libopengl32.a",
 207         "libpsapi.a",
 208         "librpcrt4.a",
 209         "libsecur32.a",
 210         "libsetupapi.a",
 211         "libshell32.a",
 212         "libsynchronization.a",
 213         "libuser32.a",
 214         "libuserenv.a",
 215         "libuuid.a",
 216         "libwinhttp.a",
 217         "libwinmm.a",
 218         "libwinspool.a",
 219         "libws2_32.a",
 220         "libwsock32.a",
 221     ];
 222
 223     //Find mingw artifacts we want to bundle
 224     let target_tools = find_files(&target_tools, &bin_path);
 225     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 226     let target_libs = find_files(&target_libs, &lib_path);
 227
 228     // Copy runtime dlls next to rustc.exe
 229     let dist_bin_dir = rust_root.join("bin/");
 230     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 231     for src in rustc_dlls {
 232         builder.copy_to_folder(&src, &dist_bin_dir);
 233     }
 234
 235     //Copy platform tools to platform-specific bin directory
 236     let target_bin_dir = plat_root
 237         .join("lib")
 238         .join("rustlib")
 239         .join(target.triple)
 240         .join("bin")
 241         .join("self-contained");
 242     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 243     for src in target_tools {
 244         builder.copy_to_folder(&src, &target_bin_dir);
 245     }
 246
 247     // Warn windows-gnu users that the bundled GCC cannot compile C files
 248     builder.create(
 249         &target_bin_dir.join("GCC-WARNING.txt"),
 250         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 251          used as a linker. In order to be able to compile projects containing C code use \
 252          the GCC provided by MinGW or Cygwin.",
 253     );
 254
 255     //Copy platform libs to platform-specific lib directory
 256     let target_lib_dir = plat_root
 257         .join("lib")
 258         .join("rustlib")
 259         .join(target.triple)
 260         .join("lib")
 261         .join("self-contained");
 262     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 263     for src in target_libs {
 264         builder.copy_to_folder(&src, &target_lib_dir);
 265     }
 266 }
 267
 268 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 269 pub struct Mingw {
 270     pub host: TargetSelection,
 271 }
 272
 273 impl Step for Mingw {
 274     type Output = Option<GeneratedTarball>;
 275     const DEFAULT: bool = true;
 276
 277     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 278         run.path("rust-mingw")
 279     }
 280
 281     fn make_run(run: RunConfig<'_>) {
 282         run.builder.ensure(Mingw { host: run.target });
 283     }
 284
 285     /// Builds the `rust-mingw` installer component.
 286     ///
 287     /// This contains all the bits and pieces to run the MinGW Windows targets
 288     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 289     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 290         let host = self.host;
 291         if !host.contains("pc-windows-gnu") {
 292             return None;
 293         }
 294
 295         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 296         tarball.set_product_name("Rust MinGW");
 297
 298         // The first argument is a "temporary directory" which is just
 299         // thrown away (this contains the runtime DLLs included in the rustc package
 300         // above) and the second argument is where to place all the MinGW components
 301         // (which is what we want).
 302         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 303
 304         Some(tarball.generate())
 305     }
 306 }
 307
 308 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 309 pub struct Rustc {
 310     pub compiler: Compiler,
 311 }
 312
 313 impl Step for Rustc {
 314     type Output = GeneratedTarball;
 315     const DEFAULT: bool = true;
 316     const ONLY_HOSTS: bool = true;
 317
 318     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 319         run.path("rustc")
 320     }
 321
 322     fn make_run(run: RunConfig<'_>) {
 323         run.builder
 324             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 325     }
 326
 327     /// Creates the `rustc` installer component.
 328     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 329         let compiler = self.compiler;
 330         let host = self.compiler.host;
 331
 332         let tarball = Tarball::new(builder, "rustc", &host.triple);
 333
 334         // Prepare the rustc "image", what will actually end up getting installed
 335         prepare_image(builder, compiler, tarball.image_dir());
 336
 337         // On MinGW we've got a few runtime DLL dependencies that we need to
 338         // include. The first argument to this script is where to put these DLLs
 339         // (the image we're creating), and the second argument is a junk directory
 340         // to ignore all other MinGW stuff the script creates.
 341         //
 342         // On 32-bit MinGW we're always including a DLL which needs some extra
 343         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 344         // anything requiring us to distribute a license, but it's likely the
 345         // install will *also* include the rust-mingw package, which also needs
 346         // licenses, so to be safe we just include it here in all MinGW packages.
 347         if host.contains("pc-windows-gnu") {
 348             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 349             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 350         }
 351
 352         return tarball.generate();
 353
 354         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 355             let host = compiler.host;
 356             let src = builder.sysroot(compiler);
 357
 358             // Copy rustc/rustdoc binaries
 359             t!(fs::create_dir_all(image.join("bin")));
 360             builder.cp_r(&src.join("bin"), &image.join("bin"));
 361
 362             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 363
 364             let libdir_relative = builder.libdir_relative(compiler);
 365
 366             // Copy runtime DLLs needed by the compiler
 367             if libdir_relative.to_str() != Some("bin") {
 368                 let libdir = builder.rustc_libdir(compiler);
 369                 for entry in builder.read_dir(&libdir) {
 370                     let name = entry.file_name();
 371                     if let Some(s) = name.to_str() {
 372                         if is_dylib(s) {
 373                             // Don't use custom libdir here because ^lib/ will be resolved again
 374                             // with installer
 375                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 376                         }
 377                     }
 378                 }
 379             }
 380
 381             // Copy over the codegen backends
 382             let backends_src = builder.sysroot_codegen_backends(compiler);
 383             let backends_rel = backends_src
 384                 .strip_prefix(&src)
 385                 .unwrap()
 386                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 387                 .unwrap();
 388             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 389             let backends_dst = image.join("lib").join(&backends_rel);
 390
 391             t!(fs::create_dir_all(&backends_dst));
 392             builder.cp_r(&backends_src, &backends_dst);
 393
 394             // Copy libLLVM.so to the lib dir as well, if needed. While not
 395             // technically needed by rustc itself it's needed by lots of other
 396             // components like the llvm tools and LLD. LLD is included below and
 397             // tools/LLDB come later, so let's just throw it in the rustc
 398             // component for now.
 399             maybe_install_llvm_runtime(builder, host, image);
 400
 401             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 402             t!(fs::create_dir_all(&dst_dir));
 403
 404             // Copy over lld if it's there
 405             if builder.config.lld_enabled {
 406                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 407                 let rust_lld = exe("rust-lld", compiler.host);
 408                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 409                 // for `-Z gcc-ld=lld`
 410                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 411                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 412                 t!(fs::create_dir(&gcc_lld_dst_dir));
 413                 for flavor in ["ld", "ld64"] {
 414                     let exe_name = exe(flavor, compiler.host);
 415                     builder
 416                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 417                 }
 418             }
 419
 420             // Man pages
 421             t!(fs::create_dir_all(image.join("share/man/man1")));
 422             let man_src = builder.src.join("src/doc/man");
 423             let man_dst = image.join("share/man/man1");
 424
 425             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 426             let time = env::var("SOURCE_DATE_EPOCH")
 427                 .map(|timestamp| {
 428                     let epoch = timestamp
 429                         .parse()
 430                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 431                         .unwrap();
 432
 433                     time::at(Timespec::new(epoch, 0))
 434                 })
 435                 .unwrap_or_else(|_| time::now());
 436
 437             let month_year = t!(time::strftime("%B %Y", &time));
 438             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 439             // to hardlink, and we don't want to edit the source templates
 440             for file_entry in builder.read_dir(&man_src) {
 441                 let page_src = file_entry.path();
 442                 let page_dst = man_dst.join(file_entry.file_name());
 443                 t!(fs::copy(&page_src, &page_dst));
 444                 // template in month/year and version number
 445                 builder.replace_in_file(
 446                     &page_dst,
 447                     &[
 448                         ("<INSERT DATE HERE>", &month_year),
 449                         ("<INSERT VERSION HERE>", &builder.version),
 450                     ],
 451                 );
 452             }
 453
 454             // Debugger scripts
 455             builder
 456                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 457
 458             // Misc license info
 459             let cp = |file: &str| {
 460                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 461             };
 462             cp("COPYRIGHT");
 463             cp("LICENSE-APACHE");
 464             cp("LICENSE-MIT");
 465             cp("README.md");
 466         }
 467     }
 468 }
 469
 470 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 471 pub struct DebuggerScripts {
 472     pub sysroot: Interned<PathBuf>,
 473     pub host: TargetSelection,
 474 }
 475
 476 impl Step for DebuggerScripts {
 477     type Output = ();
 478
 479     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 480         run.path("src/lldb_batchmode.py")
 481     }
 482
 483     fn make_run(run: RunConfig<'_>) {
 484         run.builder.ensure(DebuggerScripts {
 485             sysroot: run
 486                 .builder
 487                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 488             host: run.target,
 489         });
 490     }
 491
 492     /// Copies debugger scripts for `target` into the `sysroot` specified.
 493     fn run(self, builder: &Builder<'_>) {
 494         let host = self.host;
 495         let sysroot = self.sysroot;
 496         let dst = sysroot.join("lib/rustlib/etc");
 497         t!(fs::create_dir_all(&dst));
 498         let cp_debugger_script = |file: &str| {
 499             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 500         };
 501         if host.contains("windows-msvc") {
 502             // windbg debugger scripts
 503             builder.install(
 504                 &builder.src.join("src/etc/rust-windbg.cmd"),
 505                 &sysroot.join("bin"),
 506                 0o755,
 507             );
 508
 509             cp_debugger_script("natvis/intrinsic.natvis");
 510             cp_debugger_script("natvis/liballoc.natvis");
 511             cp_debugger_script("natvis/libcore.natvis");
 512             cp_debugger_script("natvis/libstd.natvis");
 513         } else {
 514             cp_debugger_script("rust_types.py");
 515
 516             // gdb debugger scripts
 517             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 518             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 519
 520             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 521             cp_debugger_script("gdb_lookup.py");
 522             cp_debugger_script("gdb_providers.py");
 523
 524             // lldb debugger scripts
 525             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 526
 527             cp_debugger_script("lldb_lookup.py");
 528             cp_debugger_script("lldb_providers.py");
 529             cp_debugger_script("lldb_commands")
 530         }
 531     }
 532 }
 533
 534 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 535     // The only true set of target libraries came from the build triple, so
 536     // let's reduce redundant work by only producing archives from that host.
 537     if compiler.host != builder.config.build {
 538         builder.info("\tskipping, not a build host");
 539         true
 540     } else {
 541         false
 542     }
 543 }
 544
 545 /// Copy stamped files into an image's `target/lib` directory.
 546 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 547     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 548     let self_contained_dst = dst.join("self-contained");
 549     t!(fs::create_dir_all(&dst));
 550     t!(fs::create_dir_all(&self_contained_dst));
 551     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 552         if dependency_type == DependencyType::TargetSelfContained {
 553             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 554         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 555             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 556         }
 557     }
 558 }
 559
 560 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 561 pub struct Std {
 562     pub compiler: Compiler,
 563     pub target: TargetSelection,
 564 }
 565
 566 impl Step for Std {
 567     type Output = Option<GeneratedTarball>;
 568     const DEFAULT: bool = true;
 569
 570     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 571         run.path("rust-std")
 572     }
 573
 574     fn make_run(run: RunConfig<'_>) {
 575         run.builder.ensure(Std {
 576             compiler: run.builder.compiler_for(
 577                 run.builder.top_stage,
 578                 run.builder.config.build,
 579                 run.target,
 580             ),
 581             target: run.target,
 582         });
 583     }
 584
 585     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 586         let compiler = self.compiler;
 587         let target = self.target;
 588
 589         if skip_host_target_lib(builder, compiler) {
 590             return None;
 591         }
 592
 593         builder.ensure(compile::Std { compiler, target });
 594
 595         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 596         tarball.include_target_in_component_name(true);
 597
 598         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 599         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 600         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 601
 602         Some(tarball.generate())
 603     }
 604 }
 605
 606 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 607 pub struct RustcDev {
 608     pub compiler: Compiler,
 609     pub target: TargetSelection,
 610 }
 611
 612 impl Step for RustcDev {
 613     type Output = Option<GeneratedTarball>;
 614     const DEFAULT: bool = true;
 615     const ONLY_HOSTS: bool = true;
 616
 617     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 618         run.path("rustc-dev")
 619     }
 620
 621     fn make_run(run: RunConfig<'_>) {
 622         run.builder.ensure(RustcDev {
 623             compiler: run.builder.compiler_for(
 624                 run.builder.top_stage,
 625                 run.builder.config.build,
 626                 run.target,
 627             ),
 628             target: run.target,
 629         });
 630     }
 631
 632     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 633         let compiler = self.compiler;
 634         let target = self.target;
 635         if skip_host_target_lib(builder, compiler) {
 636             return None;
 637         }
 638
 639         builder.ensure(compile::Rustc { compiler, target });
 640
 641         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 642
 643         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 644         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 645         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 646
 647         let src_files = &["Cargo.lock"];
 648         // This is the reduced set of paths which will become the rustc-dev component
 649         // (essentially the compiler crates and all of their path dependencies).
 650         copy_src_dirs(
 651             builder,
 652             &builder.src,
 653             &["compiler"],
 654             &[],
 655             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 656         );
 657         // This particular crate is used as a build dependency of the above.
 658         copy_src_dirs(
 659             builder,
 660             &builder.src,
 661             &["src/build_helper"],
 662             &[],
 663             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 664         );
 665         for file in src_files {
 666             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 667         }
 668
 669         Some(tarball.generate())
 670     }
 671 }
 672
 673 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 674 pub struct Analysis {
 675     pub compiler: Compiler,
 676     pub target: TargetSelection,
 677 }
 678
 679 impl Step for Analysis {
 680     type Output = Option<GeneratedTarball>;
 681     const DEFAULT: bool = true;
 682
 683     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 684         let default = should_build_extended_tool(&run.builder, "analysis");
 685         run.path("rust-analysis").default_condition(default)
 686     }
 687
 688     fn make_run(run: RunConfig<'_>) {
 689         run.builder.ensure(Analysis {
 690             // Find the actual compiler (handling the full bootstrap option) which
 691             // produced the save-analysis data because that data isn't copied
 692             // through the sysroot uplifting.
 693             compiler: run.builder.compiler_for(
 694                 run.builder.top_stage,
 695                 run.builder.config.build,
 696                 run.target,
 697             ),
 698             target: run.target,
 699         });
 700     }
 701
 702     /// Creates a tarball of save-analysis metadata, if available.
 703     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 704         let compiler = self.compiler;
 705         let target = self.target;
 706         if compiler.host != builder.config.build {
 707             return None;
 708         }
 709
 710         builder.ensure(compile::Std { compiler, target });
 711         let src = builder
 712             .stage_out(compiler, Mode::Std)
 713             .join(target.triple)
 714             .join(builder.cargo_dir())
 715             .join("deps")
 716             .join("save-analysis");
 717
 718         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 719         tarball.include_target_in_component_name(true);
 720         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 721         Some(tarball.generate())
 722     }
 723 }
 724
 725 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 726 /// `dst_dir`.
 727 fn copy_src_dirs(
 728     builder: &Builder<'_>,
 729     base: &Path,
 730     src_dirs: &[&str],
 731     exclude_dirs: &[&str],
 732     dst_dir: &Path,
 733 ) {
 734     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 735         let spath = match path.to_str() {
 736             Some(path) => path,
 737             None => return false,
 738         };
 739         if spath.ends_with('~') || spath.ends_with(".pyc") {
 740             return false;
 741         }
 742
 743         const LLVM_PROJECTS: &[&str] = &[
 744             "llvm-project/clang",
 745             "llvm-project\\clang",
 746             "llvm-project/libunwind",
 747             "llvm-project\\libunwind",
 748             "llvm-project/lld",
 749             "llvm-project\\lld",
 750             "llvm-project/lldb",
 751             "llvm-project\\lldb",
 752             "llvm-project/llvm",
 753             "llvm-project\\llvm",
 754             "llvm-project/compiler-rt",
 755             "llvm-project\\compiler-rt",
 756         ];
 757         if spath.contains("llvm-project")
 758             && !spath.ends_with("llvm-project")
 759             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 760         {
 761             return false;
 762         }
 763
 764         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 765         if LLVM_TEST.iter().any(|path| spath.contains(path))
 766             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 767         {
 768             return false;
 769         }
 770
 771         let full_path = Path::new(dir).join(path);
 772         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 773             return false;
 774         }
 775
 776         let excludes = [
 777             "CVS",
 778             "RCS",
 779             "SCCS",
 780             ".git",
 781             ".gitignore",
 782             ".gitmodules",
 783             ".gitattributes",
 784             ".cvsignore",
 785             ".svn",
 786             ".arch-ids",
 787             "{arch}",
 788             "=RELEASE-ID",
 789             "=meta-update",
 790             "=update",
 791             ".bzr",
 792             ".bzrignore",
 793             ".bzrtags",
 794             ".hg",
 795             ".hgignore",
 796             ".hgrags",
 797             "_darcs",
 798         ];
 799         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 800     }
 801
 802     // Copy the directories using our filter
 803     for item in src_dirs {
 804         let dst = &dst_dir.join(item);
 805         t!(fs::create_dir_all(dst));
 806         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 807     }
 808 }
 809
 810 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 811 pub struct Src;
 812
 813 impl Step for Src {
 814     /// The output path of the src installer tarball
 815     type Output = GeneratedTarball;
 816     const DEFAULT: bool = true;
 817     const ONLY_HOSTS: bool = true;
 818
 819     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 820         run.path("rust-src")
 821     }
 822
 823     fn make_run(run: RunConfig<'_>) {
 824         run.builder.ensure(Src);
 825     }
 826
 827     /// Creates the `rust-src` installer component
 828     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 829         let tarball = Tarball::new_targetless(builder, "rust-src");
 830
 831         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 832         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 833         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 834         // and fix them...
 835         //
 836         // NOTE: if you update the paths here, you also should update the "virtual" path
 837         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 838         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 839
 840         let src_files = ["Cargo.lock"];
 841         // This is the reduced set of paths which will become the rust-src component
 842         // (essentially libstd and all of its path dependencies).
 843         copy_src_dirs(
 844             builder,
 845             &builder.src,
 846             &["library", "src/llvm-project/libunwind"],
 847             &[
 848                 // not needed and contains symlinks which rustup currently
 849                 // chokes on when unpacking.
 850                 "library/backtrace/crates",
 851             ],
 852             &dst_src,
 853         );
 854         for file in src_files.iter() {
 855             builder.copy(&builder.src.join(file), &dst_src.join(file));
 856         }
 857
 858         tarball.generate()
 859     }
 860 }
 861
 862 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 863 pub struct PlainSourceTarball;
 864
 865 impl Step for PlainSourceTarball {
 866     /// Produces the location of the tarball generated
 867     type Output = GeneratedTarball;
 868     const DEFAULT: bool = true;
 869     const ONLY_HOSTS: bool = true;
 870
 871     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 872         let builder = run.builder;
 873         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 874     }
 875
 876     fn make_run(run: RunConfig<'_>) {
 877         run.builder.ensure(PlainSourceTarball);
 878     }
 879
 880     /// Creates the plain source tarball
 881     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 882         let tarball = Tarball::new(builder, "rustc", "src");
 883         let plain_dst_src = tarball.image_dir();
 884
 885         // This is the set of root paths which will become part of the source package
 886         let src_files = [
 887             "COPYRIGHT",
 888             "LICENSE-APACHE",
 889             "LICENSE-MIT",
 890             "CONTRIBUTING.md",
 891             "README.md",
 892             "RELEASES.md",
 893             "configure",
 894             "x.py",
 895             "config.toml.example",
 896             "Cargo.toml",
 897             "Cargo.lock",
 898         ];
 899         let src_dirs = ["src", "compiler", "library"];
 900
 901         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 902
 903         // Copy the files normally
 904         for item in &src_files {
 905             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 906         }
 907
 908         // Create the version file
 909         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 910         if let Some(sha) = builder.rust_sha() {
 911             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 912         }
 913
 914         // If we're building from git sources, we need to vendor a complete distribution.
 915         if builder.rust_info.is_git() {
 916             // Vendor all Cargo dependencies
 917             let mut cmd = Command::new(&builder.initial_cargo);
 918             cmd.arg("vendor")
 919                 .arg("--sync")
 920                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 921                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 922                 .current_dir(&plain_dst_src);
 923             builder.run(&mut cmd);
 924         }
 925
 926         tarball.bare()
 927     }
 928 }
 929
 930 // We have to run a few shell scripts, which choke quite a bit on both `\`
 931 // characters and on `C:\` paths, so normalize both of them away.
 932 pub fn sanitize_sh(path: &Path) -> String {
 933     let path = path.to_str().unwrap().replace("\\", "/");
 934     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 935
 936     fn unc_to_lfs(s: &str) -> &str {
 937         s.strip_prefix("//?/").unwrap_or(s)
 938     }
 939
 940     fn change_drive(s: &str) -> Option<String> {
 941         let mut ch = s.chars();
 942         let drive = ch.next().unwrap_or('C');
 943         if ch.next() != Some(':') {
 944             return None;
 945         }
 946         if ch.next() != Some('/') {
 947             return None;
 948         }
 949         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 950     }
 951 }
 952
 953 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 954 pub struct Cargo {
 955     pub compiler: Compiler,
 956     pub target: TargetSelection,
 957 }
 958
 959 impl Step for Cargo {
 960     type Output = Option<GeneratedTarball>;
 961     const DEFAULT: bool = true;
 962     const ONLY_HOSTS: bool = true;
 963
 964     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 965         let default = should_build_extended_tool(&run.builder, "cargo");
 966         run.path("cargo").default_condition(default)
 967     }
 968
 969     fn make_run(run: RunConfig<'_>) {
 970         run.builder.ensure(Cargo {
 971             compiler: run.builder.compiler_for(
 972                 run.builder.top_stage,
 973                 run.builder.config.build,
 974                 run.target,
 975             ),
 976             target: run.target,
 977         });
 978     }
 979
 980     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 981         let compiler = self.compiler;
 982         let target = self.target;
 983
 984         let cargo = builder.ensure(tool::Cargo { compiler, target });
 985         let src = builder.src.join("src/tools/cargo");
 986         let etc = src.join("src/etc");
 987
 988         // Prepare the image directory
 989         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 990         tarball.set_overlay(OverlayKind::Cargo);
 991
 992         tarball.add_file(&cargo, "bin", 0o755);
 993         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 994         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 995         tarball.add_dir(etc.join("man"), "share/man/man1");
 996         tarball.add_legal_and_readme_to("share/doc/cargo");
 997
 998         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 999             let dirent = dirent.expect("read dir entry");
1000             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1001                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1002             }
1003         }
1004
1005         Some(tarball.generate())
1006     }
1007 }
1008
1009 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1010 pub struct Rls {
1011     pub compiler: Compiler,
1012     pub target: TargetSelection,
1013 }
1014
1015 impl Step for Rls {
1016     type Output = Option<GeneratedTarball>;
1017     const ONLY_HOSTS: bool = true;
1018     const DEFAULT: bool = true;
1019
1020     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1021         let default = should_build_extended_tool(&run.builder, "rls");
1022         run.path("rls").default_condition(default)
1023     }
1024
1025     fn make_run(run: RunConfig<'_>) {
1026         run.builder.ensure(Rls {
1027             compiler: run.builder.compiler_for(
1028                 run.builder.top_stage,
1029                 run.builder.config.build,
1030                 run.target,
1031             ),
1032             target: run.target,
1033         });
1034     }
1035
1036     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1037         let compiler = self.compiler;
1038         let target = self.target;
1039
1040         let rls = builder
1041             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1042             .or_else(|| {
1043                 missing_tool("RLS", builder.build.config.missing_tools);
1044                 None
1045             })?;
1046
1047         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1048         tarball.set_overlay(OverlayKind::RLS);
1049         tarball.is_preview(true);
1050         tarball.add_file(rls, "bin", 0o755);
1051         tarball.add_legal_and_readme_to("share/doc/rls");
1052         Some(tarball.generate())
1053     }
1054 }
1055
1056 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1057 pub struct RustAnalyzer {
1058     pub compiler: Compiler,
1059     pub target: TargetSelection,
1060 }
1061
1062 impl Step for RustAnalyzer {
1063     type Output = Option<GeneratedTarball>;
1064     const DEFAULT: bool = true;
1065     const ONLY_HOSTS: bool = true;
1066
1067     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1068         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1069         run.path("rust-analyzer").default_condition(default)
1070     }
1071
1072     fn make_run(run: RunConfig<'_>) {
1073         run.builder.ensure(RustAnalyzer {
1074             compiler: run.builder.compiler_for(
1075                 run.builder.top_stage,
1076                 run.builder.config.build,
1077                 run.target,
1078             ),
1079             target: run.target,
1080         });
1081     }
1082
1083     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1084         // This prevents rust-analyzer from being built for "dist" or "install"
1085         // on the stable/beta channels. It is a nightly-only tool and should
1086         // not be included.
1087         if !builder.build.unstable_features() {
1088             return None;
1089         }
1090         let compiler = self.compiler;
1091         let target = self.target;
1092
1093         if target.contains("riscv64") {
1094             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1095             // to build. See #74813 for details.
1096             return None;
1097         }
1098
1099         let rust_analyzer = builder
1100             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1101             .expect("rust-analyzer always builds");
1102
1103         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1104         tarball.set_overlay(OverlayKind::RustAnalyzer);
1105         tarball.is_preview(true);
1106         tarball.add_file(rust_analyzer, "bin", 0o755);
1107         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1108         Some(tarball.generate())
1109     }
1110 }
1111
1112 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1113 pub struct Clippy {
1114     pub compiler: Compiler,
1115     pub target: TargetSelection,
1116 }
1117
1118 impl Step for Clippy {
1119     type Output = Option<GeneratedTarball>;
1120     const DEFAULT: bool = true;
1121     const ONLY_HOSTS: bool = true;
1122
1123     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1124         let default = should_build_extended_tool(&run.builder, "clippy");
1125         run.path("clippy").default_condition(default)
1126     }
1127
1128     fn make_run(run: RunConfig<'_>) {
1129         run.builder.ensure(Clippy {
1130             compiler: run.builder.compiler_for(
1131                 run.builder.top_stage,
1132                 run.builder.config.build,
1133                 run.target,
1134             ),
1135             target: run.target,
1136         });
1137     }
1138
1139     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1140         let compiler = self.compiler;
1141         let target = self.target;
1142
1143         // Prepare the image directory
1144         // We expect clippy to build, because we've exited this step above if tool
1145         // state for clippy isn't testing.
1146         let clippy = builder
1147             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1148             .expect("clippy expected to build - essential tool");
1149         let cargoclippy = builder
1150             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1151             .expect("clippy expected to build - essential tool");
1152
1153         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1154         tarball.set_overlay(OverlayKind::Clippy);
1155         tarball.is_preview(true);
1156         tarball.add_file(clippy, "bin", 0o755);
1157         tarball.add_file(cargoclippy, "bin", 0o755);
1158         tarball.add_legal_and_readme_to("share/doc/clippy");
1159         Some(tarball.generate())
1160     }
1161 }
1162
1163 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1164 pub struct Miri {
1165     pub compiler: Compiler,
1166     pub target: TargetSelection,
1167 }
1168
1169 impl Step for Miri {
1170     type Output = Option<GeneratedTarball>;
1171     const DEFAULT: bool = true;
1172     const ONLY_HOSTS: bool = true;
1173
1174     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1175         let default = should_build_extended_tool(&run.builder, "miri");
1176         run.path("miri").default_condition(default)
1177     }
1178
1179     fn make_run(run: RunConfig<'_>) {
1180         run.builder.ensure(Miri {
1181             compiler: run.builder.compiler_for(
1182                 run.builder.top_stage,
1183                 run.builder.config.build,
1184                 run.target,
1185             ),
1186             target: run.target,
1187         });
1188     }
1189
1190     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1191         // This prevents miri from being built for "dist" or "install"
1192         // on the stable/beta channels. It is a nightly-only tool and should
1193         // not be included.
1194         if !builder.build.unstable_features() {
1195             return None;
1196         }
1197         let compiler = self.compiler;
1198         let target = self.target;
1199
1200         let miri = builder
1201             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1202             .or_else(|| {
1203                 missing_tool("miri", builder.build.config.missing_tools);
1204                 None
1205             })?;
1206         let cargomiri = builder
1207             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1208             .or_else(|| {
1209                 missing_tool("cargo miri", builder.build.config.missing_tools);
1210                 None
1211             })?;
1212
1213         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1214         tarball.set_overlay(OverlayKind::Miri);
1215         tarball.is_preview(true);
1216         tarball.add_file(miri, "bin", 0o755);
1217         tarball.add_file(cargomiri, "bin", 0o755);
1218         tarball.add_legal_and_readme_to("share/doc/miri");
1219         Some(tarball.generate())
1220     }
1221 }
1222
1223 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1224 pub struct Rustfmt {
1225     pub compiler: Compiler,
1226     pub target: TargetSelection,
1227 }
1228
1229 impl Step for Rustfmt {
1230     type Output = Option<GeneratedTarball>;
1231     const DEFAULT: bool = true;
1232     const ONLY_HOSTS: bool = true;
1233
1234     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1235         let default = should_build_extended_tool(&run.builder, "rustfmt");
1236         run.path("rustfmt").default_condition(default)
1237     }
1238
1239     fn make_run(run: RunConfig<'_>) {
1240         run.builder.ensure(Rustfmt {
1241             compiler: run.builder.compiler_for(
1242                 run.builder.top_stage,
1243                 run.builder.config.build,
1244                 run.target,
1245             ),
1246             target: run.target,
1247         });
1248     }
1249
1250     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1251         let compiler = self.compiler;
1252         let target = self.target;
1253
1254         let rustfmt = builder
1255             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1256             .or_else(|| {
1257                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1258                 None
1259             })?;
1260         let cargofmt = builder
1261             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1262             .or_else(|| {
1263                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1264                 None
1265             })?;
1266
1267         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1268         tarball.set_overlay(OverlayKind::Rustfmt);
1269         tarball.is_preview(true);
1270         tarball.add_file(rustfmt, "bin", 0o755);
1271         tarball.add_file(cargofmt, "bin", 0o755);
1272         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1273         Some(tarball.generate())
1274     }
1275 }
1276
1277 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1278 pub struct RustDemangler {
1279     pub compiler: Compiler,
1280     pub target: TargetSelection,
1281 }
1282
1283 impl Step for RustDemangler {
1284     type Output = Option<GeneratedTarball>;
1285     const DEFAULT: bool = true;
1286     const ONLY_HOSTS: bool = true;
1287
1288     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1289         // While other tools use `should_build_extended_tool` to decide whether to be run by
1290         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1291         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1292         // we run the step by default when only `extended = true`, and decide whether to actually
1293         // run it or not later.
1294         let default = run.builder.config.extended;
1295         run.path("rust-demangler").default_condition(default)
1296     }
1297
1298     fn make_run(run: RunConfig<'_>) {
1299         run.builder.ensure(RustDemangler {
1300             compiler: run.builder.compiler_for(
1301                 run.builder.top_stage,
1302                 run.builder.config.build,
1303                 run.target,
1304             ),
1305             target: run.target,
1306         });
1307     }
1308
1309     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1310         let compiler = self.compiler;
1311         let target = self.target;
1312
1313         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1314         let condition = should_build_extended_tool(builder, "rust-demangler")
1315             || builder.config.profiler_enabled(target);
1316         if builder.config.extended && !condition {
1317             return None;
1318         }
1319
1320         let rust_demangler = builder
1321             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1322             .expect("rust-demangler expected to build - in-tree tool");
1323
1324         // Prepare the image directory
1325         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1326         tarball.set_overlay(OverlayKind::RustDemangler);
1327         tarball.is_preview(true);
1328         tarball.add_file(&rust_demangler, "bin", 0o755);
1329         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1330         Some(tarball.generate())
1331     }
1332 }
1333
1334 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1335 pub struct Extended {
1336     stage: u32,
1337     host: TargetSelection,
1338     target: TargetSelection,
1339 }
1340
1341 impl Step for Extended {
1342     type Output = ();
1343     const DEFAULT: bool = true;
1344     const ONLY_HOSTS: bool = true;
1345
1346     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1347         let builder = run.builder;
1348         run.path("extended").default_condition(builder.config.extended)
1349     }
1350
1351     fn make_run(run: RunConfig<'_>) {
1352         run.builder.ensure(Extended {
1353             stage: run.builder.top_stage,
1354             host: run.builder.config.build,
1355             target: run.target,
1356         });
1357     }
1358
1359     /// Creates a combined installer for the specified target in the provided stage.
1360     fn run(self, builder: &Builder<'_>) {
1361         let target = self.target;
1362         let stage = self.stage;
1363         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1364
1365         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1366
1367         let mut tarballs = Vec::new();
1368         let mut built_tools = HashSet::new();
1369         macro_rules! add_component {
1370             ($name:expr => $step:expr) => {
1371                 if let Some(tarball) = builder.ensure_if_default($step) {
1372                     tarballs.push(tarball);
1373                     built_tools.insert($name);
1374                 }
1375             };
1376         }
1377
1378         // When rust-std package split from rustc, we needed to ensure that during
1379         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1380         // the std files during uninstall. To do this ensure that rustc comes
1381         // before rust-std in the list below.
1382         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1383         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1384
1385         if target.contains("windows-gnu") {
1386             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1387         }
1388
1389         add_component!("rust-docs" => Docs { host: target });
1390         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1391         add_component!("cargo" => Cargo { compiler, target });
1392         add_component!("rustfmt" => Rustfmt { compiler, target });
1393         add_component!("rls" => Rls { compiler, target });
1394         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1395         add_component!("llvm-components" => LlvmTools { target });
1396         add_component!("clippy" => Clippy { compiler, target });
1397         add_component!("miri" => Miri { compiler, target });
1398         add_component!("analysis" => Analysis { compiler, target });
1399
1400         let etc = builder.src.join("src/etc/installer");
1401
1402         // Avoid producing tarballs during a dry run.
1403         if builder.config.dry_run {
1404             return;
1405         }
1406
1407         let tarball = Tarball::new(builder, "rust", &target.triple);
1408         let generated = tarball.combine(&tarballs);
1409
1410         let tmp = tmpdir(builder).join("combined-tarball");
1411         let work = generated.work_dir();
1412
1413         let mut license = String::new();
1414         license += &builder.read(&builder.src.join("COPYRIGHT"));
1415         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1416         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1417         license.push('\n');
1418         license.push('\n');
1419
1420         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1421         let mut rtf = rtf.to_string();
1422         rtf.push('\n');
1423         for line in license.lines() {
1424             rtf.push_str(line);
1425             rtf.push_str("\\line ");
1426         }
1427         rtf.push('}');
1428
1429         fn filter(contents: &str, marker: &str) -> String {
1430             let start = format!("tool-{}-start", marker);
1431             let end = format!("tool-{}-end", marker);
1432             let mut lines = Vec::new();
1433             let mut omitted = false;
1434             for line in contents.lines() {
1435                 if line.contains(&start) {
1436                     omitted = true;
1437                 } else if line.contains(&end) {
1438                     omitted = false;
1439                 } else if !omitted {
1440                     lines.push(line);
1441                 }
1442             }
1443
1444             lines.join("\n")
1445         }
1446
1447         let xform = |p: &Path| {
1448             let mut contents = t!(fs::read_to_string(p));
1449             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1450                 if !built_tools.contains(tool) {
1451                     contents = filter(&contents, tool);
1452                 }
1453             }
1454             let ret = tmp.join(p.file_name().unwrap());
1455             t!(fs::write(&ret, &contents));
1456             ret
1457         };
1458
1459         if target.contains("apple-darwin") {
1460             builder.info("building pkg installer");
1461             let pkg = tmp.join("pkg");
1462             let _ = fs::remove_dir_all(&pkg);
1463
1464             let pkgbuild = |component: &str| {
1465                 let mut cmd = Command::new("pkgbuild");
1466                 cmd.arg("--identifier")
1467                     .arg(format!("org.rust-lang.{}", component))
1468                     .arg("--scripts")
1469                     .arg(pkg.join(component))
1470                     .arg("--nopayload")
1471                     .arg(pkg.join(component).with_extension("pkg"));
1472                 builder.run(&mut cmd);
1473             };
1474
1475             let prepare = |name: &str| {
1476                 builder.create_dir(&pkg.join(name));
1477                 builder.cp_r(
1478                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1479                     &pkg.join(name),
1480                 );
1481                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1482                 pkgbuild(name);
1483             };
1484             prepare("rustc");
1485             prepare("cargo");
1486             prepare("rust-docs");
1487             prepare("rust-std");
1488             prepare("rust-analysis");
1489             prepare("clippy");
1490             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1491                 if built_tools.contains(tool) {
1492                     prepare(tool);
1493                 }
1494             }
1495             // create an 'uninstall' package
1496             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1497             pkgbuild("uninstall");
1498
1499             builder.create_dir(&pkg.join("res"));
1500             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1501             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1502             let mut cmd = Command::new("productbuild");
1503             cmd.arg("--distribution")
1504                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1505                 .arg("--resources")
1506                 .arg(pkg.join("res"))
1507                 .arg(distdir(builder).join(format!(
1508                     "{}-{}.pkg",
1509                     pkgname(builder, "rust"),
1510                     target.triple
1511                 )))
1512                 .arg("--package-path")
1513                 .arg(&pkg);
1514             let _time = timeit(builder);
1515             builder.run(&mut cmd);
1516         }
1517
1518         if target.contains("windows") {
1519             let exe = tmp.join("exe");
1520             let _ = fs::remove_dir_all(&exe);
1521
1522             let prepare = |name: &str| {
1523                 builder.create_dir(&exe.join(name));
1524                 let dir = if name == "rust-std" || name == "rust-analysis" {
1525                     format!("{}-{}", name, target.triple)
1526                 } else if name == "rls" {
1527                     "rls-preview".to_string()
1528                 } else if name == "rust-analyzer" {
1529                     "rust-analyzer-preview".to_string()
1530                 } else if name == "clippy" {
1531                     "clippy-preview".to_string()
1532                 } else if name == "rust-demangler" {
1533                     "rust-demangler-preview".to_string()
1534                 } else if name == "miri" {
1535                     "miri-preview".to_string()
1536                 } else {
1537                     name.to_string()
1538                 };
1539                 builder.cp_r(
1540                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1541                     &exe.join(name),
1542                 );
1543                 builder.remove(&exe.join(name).join("manifest.in"));
1544             };
1545             prepare("rustc");
1546             prepare("cargo");
1547             prepare("rust-analysis");
1548             prepare("rust-docs");
1549             prepare("rust-std");
1550             prepare("clippy");
1551             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1552                 if built_tools.contains(tool) {
1553                     prepare(tool);
1554                 }
1555             }
1556             if target.contains("windows-gnu") {
1557                 prepare("rust-mingw");
1558             }
1559
1560             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1561
1562             // Generate msi installer
1563             let wix_path = env::var_os("WIX")
1564                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1565             let wix = PathBuf::from(wix_path);
1566             let heat = wix.join("bin/heat.exe");
1567             let candle = wix.join("bin/candle.exe");
1568             let light = wix.join("bin/light.exe");
1569
1570             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1571             builder.run(
1572                 Command::new(&heat)
1573                     .current_dir(&exe)
1574                     .arg("dir")
1575                     .arg("rustc")
1576                     .args(&heat_flags)
1577                     .arg("-cg")
1578                     .arg("RustcGroup")
1579                     .arg("-dr")
1580                     .arg("Rustc")
1581                     .arg("-var")
1582                     .arg("var.RustcDir")
1583                     .arg("-out")
1584                     .arg(exe.join("RustcGroup.wxs")),
1585             );
1586             builder.run(
1587                 Command::new(&heat)
1588                     .current_dir(&exe)
1589                     .arg("dir")
1590                     .arg("rust-docs")
1591                     .args(&heat_flags)
1592                     .arg("-cg")
1593                     .arg("DocsGroup")
1594                     .arg("-dr")
1595                     .arg("Docs")
1596                     .arg("-var")
1597                     .arg("var.DocsDir")
1598                     .arg("-out")
1599                     .arg(exe.join("DocsGroup.wxs"))
1600                     .arg("-t")
1601                     .arg(etc.join("msi/squash-components.xsl")),
1602             );
1603             builder.run(
1604                 Command::new(&heat)
1605                     .current_dir(&exe)
1606                     .arg("dir")
1607                     .arg("cargo")
1608                     .args(&heat_flags)
1609                     .arg("-cg")
1610                     .arg("CargoGroup")
1611                     .arg("-dr")
1612                     .arg("Cargo")
1613                     .arg("-var")
1614                     .arg("var.CargoDir")
1615                     .arg("-out")
1616                     .arg(exe.join("CargoGroup.wxs"))
1617                     .arg("-t")
1618                     .arg(etc.join("msi/remove-duplicates.xsl")),
1619             );
1620             builder.run(
1621                 Command::new(&heat)
1622                     .current_dir(&exe)
1623                     .arg("dir")
1624                     .arg("rust-std")
1625                     .args(&heat_flags)
1626                     .arg("-cg")
1627                     .arg("StdGroup")
1628                     .arg("-dr")
1629                     .arg("Std")
1630                     .arg("-var")
1631                     .arg("var.StdDir")
1632                     .arg("-out")
1633                     .arg(exe.join("StdGroup.wxs")),
1634             );
1635             if built_tools.contains("rls") {
1636                 builder.run(
1637                     Command::new(&heat)
1638                         .current_dir(&exe)
1639                         .arg("dir")
1640                         .arg("rls")
1641                         .args(&heat_flags)
1642                         .arg("-cg")
1643                         .arg("RlsGroup")
1644                         .arg("-dr")
1645                         .arg("Rls")
1646                         .arg("-var")
1647                         .arg("var.RlsDir")
1648                         .arg("-out")
1649                         .arg(exe.join("RlsGroup.wxs"))
1650                         .arg("-t")
1651                         .arg(etc.join("msi/remove-duplicates.xsl")),
1652                 );
1653             }
1654             if built_tools.contains("rust-analyzer") {
1655                 builder.run(
1656                     Command::new(&heat)
1657                         .current_dir(&exe)
1658                         .arg("dir")
1659                         .arg("rust-analyzer")
1660                         .args(&heat_flags)
1661                         .arg("-cg")
1662                         .arg("RustAnalyzerGroup")
1663                         .arg("-dr")
1664                         .arg("RustAnalyzer")
1665                         .arg("-var")
1666                         .arg("var.RustAnalyzerDir")
1667                         .arg("-out")
1668                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1669                         .arg("-t")
1670                         .arg(etc.join("msi/remove-duplicates.xsl")),
1671                 );
1672             }
1673             builder.run(
1674                 Command::new(&heat)
1675                     .current_dir(&exe)
1676                     .arg("dir")
1677                     .arg("clippy")
1678                     .args(&heat_flags)
1679                     .arg("-cg")
1680                     .arg("ClippyGroup")
1681                     .arg("-dr")
1682                     .arg("Clippy")
1683                     .arg("-var")
1684                     .arg("var.ClippyDir")
1685                     .arg("-out")
1686                     .arg(exe.join("ClippyGroup.wxs"))
1687                     .arg("-t")
1688                     .arg(etc.join("msi/remove-duplicates.xsl")),
1689             );
1690             if built_tools.contains("rust-demangler") {
1691                 builder.run(
1692                     Command::new(&heat)
1693                         .current_dir(&exe)
1694                         .arg("dir")
1695                         .arg("rust-demangler")
1696                         .args(&heat_flags)
1697                         .arg("-cg")
1698                         .arg("RustDemanglerGroup")
1699                         .arg("-dr")
1700                         .arg("RustDemangler")
1701                         .arg("-var")
1702                         .arg("var.RustDemanglerDir")
1703                         .arg("-out")
1704                         .arg(exe.join("RustDemanglerGroup.wxs"))
1705                         .arg("-t")
1706                         .arg(etc.join("msi/remove-duplicates.xsl")),
1707                 );
1708             }
1709             if built_tools.contains("miri") {
1710                 builder.run(
1711                     Command::new(&heat)
1712                         .current_dir(&exe)
1713                         .arg("dir")
1714                         .arg("miri")
1715                         .args(&heat_flags)
1716                         .arg("-cg")
1717                         .arg("MiriGroup")
1718                         .arg("-dr")
1719                         .arg("Miri")
1720                         .arg("-var")
1721                         .arg("var.MiriDir")
1722                         .arg("-out")
1723                         .arg(exe.join("MiriGroup.wxs"))
1724                         .arg("-t")
1725                         .arg(etc.join("msi/remove-duplicates.xsl")),
1726                 );
1727             }
1728             builder.run(
1729                 Command::new(&heat)
1730                     .current_dir(&exe)
1731                     .arg("dir")
1732                     .arg("rust-analysis")
1733                     .args(&heat_flags)
1734                     .arg("-cg")
1735                     .arg("AnalysisGroup")
1736                     .arg("-dr")
1737                     .arg("Analysis")
1738                     .arg("-var")
1739                     .arg("var.AnalysisDir")
1740                     .arg("-out")
1741                     .arg(exe.join("AnalysisGroup.wxs"))
1742                     .arg("-t")
1743                     .arg(etc.join("msi/remove-duplicates.xsl")),
1744             );
1745             if target.contains("windows-gnu") {
1746                 builder.run(
1747                     Command::new(&heat)
1748                         .current_dir(&exe)
1749                         .arg("dir")
1750                         .arg("rust-mingw")
1751                         .args(&heat_flags)
1752                         .arg("-cg")
1753                         .arg("GccGroup")
1754                         .arg("-dr")
1755                         .arg("Gcc")
1756                         .arg("-var")
1757                         .arg("var.GccDir")
1758                         .arg("-out")
1759                         .arg(exe.join("GccGroup.wxs")),
1760                 );
1761             }
1762
1763             let candle = |input: &Path| {
1764                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1765                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1766                 let mut cmd = Command::new(&candle);
1767                 cmd.current_dir(&exe)
1768                     .arg("-nologo")
1769                     .arg("-dRustcDir=rustc")
1770                     .arg("-dDocsDir=rust-docs")
1771                     .arg("-dCargoDir=cargo")
1772                     .arg("-dStdDir=rust-std")
1773                     .arg("-dAnalysisDir=rust-analysis")
1774                     .arg("-dClippyDir=clippy")
1775                     .arg("-arch")
1776                     .arg(&arch)
1777                     .arg("-out")
1778                     .arg(&output)
1779                     .arg(&input);
1780                 add_env(builder, &mut cmd, target);
1781
1782                 if built_tools.contains("rust-demangler") {
1783                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1784                 }
1785                 if built_tools.contains("rls") {
1786                     cmd.arg("-dRlsDir=rls");
1787                 }
1788                 if built_tools.contains("rust-analyzer") {
1789                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1790                 }
1791                 if built_tools.contains("miri") {
1792                     cmd.arg("-dMiriDir=miri");
1793                 }
1794                 if target.contains("windows-gnu") {
1795                     cmd.arg("-dGccDir=rust-mingw");
1796                 }
1797                 builder.run(&mut cmd);
1798             };
1799             candle(&xform(&etc.join("msi/rust.wxs")));
1800             candle(&etc.join("msi/ui.wxs"));
1801             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1802             candle("RustcGroup.wxs".as_ref());
1803             candle("DocsGroup.wxs".as_ref());
1804             candle("CargoGroup.wxs".as_ref());
1805             candle("StdGroup.wxs".as_ref());
1806             candle("ClippyGroup.wxs".as_ref());
1807             if built_tools.contains("rust-demangler") {
1808                 candle("RustDemanglerGroup.wxs".as_ref());
1809             }
1810             if built_tools.contains("rls") {
1811                 candle("RlsGroup.wxs".as_ref());
1812             }
1813             if built_tools.contains("rust-analyzer") {
1814                 candle("RustAnalyzerGroup.wxs".as_ref());
1815             }
1816             if built_tools.contains("miri") {
1817                 candle("MiriGroup.wxs".as_ref());
1818             }
1819             candle("AnalysisGroup.wxs".as_ref());
1820
1821             if target.contains("windows-gnu") {
1822                 candle("GccGroup.wxs".as_ref());
1823             }
1824
1825             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1826             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1827             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1828
1829             builder.info(&format!("building `msi` installer with {:?}", light));
1830             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1831             let mut cmd = Command::new(&light);
1832             cmd.arg("-nologo")
1833                 .arg("-ext")
1834                 .arg("WixUIExtension")
1835                 .arg("-ext")
1836                 .arg("WixUtilExtension")
1837                 .arg("-out")
1838                 .arg(exe.join(&filename))
1839                 .arg("rust.wixobj")
1840                 .arg("ui.wixobj")
1841                 .arg("rustwelcomedlg.wixobj")
1842                 .arg("RustcGroup.wixobj")
1843                 .arg("DocsGroup.wixobj")
1844                 .arg("CargoGroup.wixobj")
1845                 .arg("StdGroup.wixobj")
1846                 .arg("AnalysisGroup.wixobj")
1847                 .arg("ClippyGroup.wixobj")
1848                 .current_dir(&exe);
1849
1850             if built_tools.contains("rls") {
1851                 cmd.arg("RlsGroup.wixobj");
1852             }
1853             if built_tools.contains("rust-analyzer") {
1854                 cmd.arg("RustAnalyzerGroup.wixobj");
1855             }
1856             if built_tools.contains("rust-demangler") {
1857                 cmd.arg("RustDemanglerGroup.wixobj");
1858             }
1859             if built_tools.contains("miri") {
1860                 cmd.arg("MiriGroup.wixobj");
1861             }
1862
1863             if target.contains("windows-gnu") {
1864                 cmd.arg("GccGroup.wixobj");
1865             }
1866             // ICE57 wrongly complains about the shortcuts
1867             cmd.arg("-sice:ICE57");
1868
1869             let _time = timeit(builder);
1870             builder.run(&mut cmd);
1871
1872             if !builder.config.dry_run {
1873                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1874             }
1875         }
1876     }
1877 }
1878
1879 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1880     let mut parts = builder.version.split('.');
1881     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1882         .env("CFG_RELEASE_NUM", &builder.version)
1883         .env("CFG_RELEASE", builder.rust_release())
1884         .env("CFG_VER_MAJOR", parts.next().unwrap())
1885         .env("CFG_VER_MINOR", parts.next().unwrap())
1886         .env("CFG_VER_PATCH", parts.next().unwrap())
1887         .env("CFG_VER_BUILD", "0") // just needed to build
1888         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1889         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1890         .env("CFG_BUILD", target.triple)
1891         .env("CFG_CHANNEL", &builder.config.channel);
1892
1893     if target.contains("windows-gnu") {
1894         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1895     } else {
1896         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1897     }
1898
1899     if target.contains("x86_64") {
1900         cmd.env("CFG_PLATFORM", "x64");
1901     } else {
1902         cmd.env("CFG_PLATFORM", "x86");
1903     }
1904 }
1905
1906 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1907 ///
1908
1909 /// Returns whether the files were actually copied.
1910 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1911     if let Some(config) = builder.config.target_config.get(&target) {
1912         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1913             // If the LLVM was externally provided, then we don't currently copy
1914             // artifacts into the sysroot. This is not necessarily the right
1915             // choice (in particular, it will require the LLVM dylib to be in
1916             // the linker's load path at runtime), but the common use case for
1917             // external LLVMs is distribution provided LLVMs, and in that case
1918             // they're usually in the standard search path (e.g., /usr/lib) and
1919             // copying them here is going to cause problems as we may end up
1920             // with the wrong files and isn't what distributions want.
1921             //
1922             // This behavior may be revisited in the future though.
1923             //
1924             // If the LLVM is coming from ourselves (just from CI) though, we
1925             // still want to install it, as it otherwise won't be available.
1926             return false;
1927         }
1928     }
1929
1930     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1931     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1932     // clear why this is the case, though. llvm-config will emit the versioned
1933     // paths and we don't want those in the sysroot (as we're expecting
1934     // unversioned paths).
1935     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1936         let src_libdir = builder.llvm_out(target).join("lib");
1937         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1938         if llvm_dylib_path.exists() {
1939             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1940         }
1941         !builder.config.dry_run
1942     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1943         let mut cmd = Command::new(llvm_config);
1944         cmd.arg("--libfiles");
1945         builder.verbose(&format!("running {:?}", cmd));
1946         let files = output(&mut cmd);
1947         let build_llvm_out = &builder.llvm_out(builder.config.build);
1948         let target_llvm_out = &builder.llvm_out(target);
1949         for file in files.trim_end().split(' ') {
1950             // If we're not using a custom LLVM, make sure we package for the target.
1951             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1952                 target_llvm_out.join(relative_path)
1953             } else {
1954                 PathBuf::from(file)
1955             };
1956             builder.install(&file, dst_libdir, 0o644);
1957         }
1958         !builder.config.dry_run
1959     } else {
1960         false
1961     }
1962 }
1963
1964 /// Maybe add libLLVM.so to the target lib-dir for linking.
1965 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1966     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1967     // We do not need to copy LLVM files into the sysroot if it is not
1968     // dynamically linked; it is already included into librustc_llvm
1969     // statically.
1970     if builder.config.llvm_link_shared {
1971         maybe_install_llvm(builder, target, &dst_libdir);
1972     }
1973 }
1974
1975 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1976 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1977     let dst_libdir =
1978         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1979     // We do not need to copy LLVM files into the sysroot if it is not
1980     // dynamically linked; it is already included into librustc_llvm
1981     // statically.
1982     if builder.config.llvm_link_shared {
1983         maybe_install_llvm(builder, target, &dst_libdir);
1984     }
1985 }
1986
1987 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1988 pub struct LlvmTools {
1989     pub target: TargetSelection,
1990 }
1991
1992 impl Step for LlvmTools {
1993     type Output = Option<GeneratedTarball>;
1994     const ONLY_HOSTS: bool = true;
1995     const DEFAULT: bool = true;
1996
1997     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1998         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1999         run.path("llvm-tools").default_condition(default)
2000     }
2001
2002     fn make_run(run: RunConfig<'_>) {
2003         run.builder.ensure(LlvmTools { target: run.target });
2004     }
2005
2006     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2007         let target = self.target;
2008
2009         /* run only if llvm-config isn't used */
2010         if let Some(config) = builder.config.target_config.get(&target) {
2011             if let Some(ref _s) = config.llvm_config {
2012                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2013                 return None;
2014             }
2015         }
2016
2017         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2018         tarball.set_overlay(OverlayKind::LLVM);
2019         tarball.is_preview(true);
2020
2021         // Prepare the image directory
2022         let src_bindir = builder.llvm_out(target).join("bin");
2023         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2024         for tool in LLVM_TOOLS {
2025             let exe = src_bindir.join(exe(tool, target));
2026             tarball.add_file(&exe, &dst_bindir, 0o755);
2027         }
2028
2029         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2030         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2031         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2032         // compiler libraries.
2033         maybe_install_llvm_target(builder, target, tarball.image_dir());
2034
2035         Some(tarball.generate())
2036     }
2037 }
2038
2039 // Tarball intended for internal consumption to ease rustc/std development.
2040 //
2041 // Should not be considered stable by end users.
2042 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2043 pub struct RustDev {
2044     pub target: TargetSelection,
2045 }
2046
2047 impl Step for RustDev {
2048     type Output = Option<GeneratedTarball>;
2049     const DEFAULT: bool = true;
2050     const ONLY_HOSTS: bool = true;
2051
2052     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2053         run.path("rust-dev")
2054     }
2055
2056     fn make_run(run: RunConfig<'_>) {
2057         run.builder.ensure(RustDev { target: run.target });
2058     }
2059
2060     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2061         let target = self.target;
2062
2063         /* run only if llvm-config isn't used */
2064         if let Some(config) = builder.config.target_config.get(&target) {
2065             if let Some(ref _s) = config.llvm_config {
2066                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2067                 return None;
2068             }
2069         }
2070
2071         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2072         tarball.set_overlay(OverlayKind::LLVM);
2073
2074         let src_bindir = builder.llvm_out(target).join("bin");
2075         for bin in &[
2076             "llvm-config",
2077             "llvm-ar",
2078             "llvm-objdump",
2079             "llvm-profdata",
2080             "llvm-bcanalyzer",
2081             "llvm-cov",
2082             "llvm-dwp",
2083         ] {
2084             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2085         }
2086
2087         // We don't build LLD on some platforms, so only add it if it exists
2088         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2089         if lld_path.exists() {
2090             tarball.add_file(lld_path, "bin", 0o755);
2091         }
2092
2093         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2094
2095         // Copy the include directory as well; needed mostly to build
2096         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2097         // just broadly useful to be able to link against the bundled LLVM.
2098         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2099
2100         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2101         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2102         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2103         // compiler libraries.
2104         let dst_libdir = tarball.image_dir().join("lib");
2105         maybe_install_llvm(builder, target, &dst_libdir);
2106         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2107         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2108
2109         Some(tarball.generate())
2110     }
2111 }
2112
2113 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2114 /// release process to avoid cloning the monorepo and building stuff.
2115 ///
2116 /// Should not be considered stable by end users.
2117 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2118 pub struct BuildManifest {
2119     pub target: TargetSelection,
2120 }
2121
2122 impl Step for BuildManifest {
2123     type Output = GeneratedTarball;
2124     const DEFAULT: bool = false;
2125     const ONLY_HOSTS: bool = true;
2126
2127     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2128         run.path("build-manifest")
2129     }
2130
2131     fn make_run(run: RunConfig<'_>) {
2132         run.builder.ensure(BuildManifest { target: run.target });
2133     }
2134
2135     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2136         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2137
2138         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2139         tarball.add_file(&build_manifest, "bin", 0o755);
2140         tarball.generate()
2141     }
2142 }
2143
2144 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2145 ///
2146 /// Currently this is the PGO profile data.
2147 ///
2148 /// Should not be considered stable by end users.
2149 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2150 pub struct ReproducibleArtifacts {
2151     pub target: TargetSelection,
2152 }
2153
2154 impl Step for ReproducibleArtifacts {
2155     type Output = Option<GeneratedTarball>;
2156     const DEFAULT: bool = true;
2157     const ONLY_HOSTS: bool = true;
2158
2159     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2160         run.path("reproducible-artifacts")
2161     }
2162
2163     fn make_run(run: RunConfig<'_>) {
2164         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2165     }
2166
2167     fn run(self, builder: &Builder<'_>) -> Self::Output {
2168         let mut added_anything = false;
2169         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2170         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2171             tarball.add_file(path, ".", 0o644);
2172             added_anything = true;
2173         }
2174         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2175             tarball.add_file(path, ".", 0o644);
2176             added_anything = true;
2177         }
2178         if added_anything { Some(tarball.generate()) } else { None }
2179     }
2180 }