src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("rust-docs").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         let builder = run.builder;
 100         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
 101     }
 102
 103     fn make_run(run: RunConfig<'_>) {
 104         run.builder.ensure(RustcDocs { host: run.target });
 105     }
 106
 107     /// Builds the `rustc-docs` installer component.
 108     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 109         let host = self.host;
 110         builder.default_doc(&[]);
 111
 112         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 113         tarball.set_product_name("Rustc Documentation");
 114         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 115         Some(tarball.generate())
 116     }
 117 }
 118
 119 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 120     let mut found = Vec::with_capacity(files.len());
 121
 122     for file in files {
 123         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 124
 125         if let Some(file_path) = file_path {
 126             found.push(file_path);
 127         } else {
 128             panic!("Could not find '{}' in {:?}", file, path);
 129         }
 130     }
 131
 132     found
 133 }
 134
 135 fn make_win_dist(
 136     rust_root: &Path,
 137     plat_root: &Path,
 138     target: TargetSelection,
 139     builder: &Builder<'_>,
 140 ) {
 141     //Ask gcc where it keeps its stuff
 142     let mut cmd = Command::new(builder.cc(target));
 143     cmd.arg("-print-search-dirs");
 144     let gcc_out = output(&mut cmd);
 145
 146     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 147     let mut lib_path = Vec::new();
 148
 149     for line in gcc_out.lines() {
 150         let idx = line.find(':').unwrap();
 151         let key = &line[..idx];
 152         let trim_chars: &[_] = &[' ', '='];
 153         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 154
 155         if key == "programs" {
 156             bin_path.extend(value);
 157         } else if key == "libraries" {
 158             lib_path.extend(value);
 159         }
 160     }
 161
 162     let compiler = if target == "i686-pc-windows-gnu" {
 163         "i686-w64-mingw32-gcc.exe"
 164     } else if target == "x86_64-pc-windows-gnu" {
 165         "x86_64-w64-mingw32-gcc.exe"
 166     } else {
 167         "gcc.exe"
 168     };
 169     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 170     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 171     if target.starts_with("i686-") {
 172         rustc_dlls.push("libgcc_s_dw2-1.dll");
 173     } else {
 174         rustc_dlls.push("libgcc_s_seh-1.dll");
 175     }
 176
 177     let target_libs = [
 178         //MinGW libs
 179         "libgcc.a",
 180         "libgcc_eh.a",
 181         "libgcc_s.a",
 182         "libm.a",
 183         "libmingw32.a",
 184         "libmingwex.a",
 185         "libstdc++.a",
 186         "libiconv.a",
 187         "libmoldname.a",
 188         "libpthread.a",
 189         //Windows import libs
 190         "libadvapi32.a",
 191         "libbcrypt.a",
 192         "libcomctl32.a",
 193         "libcomdlg32.a",
 194         "libcredui.a",
 195         "libcrypt32.a",
 196         "libdbghelp.a",
 197         "libgdi32.a",
 198         "libimagehlp.a",
 199         "libiphlpapi.a",
 200         "libkernel32.a",
 201         "libmsimg32.a",
 202         "libmsvcrt.a",
 203         "libodbc32.a",
 204         "libole32.a",
 205         "liboleaut32.a",
 206         "libopengl32.a",
 207         "libpsapi.a",
 208         "librpcrt4.a",
 209         "libsecur32.a",
 210         "libsetupapi.a",
 211         "libshell32.a",
 212         "libsynchronization.a",
 213         "libuser32.a",
 214         "libuserenv.a",
 215         "libuuid.a",
 216         "libwinhttp.a",
 217         "libwinmm.a",
 218         "libwinspool.a",
 219         "libws2_32.a",
 220         "libwsock32.a",
 221     ];
 222
 223     //Find mingw artifacts we want to bundle
 224     let target_tools = find_files(&target_tools, &bin_path);
 225     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 226     let target_libs = find_files(&target_libs, &lib_path);
 227
 228     // Copy runtime dlls next to rustc.exe
 229     let dist_bin_dir = rust_root.join("bin/");
 230     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 231     for src in rustc_dlls {
 232         builder.copy_to_folder(&src, &dist_bin_dir);
 233     }
 234
 235     //Copy platform tools to platform-specific bin directory
 236     let target_bin_dir = plat_root
 237         .join("lib")
 238         .join("rustlib")
 239         .join(target.triple)
 240         .join("bin")
 241         .join("self-contained");
 242     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 243     for src in target_tools {
 244         builder.copy_to_folder(&src, &target_bin_dir);
 245     }
 246
 247     // Warn windows-gnu users that the bundled GCC cannot compile C files
 248     builder.create(
 249         &target_bin_dir.join("GCC-WARNING.txt"),
 250         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 251          used as a linker. In order to be able to compile projects containing C code use \
 252          the GCC provided by MinGW or Cygwin.",
 253     );
 254
 255     //Copy platform libs to platform-specific lib directory
 256     let target_lib_dir = plat_root
 257         .join("lib")
 258         .join("rustlib")
 259         .join(target.triple)
 260         .join("lib")
 261         .join("self-contained");
 262     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 263     for src in target_libs {
 264         builder.copy_to_folder(&src, &target_lib_dir);
 265     }
 266 }
 267
 268 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 269 pub struct Mingw {
 270     pub host: TargetSelection,
 271 }
 272
 273 impl Step for Mingw {
 274     type Output = Option<GeneratedTarball>;
 275     const DEFAULT: bool = true;
 276
 277     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 278         run.path("rust-mingw")
 279     }
 280
 281     fn make_run(run: RunConfig<'_>) {
 282         run.builder.ensure(Mingw { host: run.target });
 283     }
 284
 285     /// Builds the `rust-mingw` installer component.
 286     ///
 287     /// This contains all the bits and pieces to run the MinGW Windows targets
 288     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 289     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 290         let host = self.host;
 291         if !host.contains("pc-windows-gnu") {
 292             return None;
 293         }
 294
 295         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 296         tarball.set_product_name("Rust MinGW");
 297
 298         // The first argument is a "temporary directory" which is just
 299         // thrown away (this contains the runtime DLLs included in the rustc package
 300         // above) and the second argument is where to place all the MinGW components
 301         // (which is what we want).
 302         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 303
 304         Some(tarball.generate())
 305     }
 306 }
 307
 308 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 309 pub struct Rustc {
 310     pub compiler: Compiler,
 311 }
 312
 313 impl Step for Rustc {
 314     type Output = GeneratedTarball;
 315     const DEFAULT: bool = true;
 316     const ONLY_HOSTS: bool = true;
 317
 318     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 319         run.path("rustc")
 320     }
 321
 322     fn make_run(run: RunConfig<'_>) {
 323         run.builder
 324             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 325     }
 326
 327     /// Creates the `rustc` installer component.
 328     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 329         let compiler = self.compiler;
 330         let host = self.compiler.host;
 331
 332         let tarball = Tarball::new(builder, "rustc", &host.triple);
 333
 334         // Prepare the rustc "image", what will actually end up getting installed
 335         prepare_image(builder, compiler, tarball.image_dir());
 336
 337         // On MinGW we've got a few runtime DLL dependencies that we need to
 338         // include. The first argument to this script is where to put these DLLs
 339         // (the image we're creating), and the second argument is a junk directory
 340         // to ignore all other MinGW stuff the script creates.
 341         //
 342         // On 32-bit MinGW we're always including a DLL which needs some extra
 343         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 344         // anything requiring us to distribute a license, but it's likely the
 345         // install will *also* include the rust-mingw package, which also needs
 346         // licenses, so to be safe we just include it here in all MinGW packages.
 347         if host.contains("pc-windows-gnu") {
 348             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 349             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 350         }
 351
 352         return tarball.generate();
 353
 354         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 355             let host = compiler.host;
 356             let src = builder.sysroot(compiler);
 357
 358             // Copy rustc/rustdoc binaries
 359             t!(fs::create_dir_all(image.join("bin")));
 360             builder.cp_r(&src.join("bin"), &image.join("bin"));
 361
 362             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 363
 364             let libdir_relative = builder.libdir_relative(compiler);
 365
 366             // Copy runtime DLLs needed by the compiler
 367             if libdir_relative.to_str() != Some("bin") {
 368                 let libdir = builder.rustc_libdir(compiler);
 369                 for entry in builder.read_dir(&libdir) {
 370                     let name = entry.file_name();
 371                     if let Some(s) = name.to_str() {
 372                         if is_dylib(s) {
 373                             // Don't use custom libdir here because ^lib/ will be resolved again
 374                             // with installer
 375                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 376                         }
 377                     }
 378                 }
 379             }
 380
 381             // Copy over the codegen backends
 382             let backends_src = builder.sysroot_codegen_backends(compiler);
 383             let backends_rel = backends_src
 384                 .strip_prefix(&src)
 385                 .unwrap()
 386                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 387                 .unwrap();
 388             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 389             let backends_dst = image.join("lib").join(&backends_rel);
 390
 391             t!(fs::create_dir_all(&backends_dst));
 392             builder.cp_r(&backends_src, &backends_dst);
 393
 394             // Copy libLLVM.so to the lib dir as well, if needed. While not
 395             // technically needed by rustc itself it's needed by lots of other
 396             // components like the llvm tools and LLD. LLD is included below and
 397             // tools/LLDB come later, so let's just throw it in the rustc
 398             // component for now.
 399             maybe_install_llvm_runtime(builder, host, image);
 400
 401             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 402             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 403             t!(fs::create_dir_all(&dst_dir));
 404
 405             // Copy over lld if it's there
 406             if builder.config.lld_enabled {
 407                 let rust_lld = exe("rust-lld", compiler.host);
 408                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 409                 // for `-Z gcc-ld=lld`
 410                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 411                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 412                 t!(fs::create_dir(&gcc_lld_dst_dir));
 413                 for flavor in ["ld", "ld64"] {
 414                     let exe_name = exe(flavor, compiler.host);
 415                     builder
 416                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 417                 }
 418             }
 419
 420             // Copy over llvm-dwp if it's there
 421             let exe = exe("rust-llvm-dwp", compiler.host);
 422             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 423
 424             // Man pages
 425             t!(fs::create_dir_all(image.join("share/man/man1")));
 426             let man_src = builder.src.join("src/doc/man");
 427             let man_dst = image.join("share/man/man1");
 428
 429             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 430             let time = env::var("SOURCE_DATE_EPOCH")
 431                 .map(|timestamp| {
 432                     let epoch = timestamp
 433                         .parse()
 434                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 435                         .unwrap();
 436
 437                     time::at(Timespec::new(epoch, 0))
 438                 })
 439                 .unwrap_or_else(|_| time::now());
 440
 441             let month_year = t!(time::strftime("%B %Y", &time));
 442             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 443             // to hardlink, and we don't want to edit the source templates
 444             for file_entry in builder.read_dir(&man_src) {
 445                 let page_src = file_entry.path();
 446                 let page_dst = man_dst.join(file_entry.file_name());
 447                 t!(fs::copy(&page_src, &page_dst));
 448                 // template in month/year and version number
 449                 builder.replace_in_file(
 450                     &page_dst,
 451                     &[
 452                         ("<INSERT DATE HERE>", &month_year),
 453                         ("<INSERT VERSION HERE>", &builder.version),
 454                     ],
 455                 );
 456             }
 457
 458             // Debugger scripts
 459             builder
 460                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 461
 462             // Misc license info
 463             let cp = |file: &str| {
 464                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 465             };
 466             cp("COPYRIGHT");
 467             cp("LICENSE-APACHE");
 468             cp("LICENSE-MIT");
 469             cp("README.md");
 470         }
 471     }
 472 }
 473
 474 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 475 pub struct DebuggerScripts {
 476     pub sysroot: Interned<PathBuf>,
 477     pub host: TargetSelection,
 478 }
 479
 480 impl Step for DebuggerScripts {
 481     type Output = ();
 482
 483     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 484         run.path("src/lldb_batchmode.py")
 485     }
 486
 487     fn make_run(run: RunConfig<'_>) {
 488         run.builder.ensure(DebuggerScripts {
 489             sysroot: run
 490                 .builder
 491                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 492             host: run.target,
 493         });
 494     }
 495
 496     /// Copies debugger scripts for `target` into the `sysroot` specified.
 497     fn run(self, builder: &Builder<'_>) {
 498         let host = self.host;
 499         let sysroot = self.sysroot;
 500         let dst = sysroot.join("lib/rustlib/etc");
 501         t!(fs::create_dir_all(&dst));
 502         let cp_debugger_script = |file: &str| {
 503             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 504         };
 505         if host.contains("windows-msvc") {
 506             // windbg debugger scripts
 507             builder.install(
 508                 &builder.src.join("src/etc/rust-windbg.cmd"),
 509                 &sysroot.join("bin"),
 510                 0o755,
 511             );
 512
 513             cp_debugger_script("natvis/intrinsic.natvis");
 514             cp_debugger_script("natvis/liballoc.natvis");
 515             cp_debugger_script("natvis/libcore.natvis");
 516             cp_debugger_script("natvis/libstd.natvis");
 517         } else {
 518             cp_debugger_script("rust_types.py");
 519
 520             // gdb debugger scripts
 521             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 522             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 523
 524             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 525             cp_debugger_script("gdb_lookup.py");
 526             cp_debugger_script("gdb_providers.py");
 527
 528             // lldb debugger scripts
 529             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 530
 531             cp_debugger_script("lldb_lookup.py");
 532             cp_debugger_script("lldb_providers.py");
 533             cp_debugger_script("lldb_commands")
 534         }
 535     }
 536 }
 537
 538 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 539     // The only true set of target libraries came from the build triple, so
 540     // let's reduce redundant work by only producing archives from that host.
 541     if compiler.host != builder.config.build {
 542         builder.info("\tskipping, not a build host");
 543         true
 544     } else {
 545         false
 546     }
 547 }
 548
 549 /// Copy stamped files into an image's `target/lib` directory.
 550 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 551     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 552     let self_contained_dst = dst.join("self-contained");
 553     t!(fs::create_dir_all(&dst));
 554     t!(fs::create_dir_all(&self_contained_dst));
 555     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 556         if dependency_type == DependencyType::TargetSelfContained {
 557             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 558         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 559             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 560         }
 561     }
 562 }
 563
 564 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 565 pub struct Std {
 566     pub compiler: Compiler,
 567     pub target: TargetSelection,
 568 }
 569
 570 impl Step for Std {
 571     type Output = Option<GeneratedTarball>;
 572     const DEFAULT: bool = true;
 573
 574     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 575         run.path("rust-std")
 576     }
 577
 578     fn make_run(run: RunConfig<'_>) {
 579         run.builder.ensure(Std {
 580             compiler: run.builder.compiler_for(
 581                 run.builder.top_stage,
 582                 run.builder.config.build,
 583                 run.target,
 584             ),
 585             target: run.target,
 586         });
 587     }
 588
 589     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 590         let compiler = self.compiler;
 591         let target = self.target;
 592
 593         if skip_host_target_lib(builder, compiler) {
 594             return None;
 595         }
 596
 597         builder.ensure(compile::Std { compiler, target });
 598
 599         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 600         tarball.include_target_in_component_name(true);
 601
 602         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 603         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 604         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 605
 606         Some(tarball.generate())
 607     }
 608 }
 609
 610 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 611 pub struct RustcDev {
 612     pub compiler: Compiler,
 613     pub target: TargetSelection,
 614 }
 615
 616 impl Step for RustcDev {
 617     type Output = Option<GeneratedTarball>;
 618     const DEFAULT: bool = true;
 619     const ONLY_HOSTS: bool = true;
 620
 621     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 622         run.path("rustc-dev")
 623     }
 624
 625     fn make_run(run: RunConfig<'_>) {
 626         run.builder.ensure(RustcDev {
 627             compiler: run.builder.compiler_for(
 628                 run.builder.top_stage,
 629                 run.builder.config.build,
 630                 run.target,
 631             ),
 632             target: run.target,
 633         });
 634     }
 635
 636     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 637         let compiler = self.compiler;
 638         let target = self.target;
 639         if skip_host_target_lib(builder, compiler) {
 640             return None;
 641         }
 642
 643         builder.ensure(compile::Rustc { compiler, target });
 644
 645         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 646
 647         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 648         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 649         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 650
 651         let src_files = &["Cargo.lock"];
 652         // This is the reduced set of paths which will become the rustc-dev component
 653         // (essentially the compiler crates and all of their path dependencies).
 654         copy_src_dirs(
 655             builder,
 656             &builder.src,
 657             &["compiler"],
 658             &[],
 659             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 660         );
 661         // This particular crate is used as a build dependency of the above.
 662         copy_src_dirs(
 663             builder,
 664             &builder.src,
 665             &["src/build_helper"],
 666             &[],
 667             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 668         );
 669         for file in src_files {
 670             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 671         }
 672
 673         Some(tarball.generate())
 674     }
 675 }
 676
 677 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 678 pub struct Analysis {
 679     pub compiler: Compiler,
 680     pub target: TargetSelection,
 681 }
 682
 683 impl Step for Analysis {
 684     type Output = Option<GeneratedTarball>;
 685     const DEFAULT: bool = true;
 686
 687     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 688         let default = should_build_extended_tool(&run.builder, "analysis");
 689         run.path("rust-analysis").default_condition(default)
 690     }
 691
 692     fn make_run(run: RunConfig<'_>) {
 693         run.builder.ensure(Analysis {
 694             // Find the actual compiler (handling the full bootstrap option) which
 695             // produced the save-analysis data because that data isn't copied
 696             // through the sysroot uplifting.
 697             compiler: run.builder.compiler_for(
 698                 run.builder.top_stage,
 699                 run.builder.config.build,
 700                 run.target,
 701             ),
 702             target: run.target,
 703         });
 704     }
 705
 706     /// Creates a tarball of save-analysis metadata, if available.
 707     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 708         let compiler = self.compiler;
 709         let target = self.target;
 710         if compiler.host != builder.config.build {
 711             return None;
 712         }
 713
 714         builder.ensure(compile::Std { compiler, target });
 715         let src = builder
 716             .stage_out(compiler, Mode::Std)
 717             .join(target.triple)
 718             .join(builder.cargo_dir())
 719             .join("deps")
 720             .join("save-analysis");
 721
 722         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 723         tarball.include_target_in_component_name(true);
 724         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 725         Some(tarball.generate())
 726     }
 727 }
 728
 729 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 730 /// `dst_dir`.
 731 fn copy_src_dirs(
 732     builder: &Builder<'_>,
 733     base: &Path,
 734     src_dirs: &[&str],
 735     exclude_dirs: &[&str],
 736     dst_dir: &Path,
 737 ) {
 738     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 739         let spath = match path.to_str() {
 740             Some(path) => path,
 741             None => return false,
 742         };
 743         if spath.ends_with('~') || spath.ends_with(".pyc") {
 744             return false;
 745         }
 746
 747         const LLVM_PROJECTS: &[&str] = &[
 748             "llvm-project/clang",
 749             "llvm-project\\clang",
 750             "llvm-project/libunwind",
 751             "llvm-project\\libunwind",
 752             "llvm-project/lld",
 753             "llvm-project\\lld",
 754             "llvm-project/lldb",
 755             "llvm-project\\lldb",
 756             "llvm-project/llvm",
 757             "llvm-project\\llvm",
 758             "llvm-project/compiler-rt",
 759             "llvm-project\\compiler-rt",
 760         ];
 761         if spath.contains("llvm-project")
 762             && !spath.ends_with("llvm-project")
 763             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 764         {
 765             return false;
 766         }
 767
 768         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 769         if LLVM_TEST.iter().any(|path| spath.contains(path))
 770             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 771         {
 772             return false;
 773         }
 774
 775         let full_path = Path::new(dir).join(path);
 776         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 777             return false;
 778         }
 779
 780         let excludes = [
 781             "CVS",
 782             "RCS",
 783             "SCCS",
 784             ".git",
 785             ".gitignore",
 786             ".gitmodules",
 787             ".gitattributes",
 788             ".cvsignore",
 789             ".svn",
 790             ".arch-ids",
 791             "{arch}",
 792             "=RELEASE-ID",
 793             "=meta-update",
 794             "=update",
 795             ".bzr",
 796             ".bzrignore",
 797             ".bzrtags",
 798             ".hg",
 799             ".hgignore",
 800             ".hgrags",
 801             "_darcs",
 802         ];
 803         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 804     }
 805
 806     // Copy the directories using our filter
 807     for item in src_dirs {
 808         let dst = &dst_dir.join(item);
 809         t!(fs::create_dir_all(dst));
 810         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 811     }
 812 }
 813
 814 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 815 pub struct Src;
 816
 817 impl Step for Src {
 818     /// The output path of the src installer tarball
 819     type Output = GeneratedTarball;
 820     const DEFAULT: bool = true;
 821     const ONLY_HOSTS: bool = true;
 822
 823     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 824         run.path("rust-src")
 825     }
 826
 827     fn make_run(run: RunConfig<'_>) {
 828         run.builder.ensure(Src);
 829     }
 830
 831     /// Creates the `rust-src` installer component
 832     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 833         let tarball = Tarball::new_targetless(builder, "rust-src");
 834
 835         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 836         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 837         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 838         // and fix them...
 839         //
 840         // NOTE: if you update the paths here, you also should update the "virtual" path
 841         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 842         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 843
 844         let src_files = ["Cargo.lock"];
 845         // This is the reduced set of paths which will become the rust-src component
 846         // (essentially libstd and all of its path dependencies).
 847         copy_src_dirs(
 848             builder,
 849             &builder.src,
 850             &["library", "src/llvm-project/libunwind"],
 851             &[
 852                 // not needed and contains symlinks which rustup currently
 853                 // chokes on when unpacking.
 854                 "library/backtrace/crates",
 855             ],
 856             &dst_src,
 857         );
 858         for file in src_files.iter() {
 859             builder.copy(&builder.src.join(file), &dst_src.join(file));
 860         }
 861
 862         tarball.generate()
 863     }
 864 }
 865
 866 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 867 pub struct PlainSourceTarball;
 868
 869 impl Step for PlainSourceTarball {
 870     /// Produces the location of the tarball generated
 871     type Output = GeneratedTarball;
 872     const DEFAULT: bool = true;
 873     const ONLY_HOSTS: bool = true;
 874
 875     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 876         let builder = run.builder;
 877         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 878     }
 879
 880     fn make_run(run: RunConfig<'_>) {
 881         run.builder.ensure(PlainSourceTarball);
 882     }
 883
 884     /// Creates the plain source tarball
 885     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 886         let tarball = Tarball::new(builder, "rustc", "src");
 887         let plain_dst_src = tarball.image_dir();
 888
 889         // This is the set of root paths which will become part of the source package
 890         let src_files = [
 891             "COPYRIGHT",
 892             "LICENSE-APACHE",
 893             "LICENSE-MIT",
 894             "CONTRIBUTING.md",
 895             "README.md",
 896             "RELEASES.md",
 897             "configure",
 898             "x.py",
 899             "config.toml.example",
 900             "Cargo.toml",
 901             "Cargo.lock",
 902         ];
 903         let src_dirs = ["src", "compiler", "library"];
 904
 905         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 906
 907         // Copy the files normally
 908         for item in &src_files {
 909             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 910         }
 911
 912         // Create the version file
 913         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 914         if let Some(sha) = builder.rust_sha() {
 915             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 916         }
 917
 918         // If we're building from git sources, we need to vendor a complete distribution.
 919         if builder.rust_info.is_git() {
 920             // Vendor all Cargo dependencies
 921             let mut cmd = Command::new(&builder.initial_cargo);
 922             cmd.arg("vendor")
 923                 .arg("--sync")
 924                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 925                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 926                 .current_dir(&plain_dst_src);
 927             builder.run(&mut cmd);
 928         }
 929
 930         tarball.bare()
 931     }
 932 }
 933
 934 // We have to run a few shell scripts, which choke quite a bit on both `\`
 935 // characters and on `C:\` paths, so normalize both of them away.
 936 pub fn sanitize_sh(path: &Path) -> String {
 937     let path = path.to_str().unwrap().replace("\\", "/");
 938     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 939
 940     fn unc_to_lfs(s: &str) -> &str {
 941         s.strip_prefix("//?/").unwrap_or(s)
 942     }
 943
 944     fn change_drive(s: &str) -> Option<String> {
 945         let mut ch = s.chars();
 946         let drive = ch.next().unwrap_or('C');
 947         if ch.next() != Some(':') {
 948             return None;
 949         }
 950         if ch.next() != Some('/') {
 951             return None;
 952         }
 953         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 954     }
 955 }
 956
 957 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 958 pub struct Cargo {
 959     pub compiler: Compiler,
 960     pub target: TargetSelection,
 961 }
 962
 963 impl Step for Cargo {
 964     type Output = Option<GeneratedTarball>;
 965     const DEFAULT: bool = true;
 966     const ONLY_HOSTS: bool = true;
 967
 968     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 969         let default = should_build_extended_tool(&run.builder, "cargo");
 970         run.path("cargo").default_condition(default)
 971     }
 972
 973     fn make_run(run: RunConfig<'_>) {
 974         run.builder.ensure(Cargo {
 975             compiler: run.builder.compiler_for(
 976                 run.builder.top_stage,
 977                 run.builder.config.build,
 978                 run.target,
 979             ),
 980             target: run.target,
 981         });
 982     }
 983
 984     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 985         let compiler = self.compiler;
 986         let target = self.target;
 987
 988         let cargo = builder.ensure(tool::Cargo { compiler, target });
 989         let src = builder.src.join("src/tools/cargo");
 990         let etc = src.join("src/etc");
 991
 992         // Prepare the image directory
 993         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 994         tarball.set_overlay(OverlayKind::Cargo);
 995
 996         tarball.add_file(&cargo, "bin", 0o755);
 997         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 998         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 999         tarball.add_dir(etc.join("man"), "share/man/man1");
1000         tarball.add_legal_and_readme_to("share/doc/cargo");
1001
1002         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1003             let dirent = dirent.expect("read dir entry");
1004             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1005                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1006             }
1007         }
1008
1009         Some(tarball.generate())
1010     }
1011 }
1012
1013 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1014 pub struct Rls {
1015     pub compiler: Compiler,
1016     pub target: TargetSelection,
1017 }
1018
1019 impl Step for Rls {
1020     type Output = Option<GeneratedTarball>;
1021     const ONLY_HOSTS: bool = true;
1022     const DEFAULT: bool = true;
1023
1024     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1025         let default = should_build_extended_tool(&run.builder, "rls");
1026         run.path("rls").default_condition(default)
1027     }
1028
1029     fn make_run(run: RunConfig<'_>) {
1030         run.builder.ensure(Rls {
1031             compiler: run.builder.compiler_for(
1032                 run.builder.top_stage,
1033                 run.builder.config.build,
1034                 run.target,
1035             ),
1036             target: run.target,
1037         });
1038     }
1039
1040     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1041         let compiler = self.compiler;
1042         let target = self.target;
1043
1044         let rls = builder
1045             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1046             .or_else(|| {
1047                 missing_tool("RLS", builder.build.config.missing_tools);
1048                 None
1049             })?;
1050
1051         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1052         tarball.set_overlay(OverlayKind::RLS);
1053         tarball.is_preview(true);
1054         tarball.add_file(rls, "bin", 0o755);
1055         tarball.add_legal_and_readme_to("share/doc/rls");
1056         Some(tarball.generate())
1057     }
1058 }
1059
1060 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1061 pub struct RustAnalyzer {
1062     pub compiler: Compiler,
1063     pub target: TargetSelection,
1064 }
1065
1066 impl Step for RustAnalyzer {
1067     type Output = Option<GeneratedTarball>;
1068     const DEFAULT: bool = true;
1069     const ONLY_HOSTS: bool = true;
1070
1071     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1072         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1073         run.path("rust-analyzer").default_condition(default)
1074     }
1075
1076     fn make_run(run: RunConfig<'_>) {
1077         run.builder.ensure(RustAnalyzer {
1078             compiler: run.builder.compiler_for(
1079                 run.builder.top_stage,
1080                 run.builder.config.build,
1081                 run.target,
1082             ),
1083             target: run.target,
1084         });
1085     }
1086
1087     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1088         // This prevents rust-analyzer from being built for "dist" or "install"
1089         // on the stable/beta channels. It is a nightly-only tool and should
1090         // not be included.
1091         if !builder.build.unstable_features() {
1092             return None;
1093         }
1094         let compiler = self.compiler;
1095         let target = self.target;
1096
1097         if target.contains("riscv64") {
1098             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1099             // to build. See #74813 for details.
1100             return None;
1101         }
1102
1103         let rust_analyzer = builder
1104             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1105             .expect("rust-analyzer always builds");
1106
1107         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1108         tarball.set_overlay(OverlayKind::RustAnalyzer);
1109         tarball.is_preview(true);
1110         tarball.add_file(rust_analyzer, "bin", 0o755);
1111         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1112         Some(tarball.generate())
1113     }
1114 }
1115
1116 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1117 pub struct Clippy {
1118     pub compiler: Compiler,
1119     pub target: TargetSelection,
1120 }
1121
1122 impl Step for Clippy {
1123     type Output = Option<GeneratedTarball>;
1124     const DEFAULT: bool = true;
1125     const ONLY_HOSTS: bool = true;
1126
1127     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1128         let default = should_build_extended_tool(&run.builder, "clippy");
1129         run.path("clippy").default_condition(default)
1130     }
1131
1132     fn make_run(run: RunConfig<'_>) {
1133         run.builder.ensure(Clippy {
1134             compiler: run.builder.compiler_for(
1135                 run.builder.top_stage,
1136                 run.builder.config.build,
1137                 run.target,
1138             ),
1139             target: run.target,
1140         });
1141     }
1142
1143     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1144         let compiler = self.compiler;
1145         let target = self.target;
1146
1147         // Prepare the image directory
1148         // We expect clippy to build, because we've exited this step above if tool
1149         // state for clippy isn't testing.
1150         let clippy = builder
1151             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1152             .expect("clippy expected to build - essential tool");
1153         let cargoclippy = builder
1154             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1155             .expect("clippy expected to build - essential tool");
1156
1157         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1158         tarball.set_overlay(OverlayKind::Clippy);
1159         tarball.is_preview(true);
1160         tarball.add_file(clippy, "bin", 0o755);
1161         tarball.add_file(cargoclippy, "bin", 0o755);
1162         tarball.add_legal_and_readme_to("share/doc/clippy");
1163         Some(tarball.generate())
1164     }
1165 }
1166
1167 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1168 pub struct Miri {
1169     pub compiler: Compiler,
1170     pub target: TargetSelection,
1171 }
1172
1173 impl Step for Miri {
1174     type Output = Option<GeneratedTarball>;
1175     const DEFAULT: bool = true;
1176     const ONLY_HOSTS: bool = true;
1177
1178     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1179         let default = should_build_extended_tool(&run.builder, "miri");
1180         run.path("miri").default_condition(default)
1181     }
1182
1183     fn make_run(run: RunConfig<'_>) {
1184         run.builder.ensure(Miri {
1185             compiler: run.builder.compiler_for(
1186                 run.builder.top_stage,
1187                 run.builder.config.build,
1188                 run.target,
1189             ),
1190             target: run.target,
1191         });
1192     }
1193
1194     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1195         // This prevents miri from being built for "dist" or "install"
1196         // on the stable/beta channels. It is a nightly-only tool and should
1197         // not be included.
1198         if !builder.build.unstable_features() {
1199             return None;
1200         }
1201         let compiler = self.compiler;
1202         let target = self.target;
1203
1204         let miri = builder
1205             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1206             .or_else(|| {
1207                 missing_tool("miri", builder.build.config.missing_tools);
1208                 None
1209             })?;
1210         let cargomiri = builder
1211             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1212             .or_else(|| {
1213                 missing_tool("cargo miri", builder.build.config.missing_tools);
1214                 None
1215             })?;
1216
1217         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1218         tarball.set_overlay(OverlayKind::Miri);
1219         tarball.is_preview(true);
1220         tarball.add_file(miri, "bin", 0o755);
1221         tarball.add_file(cargomiri, "bin", 0o755);
1222         tarball.add_legal_and_readme_to("share/doc/miri");
1223         Some(tarball.generate())
1224     }
1225 }
1226
1227 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1228 pub struct Rustfmt {
1229     pub compiler: Compiler,
1230     pub target: TargetSelection,
1231 }
1232
1233 impl Step for Rustfmt {
1234     type Output = Option<GeneratedTarball>;
1235     const DEFAULT: bool = true;
1236     const ONLY_HOSTS: bool = true;
1237
1238     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1239         let default = should_build_extended_tool(&run.builder, "rustfmt");
1240         run.path("rustfmt").default_condition(default)
1241     }
1242
1243     fn make_run(run: RunConfig<'_>) {
1244         run.builder.ensure(Rustfmt {
1245             compiler: run.builder.compiler_for(
1246                 run.builder.top_stage,
1247                 run.builder.config.build,
1248                 run.target,
1249             ),
1250             target: run.target,
1251         });
1252     }
1253
1254     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1255         let compiler = self.compiler;
1256         let target = self.target;
1257
1258         let rustfmt = builder
1259             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1260             .or_else(|| {
1261                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1262                 None
1263             })?;
1264         let cargofmt = builder
1265             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1266             .or_else(|| {
1267                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1268                 None
1269             })?;
1270
1271         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1272         tarball.set_overlay(OverlayKind::Rustfmt);
1273         tarball.is_preview(true);
1274         tarball.add_file(rustfmt, "bin", 0o755);
1275         tarball.add_file(cargofmt, "bin", 0o755);
1276         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1277         Some(tarball.generate())
1278     }
1279 }
1280
1281 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1282 pub struct RustDemangler {
1283     pub compiler: Compiler,
1284     pub target: TargetSelection,
1285 }
1286
1287 impl Step for RustDemangler {
1288     type Output = Option<GeneratedTarball>;
1289     const DEFAULT: bool = true;
1290     const ONLY_HOSTS: bool = true;
1291
1292     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1293         // While other tools use `should_build_extended_tool` to decide whether to be run by
1294         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1295         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1296         // we run the step by default when only `extended = true`, and decide whether to actually
1297         // run it or not later.
1298         let default = run.builder.config.extended;
1299         run.path("rust-demangler").default_condition(default)
1300     }
1301
1302     fn make_run(run: RunConfig<'_>) {
1303         run.builder.ensure(RustDemangler {
1304             compiler: run.builder.compiler_for(
1305                 run.builder.top_stage,
1306                 run.builder.config.build,
1307                 run.target,
1308             ),
1309             target: run.target,
1310         });
1311     }
1312
1313     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1314         let compiler = self.compiler;
1315         let target = self.target;
1316
1317         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1318         let condition = should_build_extended_tool(builder, "rust-demangler")
1319             || builder.config.profiler_enabled(target);
1320         if builder.config.extended && !condition {
1321             return None;
1322         }
1323
1324         let rust_demangler = builder
1325             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1326             .expect("rust-demangler expected to build - in-tree tool");
1327
1328         // Prepare the image directory
1329         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1330         tarball.set_overlay(OverlayKind::RustDemangler);
1331         tarball.is_preview(true);
1332         tarball.add_file(&rust_demangler, "bin", 0o755);
1333         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1334         Some(tarball.generate())
1335     }
1336 }
1337
1338 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1339 pub struct Extended {
1340     stage: u32,
1341     host: TargetSelection,
1342     target: TargetSelection,
1343 }
1344
1345 impl Step for Extended {
1346     type Output = ();
1347     const DEFAULT: bool = true;
1348     const ONLY_HOSTS: bool = true;
1349
1350     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1351         let builder = run.builder;
1352         run.path("extended").default_condition(builder.config.extended)
1353     }
1354
1355     fn make_run(run: RunConfig<'_>) {
1356         run.builder.ensure(Extended {
1357             stage: run.builder.top_stage,
1358             host: run.builder.config.build,
1359             target: run.target,
1360         });
1361     }
1362
1363     /// Creates a combined installer for the specified target in the provided stage.
1364     fn run(self, builder: &Builder<'_>) {
1365         let target = self.target;
1366         let stage = self.stage;
1367         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1368
1369         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1370
1371         let mut tarballs = Vec::new();
1372         let mut built_tools = HashSet::new();
1373         macro_rules! add_component {
1374             ($name:expr => $step:expr) => {
1375                 if let Some(tarball) = builder.ensure_if_default($step) {
1376                     tarballs.push(tarball);
1377                     built_tools.insert($name);
1378                 }
1379             };
1380         }
1381
1382         // When rust-std package split from rustc, we needed to ensure that during
1383         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1384         // the std files during uninstall. To do this ensure that rustc comes
1385         // before rust-std in the list below.
1386         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1387         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1388
1389         if target.contains("windows-gnu") {
1390             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1391         }
1392
1393         add_component!("rust-docs" => Docs { host: target });
1394         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1395         add_component!("cargo" => Cargo { compiler, target });
1396         add_component!("rustfmt" => Rustfmt { compiler, target });
1397         add_component!("rls" => Rls { compiler, target });
1398         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1399         add_component!("llvm-components" => LlvmTools { target });
1400         add_component!("clippy" => Clippy { compiler, target });
1401         add_component!("miri" => Miri { compiler, target });
1402         add_component!("analysis" => Analysis { compiler, target });
1403
1404         let etc = builder.src.join("src/etc/installer");
1405
1406         // Avoid producing tarballs during a dry run.
1407         if builder.config.dry_run {
1408             return;
1409         }
1410
1411         let tarball = Tarball::new(builder, "rust", &target.triple);
1412         let generated = tarball.combine(&tarballs);
1413
1414         let tmp = tmpdir(builder).join("combined-tarball");
1415         let work = generated.work_dir();
1416
1417         let mut license = String::new();
1418         license += &builder.read(&builder.src.join("COPYRIGHT"));
1419         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1420         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1421         license.push('\n');
1422         license.push('\n');
1423
1424         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1425         let mut rtf = rtf.to_string();
1426         rtf.push('\n');
1427         for line in license.lines() {
1428             rtf.push_str(line);
1429             rtf.push_str("\\line ");
1430         }
1431         rtf.push('}');
1432
1433         fn filter(contents: &str, marker: &str) -> String {
1434             let start = format!("tool-{}-start", marker);
1435             let end = format!("tool-{}-end", marker);
1436             let mut lines = Vec::new();
1437             let mut omitted = false;
1438             for line in contents.lines() {
1439                 if line.contains(&start) {
1440                     omitted = true;
1441                 } else if line.contains(&end) {
1442                     omitted = false;
1443                 } else if !omitted {
1444                     lines.push(line);
1445                 }
1446             }
1447
1448             lines.join("\n")
1449         }
1450
1451         let xform = |p: &Path| {
1452             let mut contents = t!(fs::read_to_string(p));
1453             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1454                 if !built_tools.contains(tool) {
1455                     contents = filter(&contents, tool);
1456                 }
1457             }
1458             let ret = tmp.join(p.file_name().unwrap());
1459             t!(fs::write(&ret, &contents));
1460             ret
1461         };
1462
1463         if target.contains("apple-darwin") {
1464             builder.info("building pkg installer");
1465             let pkg = tmp.join("pkg");
1466             let _ = fs::remove_dir_all(&pkg);
1467
1468             let pkgbuild = |component: &str| {
1469                 let mut cmd = Command::new("pkgbuild");
1470                 cmd.arg("--identifier")
1471                     .arg(format!("org.rust-lang.{}", component))
1472                     .arg("--scripts")
1473                     .arg(pkg.join(component))
1474                     .arg("--nopayload")
1475                     .arg(pkg.join(component).with_extension("pkg"));
1476                 builder.run(&mut cmd);
1477             };
1478
1479             let prepare = |name: &str| {
1480                 builder.create_dir(&pkg.join(name));
1481                 builder.cp_r(
1482                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1483                     &pkg.join(name),
1484                 );
1485                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1486                 pkgbuild(name);
1487             };
1488             prepare("rustc");
1489             prepare("cargo");
1490             prepare("rust-docs");
1491             prepare("rust-std");
1492             prepare("rust-analysis");
1493             prepare("clippy");
1494             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1495                 if built_tools.contains(tool) {
1496                     prepare(tool);
1497                 }
1498             }
1499             // create an 'uninstall' package
1500             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1501             pkgbuild("uninstall");
1502
1503             builder.create_dir(&pkg.join("res"));
1504             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1505             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1506             let mut cmd = Command::new("productbuild");
1507             cmd.arg("--distribution")
1508                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1509                 .arg("--resources")
1510                 .arg(pkg.join("res"))
1511                 .arg(distdir(builder).join(format!(
1512                     "{}-{}.pkg",
1513                     pkgname(builder, "rust"),
1514                     target.triple
1515                 )))
1516                 .arg("--package-path")
1517                 .arg(&pkg);
1518             let _time = timeit(builder);
1519             builder.run(&mut cmd);
1520         }
1521
1522         if target.contains("windows") {
1523             let exe = tmp.join("exe");
1524             let _ = fs::remove_dir_all(&exe);
1525
1526             let prepare = |name: &str| {
1527                 builder.create_dir(&exe.join(name));
1528                 let dir = if name == "rust-std" || name == "rust-analysis" {
1529                     format!("{}-{}", name, target.triple)
1530                 } else if name == "rls" {
1531                     "rls-preview".to_string()
1532                 } else if name == "rust-analyzer" {
1533                     "rust-analyzer-preview".to_string()
1534                 } else if name == "clippy" {
1535                     "clippy-preview".to_string()
1536                 } else if name == "rust-demangler" {
1537                     "rust-demangler-preview".to_string()
1538                 } else if name == "miri" {
1539                     "miri-preview".to_string()
1540                 } else {
1541                     name.to_string()
1542                 };
1543                 builder.cp_r(
1544                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1545                     &exe.join(name),
1546                 );
1547                 builder.remove(&exe.join(name).join("manifest.in"));
1548             };
1549             prepare("rustc");
1550             prepare("cargo");
1551             prepare("rust-analysis");
1552             prepare("rust-docs");
1553             prepare("rust-std");
1554             prepare("clippy");
1555             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1556                 if built_tools.contains(tool) {
1557                     prepare(tool);
1558                 }
1559             }
1560             if target.contains("windows-gnu") {
1561                 prepare("rust-mingw");
1562             }
1563
1564             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1565
1566             // Generate msi installer
1567             let wix_path = env::var_os("WIX")
1568                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1569             let wix = PathBuf::from(wix_path);
1570             let heat = wix.join("bin/heat.exe");
1571             let candle = wix.join("bin/candle.exe");
1572             let light = wix.join("bin/light.exe");
1573
1574             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1575             builder.run(
1576                 Command::new(&heat)
1577                     .current_dir(&exe)
1578                     .arg("dir")
1579                     .arg("rustc")
1580                     .args(&heat_flags)
1581                     .arg("-cg")
1582                     .arg("RustcGroup")
1583                     .arg("-dr")
1584                     .arg("Rustc")
1585                     .arg("-var")
1586                     .arg("var.RustcDir")
1587                     .arg("-out")
1588                     .arg(exe.join("RustcGroup.wxs")),
1589             );
1590             builder.run(
1591                 Command::new(&heat)
1592                     .current_dir(&exe)
1593                     .arg("dir")
1594                     .arg("rust-docs")
1595                     .args(&heat_flags)
1596                     .arg("-cg")
1597                     .arg("DocsGroup")
1598                     .arg("-dr")
1599                     .arg("Docs")
1600                     .arg("-var")
1601                     .arg("var.DocsDir")
1602                     .arg("-out")
1603                     .arg(exe.join("DocsGroup.wxs"))
1604                     .arg("-t")
1605                     .arg(etc.join("msi/squash-components.xsl")),
1606             );
1607             builder.run(
1608                 Command::new(&heat)
1609                     .current_dir(&exe)
1610                     .arg("dir")
1611                     .arg("cargo")
1612                     .args(&heat_flags)
1613                     .arg("-cg")
1614                     .arg("CargoGroup")
1615                     .arg("-dr")
1616                     .arg("Cargo")
1617                     .arg("-var")
1618                     .arg("var.CargoDir")
1619                     .arg("-out")
1620                     .arg(exe.join("CargoGroup.wxs"))
1621                     .arg("-t")
1622                     .arg(etc.join("msi/remove-duplicates.xsl")),
1623             );
1624             builder.run(
1625                 Command::new(&heat)
1626                     .current_dir(&exe)
1627                     .arg("dir")
1628                     .arg("rust-std")
1629                     .args(&heat_flags)
1630                     .arg("-cg")
1631                     .arg("StdGroup")
1632                     .arg("-dr")
1633                     .arg("Std")
1634                     .arg("-var")
1635                     .arg("var.StdDir")
1636                     .arg("-out")
1637                     .arg(exe.join("StdGroup.wxs")),
1638             );
1639             if built_tools.contains("rls") {
1640                 builder.run(
1641                     Command::new(&heat)
1642                         .current_dir(&exe)
1643                         .arg("dir")
1644                         .arg("rls")
1645                         .args(&heat_flags)
1646                         .arg("-cg")
1647                         .arg("RlsGroup")
1648                         .arg("-dr")
1649                         .arg("Rls")
1650                         .arg("-var")
1651                         .arg("var.RlsDir")
1652                         .arg("-out")
1653                         .arg(exe.join("RlsGroup.wxs"))
1654                         .arg("-t")
1655                         .arg(etc.join("msi/remove-duplicates.xsl")),
1656                 );
1657             }
1658             if built_tools.contains("rust-analyzer") {
1659                 builder.run(
1660                     Command::new(&heat)
1661                         .current_dir(&exe)
1662                         .arg("dir")
1663                         .arg("rust-analyzer")
1664                         .args(&heat_flags)
1665                         .arg("-cg")
1666                         .arg("RustAnalyzerGroup")
1667                         .arg("-dr")
1668                         .arg("RustAnalyzer")
1669                         .arg("-var")
1670                         .arg("var.RustAnalyzerDir")
1671                         .arg("-out")
1672                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1673                         .arg("-t")
1674                         .arg(etc.join("msi/remove-duplicates.xsl")),
1675                 );
1676             }
1677             builder.run(
1678                 Command::new(&heat)
1679                     .current_dir(&exe)
1680                     .arg("dir")
1681                     .arg("clippy")
1682                     .args(&heat_flags)
1683                     .arg("-cg")
1684                     .arg("ClippyGroup")
1685                     .arg("-dr")
1686                     .arg("Clippy")
1687                     .arg("-var")
1688                     .arg("var.ClippyDir")
1689                     .arg("-out")
1690                     .arg(exe.join("ClippyGroup.wxs"))
1691                     .arg("-t")
1692                     .arg(etc.join("msi/remove-duplicates.xsl")),
1693             );
1694             if built_tools.contains("rust-demangler") {
1695                 builder.run(
1696                     Command::new(&heat)
1697                         .current_dir(&exe)
1698                         .arg("dir")
1699                         .arg("rust-demangler")
1700                         .args(&heat_flags)
1701                         .arg("-cg")
1702                         .arg("RustDemanglerGroup")
1703                         .arg("-dr")
1704                         .arg("RustDemangler")
1705                         .arg("-var")
1706                         .arg("var.RustDemanglerDir")
1707                         .arg("-out")
1708                         .arg(exe.join("RustDemanglerGroup.wxs"))
1709                         .arg("-t")
1710                         .arg(etc.join("msi/remove-duplicates.xsl")),
1711                 );
1712             }
1713             if built_tools.contains("miri") {
1714                 builder.run(
1715                     Command::new(&heat)
1716                         .current_dir(&exe)
1717                         .arg("dir")
1718                         .arg("miri")
1719                         .args(&heat_flags)
1720                         .arg("-cg")
1721                         .arg("MiriGroup")
1722                         .arg("-dr")
1723                         .arg("Miri")
1724                         .arg("-var")
1725                         .arg("var.MiriDir")
1726                         .arg("-out")
1727                         .arg(exe.join("MiriGroup.wxs"))
1728                         .arg("-t")
1729                         .arg(etc.join("msi/remove-duplicates.xsl")),
1730                 );
1731             }
1732             builder.run(
1733                 Command::new(&heat)
1734                     .current_dir(&exe)
1735                     .arg("dir")
1736                     .arg("rust-analysis")
1737                     .args(&heat_flags)
1738                     .arg("-cg")
1739                     .arg("AnalysisGroup")
1740                     .arg("-dr")
1741                     .arg("Analysis")
1742                     .arg("-var")
1743                     .arg("var.AnalysisDir")
1744                     .arg("-out")
1745                     .arg(exe.join("AnalysisGroup.wxs"))
1746                     .arg("-t")
1747                     .arg(etc.join("msi/remove-duplicates.xsl")),
1748             );
1749             if target.contains("windows-gnu") {
1750                 builder.run(
1751                     Command::new(&heat)
1752                         .current_dir(&exe)
1753                         .arg("dir")
1754                         .arg("rust-mingw")
1755                         .args(&heat_flags)
1756                         .arg("-cg")
1757                         .arg("GccGroup")
1758                         .arg("-dr")
1759                         .arg("Gcc")
1760                         .arg("-var")
1761                         .arg("var.GccDir")
1762                         .arg("-out")
1763                         .arg(exe.join("GccGroup.wxs")),
1764                 );
1765             }
1766
1767             let candle = |input: &Path| {
1768                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1769                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1770                 let mut cmd = Command::new(&candle);
1771                 cmd.current_dir(&exe)
1772                     .arg("-nologo")
1773                     .arg("-dRustcDir=rustc")
1774                     .arg("-dDocsDir=rust-docs")
1775                     .arg("-dCargoDir=cargo")
1776                     .arg("-dStdDir=rust-std")
1777                     .arg("-dAnalysisDir=rust-analysis")
1778                     .arg("-dClippyDir=clippy")
1779                     .arg("-arch")
1780                     .arg(&arch)
1781                     .arg("-out")
1782                     .arg(&output)
1783                     .arg(&input);
1784                 add_env(builder, &mut cmd, target);
1785
1786                 if built_tools.contains("rust-demangler") {
1787                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1788                 }
1789                 if built_tools.contains("rls") {
1790                     cmd.arg("-dRlsDir=rls");
1791                 }
1792                 if built_tools.contains("rust-analyzer") {
1793                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1794                 }
1795                 if built_tools.contains("miri") {
1796                     cmd.arg("-dMiriDir=miri");
1797                 }
1798                 if target.contains("windows-gnu") {
1799                     cmd.arg("-dGccDir=rust-mingw");
1800                 }
1801                 builder.run(&mut cmd);
1802             };
1803             candle(&xform(&etc.join("msi/rust.wxs")));
1804             candle(&etc.join("msi/ui.wxs"));
1805             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1806             candle("RustcGroup.wxs".as_ref());
1807             candle("DocsGroup.wxs".as_ref());
1808             candle("CargoGroup.wxs".as_ref());
1809             candle("StdGroup.wxs".as_ref());
1810             candle("ClippyGroup.wxs".as_ref());
1811             if built_tools.contains("rust-demangler") {
1812                 candle("RustDemanglerGroup.wxs".as_ref());
1813             }
1814             if built_tools.contains("rls") {
1815                 candle("RlsGroup.wxs".as_ref());
1816             }
1817             if built_tools.contains("rust-analyzer") {
1818                 candle("RustAnalyzerGroup.wxs".as_ref());
1819             }
1820             if built_tools.contains("miri") {
1821                 candle("MiriGroup.wxs".as_ref());
1822             }
1823             candle("AnalysisGroup.wxs".as_ref());
1824
1825             if target.contains("windows-gnu") {
1826                 candle("GccGroup.wxs".as_ref());
1827             }
1828
1829             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1830             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1831             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1832
1833             builder.info(&format!("building `msi` installer with {:?}", light));
1834             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1835             let mut cmd = Command::new(&light);
1836             cmd.arg("-nologo")
1837                 .arg("-ext")
1838                 .arg("WixUIExtension")
1839                 .arg("-ext")
1840                 .arg("WixUtilExtension")
1841                 .arg("-out")
1842                 .arg(exe.join(&filename))
1843                 .arg("rust.wixobj")
1844                 .arg("ui.wixobj")
1845                 .arg("rustwelcomedlg.wixobj")
1846                 .arg("RustcGroup.wixobj")
1847                 .arg("DocsGroup.wixobj")
1848                 .arg("CargoGroup.wixobj")
1849                 .arg("StdGroup.wixobj")
1850                 .arg("AnalysisGroup.wixobj")
1851                 .arg("ClippyGroup.wixobj")
1852                 .current_dir(&exe);
1853
1854             if built_tools.contains("rls") {
1855                 cmd.arg("RlsGroup.wixobj");
1856             }
1857             if built_tools.contains("rust-analyzer") {
1858                 cmd.arg("RustAnalyzerGroup.wixobj");
1859             }
1860             if built_tools.contains("rust-demangler") {
1861                 cmd.arg("RustDemanglerGroup.wixobj");
1862             }
1863             if built_tools.contains("miri") {
1864                 cmd.arg("MiriGroup.wixobj");
1865             }
1866
1867             if target.contains("windows-gnu") {
1868                 cmd.arg("GccGroup.wixobj");
1869             }
1870             // ICE57 wrongly complains about the shortcuts
1871             cmd.arg("-sice:ICE57");
1872
1873             let _time = timeit(builder);
1874             builder.run(&mut cmd);
1875
1876             if !builder.config.dry_run {
1877                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1878             }
1879         }
1880     }
1881 }
1882
1883 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1884     let mut parts = builder.version.split('.');
1885     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1886         .env("CFG_RELEASE_NUM", &builder.version)
1887         .env("CFG_RELEASE", builder.rust_release())
1888         .env("CFG_VER_MAJOR", parts.next().unwrap())
1889         .env("CFG_VER_MINOR", parts.next().unwrap())
1890         .env("CFG_VER_PATCH", parts.next().unwrap())
1891         .env("CFG_VER_BUILD", "0") // just needed to build
1892         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1893         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1894         .env("CFG_BUILD", target.triple)
1895         .env("CFG_CHANNEL", &builder.config.channel);
1896
1897     if target.contains("windows-gnu") {
1898         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1899     } else {
1900         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1901     }
1902
1903     if target.contains("x86_64") {
1904         cmd.env("CFG_PLATFORM", "x64");
1905     } else {
1906         cmd.env("CFG_PLATFORM", "x86");
1907     }
1908 }
1909
1910 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1911 ///
1912
1913 /// Returns whether the files were actually copied.
1914 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1915     if let Some(config) = builder.config.target_config.get(&target) {
1916         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1917             // If the LLVM was externally provided, then we don't currently copy
1918             // artifacts into the sysroot. This is not necessarily the right
1919             // choice (in particular, it will require the LLVM dylib to be in
1920             // the linker's load path at runtime), but the common use case for
1921             // external LLVMs is distribution provided LLVMs, and in that case
1922             // they're usually in the standard search path (e.g., /usr/lib) and
1923             // copying them here is going to cause problems as we may end up
1924             // with the wrong files and isn't what distributions want.
1925             //
1926             // This behavior may be revisited in the future though.
1927             //
1928             // If the LLVM is coming from ourselves (just from CI) though, we
1929             // still want to install it, as it otherwise won't be available.
1930             return false;
1931         }
1932     }
1933
1934     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1935     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1936     // clear why this is the case, though. llvm-config will emit the versioned
1937     // paths and we don't want those in the sysroot (as we're expecting
1938     // unversioned paths).
1939     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1940         let src_libdir = builder.llvm_out(target).join("lib");
1941         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1942         if llvm_dylib_path.exists() {
1943             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1944         }
1945         !builder.config.dry_run
1946     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1947         let mut cmd = Command::new(llvm_config);
1948         cmd.arg("--libfiles");
1949         builder.verbose(&format!("running {:?}", cmd));
1950         let files = output(&mut cmd);
1951         let build_llvm_out = &builder.llvm_out(builder.config.build);
1952         let target_llvm_out = &builder.llvm_out(target);
1953         for file in files.trim_end().split(' ') {
1954             // If we're not using a custom LLVM, make sure we package for the target.
1955             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1956                 target_llvm_out.join(relative_path)
1957             } else {
1958                 PathBuf::from(file)
1959             };
1960             builder.install(&file, dst_libdir, 0o644);
1961         }
1962         !builder.config.dry_run
1963     } else {
1964         false
1965     }
1966 }
1967
1968 /// Maybe add libLLVM.so to the target lib-dir for linking.
1969 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1970     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1971     // We do not need to copy LLVM files into the sysroot if it is not
1972     // dynamically linked; it is already included into librustc_llvm
1973     // statically.
1974     if builder.config.llvm_link_shared {
1975         maybe_install_llvm(builder, target, &dst_libdir);
1976     }
1977 }
1978
1979 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1980 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1981     let dst_libdir =
1982         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1983     // We do not need to copy LLVM files into the sysroot if it is not
1984     // dynamically linked; it is already included into librustc_llvm
1985     // statically.
1986     if builder.config.llvm_link_shared {
1987         maybe_install_llvm(builder, target, &dst_libdir);
1988     }
1989 }
1990
1991 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1992 pub struct LlvmTools {
1993     pub target: TargetSelection,
1994 }
1995
1996 impl Step for LlvmTools {
1997     type Output = Option<GeneratedTarball>;
1998     const ONLY_HOSTS: bool = true;
1999     const DEFAULT: bool = true;
2000
2001     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2002         let default = should_build_extended_tool(&run.builder, "llvm-tools");
2003         run.path("llvm-tools").default_condition(default)
2004     }
2005
2006     fn make_run(run: RunConfig<'_>) {
2007         run.builder.ensure(LlvmTools { target: run.target });
2008     }
2009
2010     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2011         let target = self.target;
2012
2013         /* run only if llvm-config isn't used */
2014         if let Some(config) = builder.config.target_config.get(&target) {
2015             if let Some(ref _s) = config.llvm_config {
2016                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2017                 return None;
2018             }
2019         }
2020
2021         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2022         tarball.set_overlay(OverlayKind::LLVM);
2023         tarball.is_preview(true);
2024
2025         // Prepare the image directory
2026         let src_bindir = builder.llvm_out(target).join("bin");
2027         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2028         for tool in LLVM_TOOLS {
2029             let exe = src_bindir.join(exe(tool, target));
2030             tarball.add_file(&exe, &dst_bindir, 0o755);
2031         }
2032
2033         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2034         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2035         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2036         // compiler libraries.
2037         maybe_install_llvm_target(builder, target, tarball.image_dir());
2038
2039         Some(tarball.generate())
2040     }
2041 }
2042
2043 // Tarball intended for internal consumption to ease rustc/std development.
2044 //
2045 // Should not be considered stable by end users.
2046 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2047 pub struct RustDev {
2048     pub target: TargetSelection,
2049 }
2050
2051 impl Step for RustDev {
2052     type Output = Option<GeneratedTarball>;
2053     const DEFAULT: bool = true;
2054     const ONLY_HOSTS: bool = true;
2055
2056     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2057         run.path("rust-dev")
2058     }
2059
2060     fn make_run(run: RunConfig<'_>) {
2061         run.builder.ensure(RustDev { target: run.target });
2062     }
2063
2064     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2065         let target = self.target;
2066
2067         /* run only if llvm-config isn't used */
2068         if let Some(config) = builder.config.target_config.get(&target) {
2069             if let Some(ref _s) = config.llvm_config {
2070                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2071                 return None;
2072             }
2073         }
2074
2075         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2076         tarball.set_overlay(OverlayKind::LLVM);
2077
2078         let src_bindir = builder.llvm_out(target).join("bin");
2079         for bin in &[
2080             "llvm-config",
2081             "llvm-ar",
2082             "llvm-objdump",
2083             "llvm-profdata",
2084             "llvm-bcanalyzer",
2085             "llvm-cov",
2086             "llvm-dwp",
2087         ] {
2088             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2089         }
2090
2091         // We don't build LLD on some platforms, so only add it if it exists
2092         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2093         if lld_path.exists() {
2094             tarball.add_file(lld_path, "bin", 0o755);
2095         }
2096
2097         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2098
2099         // Copy the include directory as well; needed mostly to build
2100         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2101         // just broadly useful to be able to link against the bundled LLVM.
2102         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2103
2104         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2105         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2106         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2107         // compiler libraries.
2108         let dst_libdir = tarball.image_dir().join("lib");
2109         maybe_install_llvm(builder, target, &dst_libdir);
2110         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2111         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2112
2113         Some(tarball.generate())
2114     }
2115 }
2116
2117 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2118 /// release process to avoid cloning the monorepo and building stuff.
2119 ///
2120 /// Should not be considered stable by end users.
2121 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2122 pub struct BuildManifest {
2123     pub target: TargetSelection,
2124 }
2125
2126 impl Step for BuildManifest {
2127     type Output = GeneratedTarball;
2128     const DEFAULT: bool = false;
2129     const ONLY_HOSTS: bool = true;
2130
2131     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2132         run.path("build-manifest")
2133     }
2134
2135     fn make_run(run: RunConfig<'_>) {
2136         run.builder.ensure(BuildManifest { target: run.target });
2137     }
2138
2139     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2140         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2141
2142         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2143         tarball.add_file(&build_manifest, "bin", 0o755);
2144         tarball.generate()
2145     }
2146 }
2147
2148 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2149 ///
2150 /// Currently this is the PGO profile data.
2151 ///
2152 /// Should not be considered stable by end users.
2153 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2154 pub struct ReproducibleArtifacts {
2155     pub target: TargetSelection,
2156 }
2157
2158 impl Step for ReproducibleArtifacts {
2159     type Output = Option<GeneratedTarball>;
2160     const DEFAULT: bool = true;
2161     const ONLY_HOSTS: bool = true;
2162
2163     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2164         run.path("reproducible-artifacts")
2165     }
2166
2167     fn make_run(run: RunConfig<'_>) {
2168         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2169     }
2170
2171     fn run(self, builder: &Builder<'_>) -> Self::Output {
2172         let mut added_anything = false;
2173         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2174         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2175             tarball.add_file(path, ".", 0o644);
2176             added_anything = true;
2177         }
2178         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2179             tarball.add_file(path, ".", 0o644);
2180             added_anything = true;
2181         }
2182         if added_anything { Some(tarball.generate()) } else { None }
2183     }
2184 }