src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.path("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.path("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.contains("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.path("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.contains("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 for flavor in ["ld", "ld64"] {
 411                     let exe_name = exe(flavor, compiler.host);
 412                     builder
 413                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 414                 }
 415             }
 416
 417             // Man pages
 418             t!(fs::create_dir_all(image.join("share/man/man1")));
 419             let man_src = builder.src.join("src/doc/man");
 420             let man_dst = image.join("share/man/man1");
 421
 422             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 423             // to hardlink, and we don't want to edit the source templates
 424             for file_entry in builder.read_dir(&man_src) {
 425                 let page_src = file_entry.path();
 426                 let page_dst = man_dst.join(file_entry.file_name());
 427                 let src_text = t!(std::fs::read_to_string(&page_src));
 428                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 429                 t!(std::fs::write(&page_dst, &new_text));
 430                 t!(fs::copy(&page_src, &page_dst));
 431             }
 432
 433             // Debugger scripts
 434             builder
 435                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 436
 437             // Misc license info
 438             let cp = |file: &str| {
 439                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 440             };
 441             cp("COPYRIGHT");
 442             cp("LICENSE-APACHE");
 443             cp("LICENSE-MIT");
 444             cp("README.md");
 445         }
 446     }
 447 }
 448
 449 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 450 pub struct DebuggerScripts {
 451     pub sysroot: Interned<PathBuf>,
 452     pub host: TargetSelection,
 453 }
 454
 455 impl Step for DebuggerScripts {
 456     type Output = ();
 457
 458     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 459         run.path("src/lldb_batchmode.py")
 460     }
 461
 462     fn make_run(run: RunConfig<'_>) {
 463         run.builder.ensure(DebuggerScripts {
 464             sysroot: run
 465                 .builder
 466                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 467             host: run.target,
 468         });
 469     }
 470
 471     /// Copies debugger scripts for `target` into the `sysroot` specified.
 472     fn run(self, builder: &Builder<'_>) {
 473         let host = self.host;
 474         let sysroot = self.sysroot;
 475         let dst = sysroot.join("lib/rustlib/etc");
 476         t!(fs::create_dir_all(&dst));
 477         let cp_debugger_script = |file: &str| {
 478             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 479         };
 480         if host.contains("windows-msvc") {
 481             // windbg debugger scripts
 482             builder.install(
 483                 &builder.src.join("src/etc/rust-windbg.cmd"),
 484                 &sysroot.join("bin"),
 485                 0o755,
 486             );
 487
 488             cp_debugger_script("natvis/intrinsic.natvis");
 489             cp_debugger_script("natvis/liballoc.natvis");
 490             cp_debugger_script("natvis/libcore.natvis");
 491             cp_debugger_script("natvis/libstd.natvis");
 492         } else {
 493             cp_debugger_script("rust_types.py");
 494
 495             // gdb debugger scripts
 496             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 497             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 498
 499             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 500             cp_debugger_script("gdb_lookup.py");
 501             cp_debugger_script("gdb_providers.py");
 502
 503             // lldb debugger scripts
 504             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 505
 506             cp_debugger_script("lldb_lookup.py");
 507             cp_debugger_script("lldb_providers.py");
 508             cp_debugger_script("lldb_commands")
 509         }
 510     }
 511 }
 512
 513 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 514     // The only true set of target libraries came from the build triple, so
 515     // let's reduce redundant work by only producing archives from that host.
 516     if compiler.host != builder.config.build {
 517         builder.info("\tskipping, not a build host");
 518         true
 519     } else {
 520         false
 521     }
 522 }
 523
 524 /// Copy stamped files into an image's `target/lib` directory.
 525 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 526     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 527     let self_contained_dst = dst.join("self-contained");
 528     t!(fs::create_dir_all(&dst));
 529     t!(fs::create_dir_all(&self_contained_dst));
 530     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 531         if dependency_type == DependencyType::TargetSelfContained {
 532             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 533         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 534             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: TargetSelection,
 543 }
 544
 545 impl Step for Std {
 546     type Output = Option<GeneratedTarball>;
 547     const DEFAULT: bool = true;
 548
 549     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 550         run.path("rust-std")
 551     }
 552
 553     fn make_run(run: RunConfig<'_>) {
 554         run.builder.ensure(Std {
 555             compiler: run.builder.compiler_for(
 556                 run.builder.top_stage,
 557                 run.builder.config.build,
 558                 run.target,
 559             ),
 560             target: run.target,
 561         });
 562     }
 563
 564     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 565         let compiler = self.compiler;
 566         let target = self.target;
 567
 568         if skip_host_target_lib(builder, compiler) {
 569             return None;
 570         }
 571
 572         builder.ensure(compile::Std { compiler, target });
 573
 574         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 575         tarball.include_target_in_component_name(true);
 576
 577         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 578         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 579         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 580
 581         Some(tarball.generate())
 582     }
 583 }
 584
 585 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 586 pub struct RustcDev {
 587     pub compiler: Compiler,
 588     pub target: TargetSelection,
 589 }
 590
 591 impl Step for RustcDev {
 592     type Output = Option<GeneratedTarball>;
 593     const DEFAULT: bool = true;
 594     const ONLY_HOSTS: bool = true;
 595
 596     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 597         run.path("rustc-dev")
 598     }
 599
 600     fn make_run(run: RunConfig<'_>) {
 601         run.builder.ensure(RustcDev {
 602             compiler: run.builder.compiler_for(
 603                 run.builder.top_stage,
 604                 run.builder.config.build,
 605                 run.target,
 606             ),
 607             target: run.target,
 608         });
 609     }
 610
 611     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 612         let compiler = self.compiler;
 613         let target = self.target;
 614         if skip_host_target_lib(builder, compiler) {
 615             return None;
 616         }
 617
 618         builder.ensure(compile::Rustc { compiler, target });
 619
 620         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 621
 622         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 623         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 624         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 625
 626         let src_files = &["Cargo.lock"];
 627         // This is the reduced set of paths which will become the rustc-dev component
 628         // (essentially the compiler crates and all of their path dependencies).
 629         copy_src_dirs(
 630             builder,
 631             &builder.src,
 632             &["compiler"],
 633             &[],
 634             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 635         );
 636         for file in src_files {
 637             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 638         }
 639
 640         Some(tarball.generate())
 641     }
 642 }
 643
 644 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 645 pub struct Analysis {
 646     pub compiler: Compiler,
 647     pub target: TargetSelection,
 648 }
 649
 650 impl Step for Analysis {
 651     type Output = Option<GeneratedTarball>;
 652     const DEFAULT: bool = true;
 653
 654     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 655         let default = should_build_extended_tool(&run.builder, "analysis");
 656         run.path("rust-analysis").default_condition(default)
 657     }
 658
 659     fn make_run(run: RunConfig<'_>) {
 660         run.builder.ensure(Analysis {
 661             // Find the actual compiler (handling the full bootstrap option) which
 662             // produced the save-analysis data because that data isn't copied
 663             // through the sysroot uplifting.
 664             compiler: run.builder.compiler_for(
 665                 run.builder.top_stage,
 666                 run.builder.config.build,
 667                 run.target,
 668             ),
 669             target: run.target,
 670         });
 671     }
 672
 673     /// Creates a tarball of save-analysis metadata, if available.
 674     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 675         let compiler = self.compiler;
 676         let target = self.target;
 677         if compiler.host != builder.config.build {
 678             return None;
 679         }
 680
 681         builder.ensure(compile::Std { compiler, target });
 682         let src = builder
 683             .stage_out(compiler, Mode::Std)
 684             .join(target.triple)
 685             .join(builder.cargo_dir())
 686             .join("deps")
 687             .join("save-analysis");
 688
 689         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 690         tarball.include_target_in_component_name(true);
 691         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 692         Some(tarball.generate())
 693     }
 694 }
 695
 696 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 697 /// `dst_dir`.
 698 fn copy_src_dirs(
 699     builder: &Builder<'_>,
 700     base: &Path,
 701     src_dirs: &[&str],
 702     exclude_dirs: &[&str],
 703     dst_dir: &Path,
 704 ) {
 705     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 706         let spath = match path.to_str() {
 707             Some(path) => path,
 708             None => return false,
 709         };
 710         if spath.ends_with('~') || spath.ends_with(".pyc") {
 711             return false;
 712         }
 713
 714         const LLVM_PROJECTS: &[&str] = &[
 715             "llvm-project/clang",
 716             "llvm-project\\clang",
 717             "llvm-project/libunwind",
 718             "llvm-project\\libunwind",
 719             "llvm-project/lld",
 720             "llvm-project\\lld",
 721             "llvm-project/lldb",
 722             "llvm-project\\lldb",
 723             "llvm-project/llvm",
 724             "llvm-project\\llvm",
 725             "llvm-project/compiler-rt",
 726             "llvm-project\\compiler-rt",
 727             "llvm-project/cmake",
 728             "llvm-project\\cmake",
 729         ];
 730         if spath.contains("llvm-project")
 731             && !spath.ends_with("llvm-project")
 732             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 733         {
 734             return false;
 735         }
 736
 737         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 738         if LLVM_TEST.iter().any(|path| spath.contains(path))
 739             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 740         {
 741             return false;
 742         }
 743
 744         let full_path = Path::new(dir).join(path);
 745         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 746             return false;
 747         }
 748
 749         let excludes = [
 750             "CVS",
 751             "RCS",
 752             "SCCS",
 753             ".git",
 754             ".gitignore",
 755             ".gitmodules",
 756             ".gitattributes",
 757             ".cvsignore",
 758             ".svn",
 759             ".arch-ids",
 760             "{arch}",
 761             "=RELEASE-ID",
 762             "=meta-update",
 763             "=update",
 764             ".bzr",
 765             ".bzrignore",
 766             ".bzrtags",
 767             ".hg",
 768             ".hgignore",
 769             ".hgrags",
 770             "_darcs",
 771         ];
 772         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 773     }
 774
 775     // Copy the directories using our filter
 776     for item in src_dirs {
 777         let dst = &dst_dir.join(item);
 778         t!(fs::create_dir_all(dst));
 779         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 780     }
 781 }
 782
 783 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 784 pub struct Src;
 785
 786 impl Step for Src {
 787     /// The output path of the src installer tarball
 788     type Output = GeneratedTarball;
 789     const DEFAULT: bool = true;
 790     const ONLY_HOSTS: bool = true;
 791
 792     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 793         run.path("rust-src")
 794     }
 795
 796     fn make_run(run: RunConfig<'_>) {
 797         run.builder.ensure(Src);
 798     }
 799
 800     /// Creates the `rust-src` installer component
 801     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 802         let tarball = Tarball::new_targetless(builder, "rust-src");
 803
 804         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 805         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 806         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 807         // and fix them...
 808         //
 809         // NOTE: if you update the paths here, you also should update the "virtual" path
 810         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 811         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 812
 813         let src_files = ["Cargo.lock"];
 814         // This is the reduced set of paths which will become the rust-src component
 815         // (essentially libstd and all of its path dependencies).
 816         copy_src_dirs(
 817             builder,
 818             &builder.src,
 819             &["library", "src/llvm-project/libunwind"],
 820             &[
 821                 // not needed and contains symlinks which rustup currently
 822                 // chokes on when unpacking.
 823                 "library/backtrace/crates",
 824                 // these are 30MB combined and aren't necessary for building
 825                 // the standard library.
 826                 "library/stdarch/crates/Cargo.toml",
 827                 "library/stdarch/crates/stdarch-verify",
 828                 "library/stdarch/crates/intrinsic-test",
 829             ],
 830             &dst_src,
 831         );
 832         for file in src_files.iter() {
 833             builder.copy(&builder.src.join(file), &dst_src.join(file));
 834         }
 835
 836         tarball.generate()
 837     }
 838 }
 839
 840 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 841 pub struct PlainSourceTarball;
 842
 843 impl Step for PlainSourceTarball {
 844     /// Produces the location of the tarball generated
 845     type Output = GeneratedTarball;
 846     const DEFAULT: bool = true;
 847     const ONLY_HOSTS: bool = true;
 848
 849     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 850         let builder = run.builder;
 851         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 852     }
 853
 854     fn make_run(run: RunConfig<'_>) {
 855         run.builder.ensure(PlainSourceTarball);
 856     }
 857
 858     /// Creates the plain source tarball
 859     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 860         let tarball = Tarball::new(builder, "rustc", "src");
 861         let plain_dst_src = tarball.image_dir();
 862
 863         // This is the set of root paths which will become part of the source package
 864         let src_files = [
 865             "COPYRIGHT",
 866             "LICENSE-APACHE",
 867             "LICENSE-MIT",
 868             "CONTRIBUTING.md",
 869             "README.md",
 870             "RELEASES.md",
 871             "configure",
 872             "x.py",
 873             "config.toml.example",
 874             "Cargo.toml",
 875             "Cargo.lock",
 876         ];
 877         let src_dirs = ["src", "compiler", "library"];
 878
 879         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 880
 881         // Copy the files normally
 882         for item in &src_files {
 883             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 884         }
 885
 886         // Create the version file
 887         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 888         if let Some(sha) = builder.rust_sha() {
 889             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 890         }
 891
 892         // If we're building from git sources, we need to vendor a complete distribution.
 893         if builder.rust_info.is_git() {
 894             // Vendor all Cargo dependencies
 895             let mut cmd = Command::new(&builder.initial_cargo);
 896             cmd.arg("vendor")
 897                 .arg("--sync")
 898                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 899                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 900                 .current_dir(&plain_dst_src);
 901             builder.run(&mut cmd);
 902         }
 903
 904         tarball.bare()
 905     }
 906 }
 907
 908 // We have to run a few shell scripts, which choke quite a bit on both `\`
 909 // characters and on `C:\` paths, so normalize both of them away.
 910 pub fn sanitize_sh(path: &Path) -> String {
 911     let path = path.to_str().unwrap().replace("\\", "/");
 912     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 913
 914     fn unc_to_lfs(s: &str) -> &str {
 915         s.strip_prefix("//?/").unwrap_or(s)
 916     }
 917
 918     fn change_drive(s: &str) -> Option<String> {
 919         let mut ch = s.chars();
 920         let drive = ch.next().unwrap_or('C');
 921         if ch.next() != Some(':') {
 922             return None;
 923         }
 924         if ch.next() != Some('/') {
 925             return None;
 926         }
 927         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 928     }
 929 }
 930
 931 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 932 pub struct Cargo {
 933     pub compiler: Compiler,
 934     pub target: TargetSelection,
 935 }
 936
 937 impl Step for Cargo {
 938     type Output = Option<GeneratedTarball>;
 939     const DEFAULT: bool = true;
 940     const ONLY_HOSTS: bool = true;
 941
 942     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 943         let default = should_build_extended_tool(&run.builder, "cargo");
 944         run.path("cargo").default_condition(default)
 945     }
 946
 947     fn make_run(run: RunConfig<'_>) {
 948         run.builder.ensure(Cargo {
 949             compiler: run.builder.compiler_for(
 950                 run.builder.top_stage,
 951                 run.builder.config.build,
 952                 run.target,
 953             ),
 954             target: run.target,
 955         });
 956     }
 957
 958     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 959         let compiler = self.compiler;
 960         let target = self.target;
 961
 962         let cargo = builder.ensure(tool::Cargo { compiler, target });
 963         let src = builder.src.join("src/tools/cargo");
 964         let etc = src.join("src/etc");
 965
 966         // Prepare the image directory
 967         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 968         tarball.set_overlay(OverlayKind::Cargo);
 969
 970         tarball.add_file(&cargo, "bin", 0o755);
 971         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 972         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 973         tarball.add_dir(etc.join("man"), "share/man/man1");
 974         tarball.add_legal_and_readme_to("share/doc/cargo");
 975
 976         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 977             let dirent = dirent.expect("read dir entry");
 978             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 979                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 980             }
 981         }
 982
 983         Some(tarball.generate())
 984     }
 985 }
 986
 987 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 988 pub struct Rls {
 989     pub compiler: Compiler,
 990     pub target: TargetSelection,
 991 }
 992
 993 impl Step for Rls {
 994     type Output = Option<GeneratedTarball>;
 995     const ONLY_HOSTS: bool = true;
 996     const DEFAULT: bool = true;
 997
 998     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 999         let default = should_build_extended_tool(&run.builder, "rls");
1000         run.path("rls").default_condition(default)
1001     }
1002
1003     fn make_run(run: RunConfig<'_>) {
1004         run.builder.ensure(Rls {
1005             compiler: run.builder.compiler_for(
1006                 run.builder.top_stage,
1007                 run.builder.config.build,
1008                 run.target,
1009             ),
1010             target: run.target,
1011         });
1012     }
1013
1014     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1015         let compiler = self.compiler;
1016         let target = self.target;
1017
1018         let rls = builder
1019             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1020             .or_else(|| {
1021                 missing_tool("RLS", builder.build.config.missing_tools);
1022                 None
1023             })?;
1024
1025         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1026         tarball.set_overlay(OverlayKind::RLS);
1027         tarball.is_preview(true);
1028         tarball.add_file(rls, "bin", 0o755);
1029         tarball.add_legal_and_readme_to("share/doc/rls");
1030         Some(tarball.generate())
1031     }
1032 }
1033
1034 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1035 pub struct RustAnalyzer {
1036     pub compiler: Compiler,
1037     pub target: TargetSelection,
1038 }
1039
1040 impl Step for RustAnalyzer {
1041     type Output = Option<GeneratedTarball>;
1042     const DEFAULT: bool = true;
1043     const ONLY_HOSTS: bool = true;
1044
1045     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1046         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1047         run.path("rust-analyzer").default_condition(default)
1048     }
1049
1050     fn make_run(run: RunConfig<'_>) {
1051         run.builder.ensure(RustAnalyzer {
1052             compiler: run.builder.compiler_for(
1053                 run.builder.top_stage,
1054                 run.builder.config.build,
1055                 run.target,
1056             ),
1057             target: run.target,
1058         });
1059     }
1060
1061     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1062         // This prevents rust-analyzer from being built for "dist" or "install"
1063         // on the stable/beta channels. It is a nightly-only tool and should
1064         // not be included.
1065         if !builder.build.unstable_features() {
1066             return None;
1067         }
1068         let compiler = self.compiler;
1069         let target = self.target;
1070
1071         if target.contains("riscv64") {
1072             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1073             // to build. See #74813 for details.
1074             return None;
1075         }
1076
1077         let rust_analyzer = builder
1078             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1079             .expect("rust-analyzer always builds");
1080
1081         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1082         tarball.set_overlay(OverlayKind::RustAnalyzer);
1083         tarball.is_preview(true);
1084         tarball.add_file(rust_analyzer, "bin", 0o755);
1085         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1086         Some(tarball.generate())
1087     }
1088 }
1089
1090 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1091 pub struct Clippy {
1092     pub compiler: Compiler,
1093     pub target: TargetSelection,
1094 }
1095
1096 impl Step for Clippy {
1097     type Output = Option<GeneratedTarball>;
1098     const DEFAULT: bool = true;
1099     const ONLY_HOSTS: bool = true;
1100
1101     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1102         let default = should_build_extended_tool(&run.builder, "clippy");
1103         run.path("clippy").default_condition(default)
1104     }
1105
1106     fn make_run(run: RunConfig<'_>) {
1107         run.builder.ensure(Clippy {
1108             compiler: run.builder.compiler_for(
1109                 run.builder.top_stage,
1110                 run.builder.config.build,
1111                 run.target,
1112             ),
1113             target: run.target,
1114         });
1115     }
1116
1117     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1118         let compiler = self.compiler;
1119         let target = self.target;
1120
1121         // Prepare the image directory
1122         // We expect clippy to build, because we've exited this step above if tool
1123         // state for clippy isn't testing.
1124         let clippy = builder
1125             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1126             .expect("clippy expected to build - essential tool");
1127         let cargoclippy = builder
1128             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1129             .expect("clippy expected to build - essential tool");
1130
1131         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1132         tarball.set_overlay(OverlayKind::Clippy);
1133         tarball.is_preview(true);
1134         tarball.add_file(clippy, "bin", 0o755);
1135         tarball.add_file(cargoclippy, "bin", 0o755);
1136         tarball.add_legal_and_readme_to("share/doc/clippy");
1137         Some(tarball.generate())
1138     }
1139 }
1140
1141 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1142 pub struct Miri {
1143     pub compiler: Compiler,
1144     pub target: TargetSelection,
1145 }
1146
1147 impl Step for Miri {
1148     type Output = Option<GeneratedTarball>;
1149     const DEFAULT: bool = true;
1150     const ONLY_HOSTS: bool = true;
1151
1152     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1153         let default = should_build_extended_tool(&run.builder, "miri");
1154         run.path("miri").default_condition(default)
1155     }
1156
1157     fn make_run(run: RunConfig<'_>) {
1158         run.builder.ensure(Miri {
1159             compiler: run.builder.compiler_for(
1160                 run.builder.top_stage,
1161                 run.builder.config.build,
1162                 run.target,
1163             ),
1164             target: run.target,
1165         });
1166     }
1167
1168     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1169         // This prevents miri from being built for "dist" or "install"
1170         // on the stable/beta channels. It is a nightly-only tool and should
1171         // not be included.
1172         if !builder.build.unstable_features() {
1173             return None;
1174         }
1175         let compiler = self.compiler;
1176         let target = self.target;
1177
1178         let miri = builder
1179             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1180             .or_else(|| {
1181                 missing_tool("miri", builder.build.config.missing_tools);
1182                 None
1183             })?;
1184         let cargomiri = builder
1185             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1186             .or_else(|| {
1187                 missing_tool("cargo miri", builder.build.config.missing_tools);
1188                 None
1189             })?;
1190
1191         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1192         tarball.set_overlay(OverlayKind::Miri);
1193         tarball.is_preview(true);
1194         tarball.add_file(miri, "bin", 0o755);
1195         tarball.add_file(cargomiri, "bin", 0o755);
1196         tarball.add_legal_and_readme_to("share/doc/miri");
1197         Some(tarball.generate())
1198     }
1199 }
1200
1201 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1202 pub struct Rustfmt {
1203     pub compiler: Compiler,
1204     pub target: TargetSelection,
1205 }
1206
1207 impl Step for Rustfmt {
1208     type Output = Option<GeneratedTarball>;
1209     const DEFAULT: bool = true;
1210     const ONLY_HOSTS: bool = true;
1211
1212     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1213         let default = should_build_extended_tool(&run.builder, "rustfmt");
1214         run.path("rustfmt").default_condition(default)
1215     }
1216
1217     fn make_run(run: RunConfig<'_>) {
1218         run.builder.ensure(Rustfmt {
1219             compiler: run.builder.compiler_for(
1220                 run.builder.top_stage,
1221                 run.builder.config.build,
1222                 run.target,
1223             ),
1224             target: run.target,
1225         });
1226     }
1227
1228     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1229         let compiler = self.compiler;
1230         let target = self.target;
1231
1232         let rustfmt = builder
1233             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1234             .or_else(|| {
1235                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1236                 None
1237             })?;
1238         let cargofmt = builder
1239             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1240             .or_else(|| {
1241                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1242                 None
1243             })?;
1244
1245         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1246         tarball.set_overlay(OverlayKind::Rustfmt);
1247         tarball.is_preview(true);
1248         tarball.add_file(rustfmt, "bin", 0o755);
1249         tarball.add_file(cargofmt, "bin", 0o755);
1250         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1251         Some(tarball.generate())
1252     }
1253 }
1254
1255 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1256 pub struct RustDemangler {
1257     pub compiler: Compiler,
1258     pub target: TargetSelection,
1259 }
1260
1261 impl Step for RustDemangler {
1262     type Output = Option<GeneratedTarball>;
1263     const DEFAULT: bool = true;
1264     const ONLY_HOSTS: bool = true;
1265
1266     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1267         // While other tools use `should_build_extended_tool` to decide whether to be run by
1268         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1269         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1270         // we run the step by default when only `extended = true`, and decide whether to actually
1271         // run it or not later.
1272         let default = run.builder.config.extended;
1273         run.path("rust-demangler").default_condition(default)
1274     }
1275
1276     fn make_run(run: RunConfig<'_>) {
1277         run.builder.ensure(RustDemangler {
1278             compiler: run.builder.compiler_for(
1279                 run.builder.top_stage,
1280                 run.builder.config.build,
1281                 run.target,
1282             ),
1283             target: run.target,
1284         });
1285     }
1286
1287     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1288         let compiler = self.compiler;
1289         let target = self.target;
1290
1291         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1292         let condition = should_build_extended_tool(builder, "rust-demangler")
1293             || builder.config.profiler_enabled(target);
1294         if builder.config.extended && !condition {
1295             return None;
1296         }
1297
1298         let rust_demangler = builder
1299             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1300             .expect("rust-demangler expected to build - in-tree tool");
1301
1302         // Prepare the image directory
1303         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1304         tarball.set_overlay(OverlayKind::RustDemangler);
1305         tarball.is_preview(true);
1306         tarball.add_file(&rust_demangler, "bin", 0o755);
1307         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1308         Some(tarball.generate())
1309     }
1310 }
1311
1312 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1313 pub struct Extended {
1314     stage: u32,
1315     host: TargetSelection,
1316     target: TargetSelection,
1317 }
1318
1319 impl Step for Extended {
1320     type Output = ();
1321     const DEFAULT: bool = true;
1322     const ONLY_HOSTS: bool = true;
1323
1324     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1325         let builder = run.builder;
1326         run.path("extended").default_condition(builder.config.extended)
1327     }
1328
1329     fn make_run(run: RunConfig<'_>) {
1330         run.builder.ensure(Extended {
1331             stage: run.builder.top_stage,
1332             host: run.builder.config.build,
1333             target: run.target,
1334         });
1335     }
1336
1337     /// Creates a combined installer for the specified target in the provided stage.
1338     fn run(self, builder: &Builder<'_>) {
1339         let target = self.target;
1340         let stage = self.stage;
1341         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1342
1343         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1344
1345         let mut tarballs = Vec::new();
1346         let mut built_tools = HashSet::new();
1347         macro_rules! add_component {
1348             ($name:expr => $step:expr) => {
1349                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1350                     tarballs.push(tarball);
1351                     built_tools.insert($name);
1352                 }
1353             };
1354         }
1355
1356         // When rust-std package split from rustc, we needed to ensure that during
1357         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1358         // the std files during uninstall. To do this ensure that rustc comes
1359         // before rust-std in the list below.
1360         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1361         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1362
1363         if target.contains("windows-gnu") {
1364             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1365         }
1366
1367         add_component!("rust-docs" => Docs { host: target });
1368         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1369         add_component!("cargo" => Cargo { compiler, target });
1370         add_component!("rustfmt" => Rustfmt { compiler, target });
1371         add_component!("rls" => Rls { compiler, target });
1372         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1373         add_component!("llvm-components" => LlvmTools { target });
1374         add_component!("clippy" => Clippy { compiler, target });
1375         add_component!("miri" => Miri { compiler, target });
1376         add_component!("analysis" => Analysis { compiler, target });
1377
1378         let etc = builder.src.join("src/etc/installer");
1379
1380         // Avoid producing tarballs during a dry run.
1381         if builder.config.dry_run {
1382             return;
1383         }
1384
1385         let tarball = Tarball::new(builder, "rust", &target.triple);
1386         let generated = tarball.combine(&tarballs);
1387
1388         let tmp = tmpdir(builder).join("combined-tarball");
1389         let work = generated.work_dir();
1390
1391         let mut license = String::new();
1392         license += &builder.read(&builder.src.join("COPYRIGHT"));
1393         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1394         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1395         license.push('\n');
1396         license.push('\n');
1397
1398         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1399         let mut rtf = rtf.to_string();
1400         rtf.push('\n');
1401         for line in license.lines() {
1402             rtf.push_str(line);
1403             rtf.push_str("\\line ");
1404         }
1405         rtf.push('}');
1406
1407         fn filter(contents: &str, marker: &str) -> String {
1408             let start = format!("tool-{}-start", marker);
1409             let end = format!("tool-{}-end", marker);
1410             let mut lines = Vec::new();
1411             let mut omitted = false;
1412             for line in contents.lines() {
1413                 if line.contains(&start) {
1414                     omitted = true;
1415                 } else if line.contains(&end) {
1416                     omitted = false;
1417                 } else if !omitted {
1418                     lines.push(line);
1419                 }
1420             }
1421
1422             lines.join("\n")
1423         }
1424
1425         let xform = |p: &Path| {
1426             let mut contents = t!(fs::read_to_string(p));
1427             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1428                 if !built_tools.contains(tool) {
1429                     contents = filter(&contents, tool);
1430                 }
1431             }
1432             let ret = tmp.join(p.file_name().unwrap());
1433             t!(fs::write(&ret, &contents));
1434             ret
1435         };
1436
1437         if target.contains("apple-darwin") {
1438             builder.info("building pkg installer");
1439             let pkg = tmp.join("pkg");
1440             let _ = fs::remove_dir_all(&pkg);
1441
1442             let pkgbuild = |component: &str| {
1443                 let mut cmd = Command::new("pkgbuild");
1444                 cmd.arg("--identifier")
1445                     .arg(format!("org.rust-lang.{}", component))
1446                     .arg("--scripts")
1447                     .arg(pkg.join(component))
1448                     .arg("--nopayload")
1449                     .arg(pkg.join(component).with_extension("pkg"));
1450                 builder.run(&mut cmd);
1451             };
1452
1453             let prepare = |name: &str| {
1454                 builder.create_dir(&pkg.join(name));
1455                 builder.cp_r(
1456                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1457                     &pkg.join(name),
1458                 );
1459                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1460                 pkgbuild(name);
1461             };
1462             prepare("rustc");
1463             prepare("cargo");
1464             prepare("rust-std");
1465             prepare("rust-analysis");
1466             prepare("clippy");
1467             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1468                 if built_tools.contains(tool) {
1469                     prepare(tool);
1470                 }
1471             }
1472             // create an 'uninstall' package
1473             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1474             pkgbuild("uninstall");
1475
1476             builder.create_dir(&pkg.join("res"));
1477             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1478             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1479             let mut cmd = Command::new("productbuild");
1480             cmd.arg("--distribution")
1481                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1482                 .arg("--resources")
1483                 .arg(pkg.join("res"))
1484                 .arg(distdir(builder).join(format!(
1485                     "{}-{}.pkg",
1486                     pkgname(builder, "rust"),
1487                     target.triple
1488                 )))
1489                 .arg("--package-path")
1490                 .arg(&pkg);
1491             let _time = timeit(builder);
1492             builder.run(&mut cmd);
1493         }
1494
1495         if target.contains("windows") {
1496             let exe = tmp.join("exe");
1497             let _ = fs::remove_dir_all(&exe);
1498
1499             let prepare = |name: &str| {
1500                 builder.create_dir(&exe.join(name));
1501                 let dir = if name == "rust-std" || name == "rust-analysis" {
1502                     format!("{}-{}", name, target.triple)
1503                 } else if name == "rls" {
1504                     "rls-preview".to_string()
1505                 } else if name == "rust-analyzer" {
1506                     "rust-analyzer-preview".to_string()
1507                 } else if name == "clippy" {
1508                     "clippy-preview".to_string()
1509                 } else if name == "rust-demangler" {
1510                     "rust-demangler-preview".to_string()
1511                 } else if name == "miri" {
1512                     "miri-preview".to_string()
1513                 } else {
1514                     name.to_string()
1515                 };
1516                 builder.cp_r(
1517                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1518                     &exe.join(name),
1519                 );
1520                 builder.remove(&exe.join(name).join("manifest.in"));
1521             };
1522             prepare("rustc");
1523             prepare("cargo");
1524             prepare("rust-analysis");
1525             prepare("rust-docs");
1526             prepare("rust-std");
1527             prepare("clippy");
1528             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1529                 if built_tools.contains(tool) {
1530                     prepare(tool);
1531                 }
1532             }
1533             if target.contains("windows-gnu") {
1534                 prepare("rust-mingw");
1535             }
1536
1537             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1538
1539             // Generate msi installer
1540             let wix_path = env::var_os("WIX")
1541                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1542             let wix = PathBuf::from(wix_path);
1543             let heat = wix.join("bin/heat.exe");
1544             let candle = wix.join("bin/candle.exe");
1545             let light = wix.join("bin/light.exe");
1546
1547             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1548             builder.run(
1549                 Command::new(&heat)
1550                     .current_dir(&exe)
1551                     .arg("dir")
1552                     .arg("rustc")
1553                     .args(&heat_flags)
1554                     .arg("-cg")
1555                     .arg("RustcGroup")
1556                     .arg("-dr")
1557                     .arg("Rustc")
1558                     .arg("-var")
1559                     .arg("var.RustcDir")
1560                     .arg("-out")
1561                     .arg(exe.join("RustcGroup.wxs")),
1562             );
1563             builder.run(
1564                 Command::new(&heat)
1565                     .current_dir(&exe)
1566                     .arg("dir")
1567                     .arg("rust-docs")
1568                     .args(&heat_flags)
1569                     .arg("-cg")
1570                     .arg("DocsGroup")
1571                     .arg("-dr")
1572                     .arg("Docs")
1573                     .arg("-var")
1574                     .arg("var.DocsDir")
1575                     .arg("-out")
1576                     .arg(exe.join("DocsGroup.wxs"))
1577                     .arg("-t")
1578                     .arg(etc.join("msi/squash-components.xsl")),
1579             );
1580             builder.run(
1581                 Command::new(&heat)
1582                     .current_dir(&exe)
1583                     .arg("dir")
1584                     .arg("cargo")
1585                     .args(&heat_flags)
1586                     .arg("-cg")
1587                     .arg("CargoGroup")
1588                     .arg("-dr")
1589                     .arg("Cargo")
1590                     .arg("-var")
1591                     .arg("var.CargoDir")
1592                     .arg("-out")
1593                     .arg(exe.join("CargoGroup.wxs"))
1594                     .arg("-t")
1595                     .arg(etc.join("msi/remove-duplicates.xsl")),
1596             );
1597             builder.run(
1598                 Command::new(&heat)
1599                     .current_dir(&exe)
1600                     .arg("dir")
1601                     .arg("rust-std")
1602                     .args(&heat_flags)
1603                     .arg("-cg")
1604                     .arg("StdGroup")
1605                     .arg("-dr")
1606                     .arg("Std")
1607                     .arg("-var")
1608                     .arg("var.StdDir")
1609                     .arg("-out")
1610                     .arg(exe.join("StdGroup.wxs")),
1611             );
1612             if built_tools.contains("rls") {
1613                 builder.run(
1614                     Command::new(&heat)
1615                         .current_dir(&exe)
1616                         .arg("dir")
1617                         .arg("rls")
1618                         .args(&heat_flags)
1619                         .arg("-cg")
1620                         .arg("RlsGroup")
1621                         .arg("-dr")
1622                         .arg("Rls")
1623                         .arg("-var")
1624                         .arg("var.RlsDir")
1625                         .arg("-out")
1626                         .arg(exe.join("RlsGroup.wxs"))
1627                         .arg("-t")
1628                         .arg(etc.join("msi/remove-duplicates.xsl")),
1629                 );
1630             }
1631             if built_tools.contains("rust-analyzer") {
1632                 builder.run(
1633                     Command::new(&heat)
1634                         .current_dir(&exe)
1635                         .arg("dir")
1636                         .arg("rust-analyzer")
1637                         .args(&heat_flags)
1638                         .arg("-cg")
1639                         .arg("RustAnalyzerGroup")
1640                         .arg("-dr")
1641                         .arg("RustAnalyzer")
1642                         .arg("-var")
1643                         .arg("var.RustAnalyzerDir")
1644                         .arg("-out")
1645                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1646                         .arg("-t")
1647                         .arg(etc.join("msi/remove-duplicates.xsl")),
1648                 );
1649             }
1650             builder.run(
1651                 Command::new(&heat)
1652                     .current_dir(&exe)
1653                     .arg("dir")
1654                     .arg("clippy")
1655                     .args(&heat_flags)
1656                     .arg("-cg")
1657                     .arg("ClippyGroup")
1658                     .arg("-dr")
1659                     .arg("Clippy")
1660                     .arg("-var")
1661                     .arg("var.ClippyDir")
1662                     .arg("-out")
1663                     .arg(exe.join("ClippyGroup.wxs"))
1664                     .arg("-t")
1665                     .arg(etc.join("msi/remove-duplicates.xsl")),
1666             );
1667             if built_tools.contains("rust-demangler") {
1668                 builder.run(
1669                     Command::new(&heat)
1670                         .current_dir(&exe)
1671                         .arg("dir")
1672                         .arg("rust-demangler")
1673                         .args(&heat_flags)
1674                         .arg("-cg")
1675                         .arg("RustDemanglerGroup")
1676                         .arg("-dr")
1677                         .arg("RustDemangler")
1678                         .arg("-var")
1679                         .arg("var.RustDemanglerDir")
1680                         .arg("-out")
1681                         .arg(exe.join("RustDemanglerGroup.wxs"))
1682                         .arg("-t")
1683                         .arg(etc.join("msi/remove-duplicates.xsl")),
1684                 );
1685             }
1686             if built_tools.contains("miri") {
1687                 builder.run(
1688                     Command::new(&heat)
1689                         .current_dir(&exe)
1690                         .arg("dir")
1691                         .arg("miri")
1692                         .args(&heat_flags)
1693                         .arg("-cg")
1694                         .arg("MiriGroup")
1695                         .arg("-dr")
1696                         .arg("Miri")
1697                         .arg("-var")
1698                         .arg("var.MiriDir")
1699                         .arg("-out")
1700                         .arg(exe.join("MiriGroup.wxs"))
1701                         .arg("-t")
1702                         .arg(etc.join("msi/remove-duplicates.xsl")),
1703                 );
1704             }
1705             builder.run(
1706                 Command::new(&heat)
1707                     .current_dir(&exe)
1708                     .arg("dir")
1709                     .arg("rust-analysis")
1710                     .args(&heat_flags)
1711                     .arg("-cg")
1712                     .arg("AnalysisGroup")
1713                     .arg("-dr")
1714                     .arg("Analysis")
1715                     .arg("-var")
1716                     .arg("var.AnalysisDir")
1717                     .arg("-out")
1718                     .arg(exe.join("AnalysisGroup.wxs"))
1719                     .arg("-t")
1720                     .arg(etc.join("msi/remove-duplicates.xsl")),
1721             );
1722             if target.contains("windows-gnu") {
1723                 builder.run(
1724                     Command::new(&heat)
1725                         .current_dir(&exe)
1726                         .arg("dir")
1727                         .arg("rust-mingw")
1728                         .args(&heat_flags)
1729                         .arg("-cg")
1730                         .arg("GccGroup")
1731                         .arg("-dr")
1732                         .arg("Gcc")
1733                         .arg("-var")
1734                         .arg("var.GccDir")
1735                         .arg("-out")
1736                         .arg(exe.join("GccGroup.wxs")),
1737                 );
1738             }
1739
1740             let candle = |input: &Path| {
1741                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1742                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1743                 let mut cmd = Command::new(&candle);
1744                 cmd.current_dir(&exe)
1745                     .arg("-nologo")
1746                     .arg("-dRustcDir=rustc")
1747                     .arg("-dDocsDir=rust-docs")
1748                     .arg("-dCargoDir=cargo")
1749                     .arg("-dStdDir=rust-std")
1750                     .arg("-dAnalysisDir=rust-analysis")
1751                     .arg("-dClippyDir=clippy")
1752                     .arg("-arch")
1753                     .arg(&arch)
1754                     .arg("-out")
1755                     .arg(&output)
1756                     .arg(&input);
1757                 add_env(builder, &mut cmd, target);
1758
1759                 if built_tools.contains("rust-demangler") {
1760                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1761                 }
1762                 if built_tools.contains("rls") {
1763                     cmd.arg("-dRlsDir=rls");
1764                 }
1765                 if built_tools.contains("rust-analyzer") {
1766                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1767                 }
1768                 if built_tools.contains("miri") {
1769                     cmd.arg("-dMiriDir=miri");
1770                 }
1771                 if target.contains("windows-gnu") {
1772                     cmd.arg("-dGccDir=rust-mingw");
1773                 }
1774                 builder.run(&mut cmd);
1775             };
1776             candle(&xform(&etc.join("msi/rust.wxs")));
1777             candle(&etc.join("msi/ui.wxs"));
1778             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1779             candle("RustcGroup.wxs".as_ref());
1780             candle("DocsGroup.wxs".as_ref());
1781             candle("CargoGroup.wxs".as_ref());
1782             candle("StdGroup.wxs".as_ref());
1783             candle("ClippyGroup.wxs".as_ref());
1784             if built_tools.contains("rust-demangler") {
1785                 candle("RustDemanglerGroup.wxs".as_ref());
1786             }
1787             if built_tools.contains("rls") {
1788                 candle("RlsGroup.wxs".as_ref());
1789             }
1790             if built_tools.contains("rust-analyzer") {
1791                 candle("RustAnalyzerGroup.wxs".as_ref());
1792             }
1793             if built_tools.contains("miri") {
1794                 candle("MiriGroup.wxs".as_ref());
1795             }
1796             candle("AnalysisGroup.wxs".as_ref());
1797
1798             if target.contains("windows-gnu") {
1799                 candle("GccGroup.wxs".as_ref());
1800             }
1801
1802             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1803             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1804             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1805
1806             builder.info(&format!("building `msi` installer with {:?}", light));
1807             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1808             let mut cmd = Command::new(&light);
1809             cmd.arg("-nologo")
1810                 .arg("-ext")
1811                 .arg("WixUIExtension")
1812                 .arg("-ext")
1813                 .arg("WixUtilExtension")
1814                 .arg("-out")
1815                 .arg(exe.join(&filename))
1816                 .arg("rust.wixobj")
1817                 .arg("ui.wixobj")
1818                 .arg("rustwelcomedlg.wixobj")
1819                 .arg("RustcGroup.wixobj")
1820                 .arg("DocsGroup.wixobj")
1821                 .arg("CargoGroup.wixobj")
1822                 .arg("StdGroup.wixobj")
1823                 .arg("AnalysisGroup.wixobj")
1824                 .arg("ClippyGroup.wixobj")
1825                 .current_dir(&exe);
1826
1827             if built_tools.contains("rls") {
1828                 cmd.arg("RlsGroup.wixobj");
1829             }
1830             if built_tools.contains("rust-analyzer") {
1831                 cmd.arg("RustAnalyzerGroup.wixobj");
1832             }
1833             if built_tools.contains("rust-demangler") {
1834                 cmd.arg("RustDemanglerGroup.wixobj");
1835             }
1836             if built_tools.contains("miri") {
1837                 cmd.arg("MiriGroup.wixobj");
1838             }
1839
1840             if target.contains("windows-gnu") {
1841                 cmd.arg("GccGroup.wixobj");
1842             }
1843             // ICE57 wrongly complains about the shortcuts
1844             cmd.arg("-sice:ICE57");
1845
1846             let _time = timeit(builder);
1847             builder.run(&mut cmd);
1848
1849             if !builder.config.dry_run {
1850                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1851             }
1852         }
1853     }
1854 }
1855
1856 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1857     let mut parts = builder.version.split('.');
1858     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1859         .env("CFG_RELEASE_NUM", &builder.version)
1860         .env("CFG_RELEASE", builder.rust_release())
1861         .env("CFG_VER_MAJOR", parts.next().unwrap())
1862         .env("CFG_VER_MINOR", parts.next().unwrap())
1863         .env("CFG_VER_PATCH", parts.next().unwrap())
1864         .env("CFG_VER_BUILD", "0") // just needed to build
1865         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1866         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1867         .env("CFG_BUILD", target.triple)
1868         .env("CFG_CHANNEL", &builder.config.channel);
1869
1870     if target.contains("windows-gnu") {
1871         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1872     } else {
1873         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1874     }
1875 }
1876
1877 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1878 ///
1879
1880 /// Returns whether the files were actually copied.
1881 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1882     if let Some(config) = builder.config.target_config.get(&target) {
1883         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1884             // If the LLVM was externally provided, then we don't currently copy
1885             // artifacts into the sysroot. This is not necessarily the right
1886             // choice (in particular, it will require the LLVM dylib to be in
1887             // the linker's load path at runtime), but the common use case for
1888             // external LLVMs is distribution provided LLVMs, and in that case
1889             // they're usually in the standard search path (e.g., /usr/lib) and
1890             // copying them here is going to cause problems as we may end up
1891             // with the wrong files and isn't what distributions want.
1892             //
1893             // This behavior may be revisited in the future though.
1894             //
1895             // If the LLVM is coming from ourselves (just from CI) though, we
1896             // still want to install it, as it otherwise won't be available.
1897             return false;
1898         }
1899     }
1900
1901     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1902     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1903     // clear why this is the case, though. llvm-config will emit the versioned
1904     // paths and we don't want those in the sysroot (as we're expecting
1905     // unversioned paths).
1906     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1907         let src_libdir = builder.llvm_out(target).join("lib");
1908         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1909         if llvm_dylib_path.exists() {
1910             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1911         }
1912         !builder.config.dry_run
1913     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1914         let mut cmd = Command::new(llvm_config);
1915         cmd.arg("--libfiles");
1916         builder.verbose(&format!("running {:?}", cmd));
1917         let files = output(&mut cmd);
1918         let build_llvm_out = &builder.llvm_out(builder.config.build);
1919         let target_llvm_out = &builder.llvm_out(target);
1920         for file in files.trim_end().split(' ') {
1921             // If we're not using a custom LLVM, make sure we package for the target.
1922             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1923                 target_llvm_out.join(relative_path)
1924             } else {
1925                 PathBuf::from(file)
1926             };
1927             builder.install(&file, dst_libdir, 0o644);
1928         }
1929         !builder.config.dry_run
1930     } else {
1931         false
1932     }
1933 }
1934
1935 /// Maybe add libLLVM.so to the target lib-dir for linking.
1936 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1937     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1938     // We do not need to copy LLVM files into the sysroot if it is not
1939     // dynamically linked; it is already included into librustc_llvm
1940     // statically.
1941     if builder.config.llvm_link_shared {
1942         maybe_install_llvm(builder, target, &dst_libdir);
1943     }
1944 }
1945
1946 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1947 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1948     let dst_libdir =
1949         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1950     // We do not need to copy LLVM files into the sysroot if it is not
1951     // dynamically linked; it is already included into librustc_llvm
1952     // statically.
1953     if builder.config.llvm_link_shared {
1954         maybe_install_llvm(builder, target, &dst_libdir);
1955     }
1956 }
1957
1958 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1959 pub struct LlvmTools {
1960     pub target: TargetSelection,
1961 }
1962
1963 impl Step for LlvmTools {
1964     type Output = Option<GeneratedTarball>;
1965     const ONLY_HOSTS: bool = true;
1966     const DEFAULT: bool = true;
1967
1968     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1969         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1970         run.path("llvm-tools").default_condition(default)
1971     }
1972
1973     fn make_run(run: RunConfig<'_>) {
1974         run.builder.ensure(LlvmTools { target: run.target });
1975     }
1976
1977     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1978         let target = self.target;
1979
1980         /* run only if llvm-config isn't used */
1981         if let Some(config) = builder.config.target_config.get(&target) {
1982             if let Some(ref _s) = config.llvm_config {
1983                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1984                 return None;
1985             }
1986         }
1987
1988         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1989         tarball.set_overlay(OverlayKind::LLVM);
1990         tarball.is_preview(true);
1991
1992         // Prepare the image directory
1993         let src_bindir = builder.llvm_out(target).join("bin");
1994         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1995         for tool in LLVM_TOOLS {
1996             let exe = src_bindir.join(exe(tool, target));
1997             tarball.add_file(&exe, &dst_bindir, 0o755);
1998         }
1999
2000         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2001         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2002         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2003         // compiler libraries.
2004         maybe_install_llvm_target(builder, target, tarball.image_dir());
2005
2006         Some(tarball.generate())
2007     }
2008 }
2009
2010 // Tarball intended for internal consumption to ease rustc/std development.
2011 //
2012 // Should not be considered stable by end users.
2013 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2014 pub struct RustDev {
2015     pub target: TargetSelection,
2016 }
2017
2018 impl Step for RustDev {
2019     type Output = Option<GeneratedTarball>;
2020     const DEFAULT: bool = true;
2021     const ONLY_HOSTS: bool = true;
2022
2023     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2024         run.path("rust-dev")
2025     }
2026
2027     fn make_run(run: RunConfig<'_>) {
2028         run.builder.ensure(RustDev { target: run.target });
2029     }
2030
2031     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2032         let target = self.target;
2033
2034         /* run only if llvm-config isn't used */
2035         if let Some(config) = builder.config.target_config.get(&target) {
2036             if let Some(ref _s) = config.llvm_config {
2037                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2038                 return None;
2039             }
2040         }
2041
2042         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2043         tarball.set_overlay(OverlayKind::LLVM);
2044
2045         let src_bindir = builder.llvm_out(target).join("bin");
2046         for bin in &[
2047             "llvm-config",
2048             "llvm-ar",
2049             "llvm-objdump",
2050             "llvm-profdata",
2051             "llvm-bcanalyzer",
2052             "llvm-cov",
2053             "llvm-dwp",
2054             "llvm-nm",
2055         ] {
2056             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2057         }
2058
2059         // We don't build LLD on some platforms, so only add it if it exists
2060         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2061         if lld_path.exists() {
2062             tarball.add_file(lld_path, "bin", 0o755);
2063         }
2064
2065         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2066
2067         // Copy the include directory as well; needed mostly to build
2068         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2069         // just broadly useful to be able to link against the bundled LLVM.
2070         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2071
2072         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2073         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2074         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2075         // compiler libraries.
2076         let dst_libdir = tarball.image_dir().join("lib");
2077         maybe_install_llvm(builder, target, &dst_libdir);
2078         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2079         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2080
2081         Some(tarball.generate())
2082     }
2083 }
2084
2085 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2086 /// release process to avoid cloning the monorepo and building stuff.
2087 ///
2088 /// Should not be considered stable by end users.
2089 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2090 pub struct BuildManifest {
2091     pub target: TargetSelection,
2092 }
2093
2094 impl Step for BuildManifest {
2095     type Output = GeneratedTarball;
2096     const DEFAULT: bool = false;
2097     const ONLY_HOSTS: bool = true;
2098
2099     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2100         run.path("build-manifest")
2101     }
2102
2103     fn make_run(run: RunConfig<'_>) {
2104         run.builder.ensure(BuildManifest { target: run.target });
2105     }
2106
2107     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2108         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2109
2110         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2111         tarball.add_file(&build_manifest, "bin", 0o755);
2112         tarball.generate()
2113     }
2114 }
2115
2116 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2117 ///
2118 /// Currently this is the PGO profile data.
2119 ///
2120 /// Should not be considered stable by end users.
2121 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2122 pub struct ReproducibleArtifacts {
2123     pub target: TargetSelection,
2124 }
2125
2126 impl Step for ReproducibleArtifacts {
2127     type Output = Option<GeneratedTarball>;
2128     const DEFAULT: bool = true;
2129     const ONLY_HOSTS: bool = true;
2130
2131     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2132         run.path("reproducible-artifacts")
2133     }
2134
2135     fn make_run(run: RunConfig<'_>) {
2136         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2137     }
2138
2139     fn run(self, builder: &Builder<'_>) -> Self::Output {
2140         let mut added_anything = false;
2141         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2142         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2143             tarball.add_file(path, ".", 0o644);
2144             added_anything = true;
2145         }
2146         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2147             tarball.add_file(path, ".", 0o644);
2148             added_anything = true;
2149         }
2150         if added_anything { Some(tarball.generate()) } else { None }
2151     }
2152 }