src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn missing_tool(tool_name: &str, skip: bool) {
  39     if skip {
  40         println!("Unable to build {}, skipping dist", tool_name)
  41     } else {
  42         panic!("Unable to build {}", tool_name)
  43     }
  44 }
  45
  46 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  47     if !builder.config.extended {
  48         return false;
  49     }
  50     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  51 }
  52
  53 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  54 pub struct Docs {
  55     pub host: TargetSelection,
  56 }
  57
  58 impl Step for Docs {
  59     type Output = Option<GeneratedTarball>;
  60     const DEFAULT: bool = true;
  61
  62     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  63         let default = run.builder.config.docs;
  64         run.path("rust-docs").default_condition(default)
  65     }
  66
  67     fn make_run(run: RunConfig<'_>) {
  68         run.builder.ensure(Docs { host: run.target });
  69     }
  70
  71     /// Builds the `rust-docs` installer component.
  72     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  73         let host = self.host;
  74         builder.default_doc(&[]);
  75
  76         let dest = "share/doc/rust/html";
  77
  78         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  79         tarball.set_product_name("Rust Documentation");
  80         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  81         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  82         Some(tarball.generate())
  83     }
  84 }
  85
  86 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  87 pub struct RustcDocs {
  88     pub host: TargetSelection,
  89 }
  90
  91 impl Step for RustcDocs {
  92     type Output = Option<GeneratedTarball>;
  93     const DEFAULT: bool = true;
  94
  95     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  96         let builder = run.builder;
  97         run.path("rustc-docs").default_condition(builder.config.compiler_docs)
  98     }
  99
 100     fn make_run(run: RunConfig<'_>) {
 101         run.builder.ensure(RustcDocs { host: run.target });
 102     }
 103
 104     /// Builds the `rustc-docs` installer component.
 105     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 106         let host = self.host;
 107         builder.default_doc(&[]);
 108
 109         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 110         tarball.set_product_name("Rustc Documentation");
 111         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 112         Some(tarball.generate())
 113     }
 114 }
 115
 116 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 117     let mut found = Vec::with_capacity(files.len());
 118
 119     for file in files {
 120         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 121
 122         if let Some(file_path) = file_path {
 123             found.push(file_path);
 124         } else {
 125             panic!("Could not find '{}' in {:?}", file, path);
 126         }
 127     }
 128
 129     found
 130 }
 131
 132 fn make_win_dist(
 133     rust_root: &Path,
 134     plat_root: &Path,
 135     target: TargetSelection,
 136     builder: &Builder<'_>,
 137 ) {
 138     //Ask gcc where it keeps its stuff
 139     let mut cmd = Command::new(builder.cc(target));
 140     cmd.arg("-print-search-dirs");
 141     let gcc_out = output(&mut cmd);
 142
 143     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 144     let mut lib_path = Vec::new();
 145
 146     for line in gcc_out.lines() {
 147         let idx = line.find(':').unwrap();
 148         let key = &line[..idx];
 149         let trim_chars: &[_] = &[' ', '='];
 150         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 151
 152         if key == "programs" {
 153             bin_path.extend(value);
 154         } else if key == "libraries" {
 155             lib_path.extend(value);
 156         }
 157     }
 158
 159     let compiler = if target == "i686-pc-windows-gnu" {
 160         "i686-w64-mingw32-gcc.exe"
 161     } else if target == "x86_64-pc-windows-gnu" {
 162         "x86_64-w64-mingw32-gcc.exe"
 163     } else {
 164         "gcc.exe"
 165     };
 166     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 167     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 168     if target.starts_with("i686-") {
 169         rustc_dlls.push("libgcc_s_dw2-1.dll");
 170     } else {
 171         rustc_dlls.push("libgcc_s_seh-1.dll");
 172     }
 173
 174     let target_libs = [
 175         //MinGW libs
 176         "libgcc.a",
 177         "libgcc_eh.a",
 178         "libgcc_s.a",
 179         "libm.a",
 180         "libmingw32.a",
 181         "libmingwex.a",
 182         "libstdc++.a",
 183         "libiconv.a",
 184         "libmoldname.a",
 185         "libpthread.a",
 186         //Windows import libs
 187         "libadvapi32.a",
 188         "libbcrypt.a",
 189         "libcomctl32.a",
 190         "libcomdlg32.a",
 191         "libcredui.a",
 192         "libcrypt32.a",
 193         "libdbghelp.a",
 194         "libgdi32.a",
 195         "libimagehlp.a",
 196         "libiphlpapi.a",
 197         "libkernel32.a",
 198         "libmsimg32.a",
 199         "libmsvcrt.a",
 200         "libodbc32.a",
 201         "libole32.a",
 202         "liboleaut32.a",
 203         "libopengl32.a",
 204         "libpsapi.a",
 205         "librpcrt4.a",
 206         "libsecur32.a",
 207         "libsetupapi.a",
 208         "libshell32.a",
 209         "libsynchronization.a",
 210         "libuser32.a",
 211         "libuserenv.a",
 212         "libuuid.a",
 213         "libwinhttp.a",
 214         "libwinmm.a",
 215         "libwinspool.a",
 216         "libws2_32.a",
 217         "libwsock32.a",
 218     ];
 219
 220     //Find mingw artifacts we want to bundle
 221     let target_tools = find_files(&target_tools, &bin_path);
 222     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 223     let target_libs = find_files(&target_libs, &lib_path);
 224
 225     // Copy runtime dlls next to rustc.exe
 226     let dist_bin_dir = rust_root.join("bin/");
 227     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 228     for src in rustc_dlls {
 229         builder.copy_to_folder(&src, &dist_bin_dir);
 230     }
 231
 232     //Copy platform tools to platform-specific bin directory
 233     let target_bin_dir = plat_root
 234         .join("lib")
 235         .join("rustlib")
 236         .join(target.triple)
 237         .join("bin")
 238         .join("self-contained");
 239     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 240     for src in target_tools {
 241         builder.copy_to_folder(&src, &target_bin_dir);
 242     }
 243
 244     // Warn windows-gnu users that the bundled GCC cannot compile C files
 245     builder.create(
 246         &target_bin_dir.join("GCC-WARNING.txt"),
 247         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 248          used as a linker. In order to be able to compile projects containing C code use \
 249          the GCC provided by MinGW or Cygwin.",
 250     );
 251
 252     //Copy platform libs to platform-specific lib directory
 253     let target_lib_dir = plat_root
 254         .join("lib")
 255         .join("rustlib")
 256         .join(target.triple)
 257         .join("lib")
 258         .join("self-contained");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         builder.copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     pub host: TargetSelection,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<GeneratedTarball>;
 272     const DEFAULT: bool = true;
 273
 274     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 275         run.path("rust-mingw")
 276     }
 277
 278     fn make_run(run: RunConfig<'_>) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Builds the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 287         let host = self.host;
 288         if !host.contains("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 293         tarball.set_product_name("Rust MinGW");
 294
 295         // The first argument is a "temporary directory" which is just
 296         // thrown away (this contains the runtime DLLs included in the rustc package
 297         // above) and the second argument is where to place all the MinGW components
 298         // (which is what we want).
 299         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 300
 301         Some(tarball.generate())
 302     }
 303 }
 304
 305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 306 pub struct Rustc {
 307     pub compiler: Compiler,
 308 }
 309
 310 impl Step for Rustc {
 311     type Output = GeneratedTarball;
 312     const DEFAULT: bool = true;
 313     const ONLY_HOSTS: bool = true;
 314
 315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 316         run.path("rustc")
 317     }
 318
 319     fn make_run(run: RunConfig<'_>) {
 320         run.builder
 321             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 322     }
 323
 324     /// Creates the `rustc` installer component.
 325     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 326         let compiler = self.compiler;
 327         let host = self.compiler.host;
 328
 329         let tarball = Tarball::new(builder, "rustc", &host.triple);
 330
 331         // Prepare the rustc "image", what will actually end up getting installed
 332         prepare_image(builder, compiler, tarball.image_dir());
 333
 334         // On MinGW we've got a few runtime DLL dependencies that we need to
 335         // include. The first argument to this script is where to put these DLLs
 336         // (the image we're creating), and the second argument is a junk directory
 337         // to ignore all other MinGW stuff the script creates.
 338         //
 339         // On 32-bit MinGW we're always including a DLL which needs some extra
 340         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 341         // anything requiring us to distribute a license, but it's likely the
 342         // install will *also* include the rust-mingw package, which also needs
 343         // licenses, so to be safe we just include it here in all MinGW packages.
 344         if host.contains("pc-windows-gnu") {
 345             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 346             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 347         }
 348
 349         return tarball.generate();
 350
 351         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 352             let host = compiler.host;
 353             let src = builder.sysroot(compiler);
 354
 355             // Copy rustc/rustdoc binaries
 356             t!(fs::create_dir_all(image.join("bin")));
 357             builder.cp_r(&src.join("bin"), &image.join("bin"));
 358
 359             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 360
 361             let libdir_relative = builder.libdir_relative(compiler);
 362
 363             // Copy runtime DLLs needed by the compiler
 364             if libdir_relative.to_str() != Some("bin") {
 365                 let libdir = builder.rustc_libdir(compiler);
 366                 for entry in builder.read_dir(&libdir) {
 367                     let name = entry.file_name();
 368                     if let Some(s) = name.to_str() {
 369                         if is_dylib(s) {
 370                             // Don't use custom libdir here because ^lib/ will be resolved again
 371                             // with installer
 372                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 373                         }
 374                     }
 375                 }
 376             }
 377
 378             // Copy over the codegen backends
 379             let backends_src = builder.sysroot_codegen_backends(compiler);
 380             let backends_rel = backends_src
 381                 .strip_prefix(&src)
 382                 .unwrap()
 383                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 384                 .unwrap();
 385             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 386             let backends_dst = image.join("lib").join(&backends_rel);
 387
 388             t!(fs::create_dir_all(&backends_dst));
 389             builder.cp_r(&backends_src, &backends_dst);
 390
 391             // Copy libLLVM.so to the lib dir as well, if needed. While not
 392             // technically needed by rustc itself it's needed by lots of other
 393             // components like the llvm tools and LLD. LLD is included below and
 394             // tools/LLDB come later, so let's just throw it in the rustc
 395             // component for now.
 396             maybe_install_llvm_runtime(builder, host, image);
 397
 398             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 399             t!(fs::create_dir_all(&dst_dir));
 400
 401             // Copy over lld if it's there
 402             if builder.config.lld_enabled {
 403                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404                 let rust_lld = exe("rust-lld", compiler.host);
 405                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 406                 // for `-Z gcc-ld=lld`
 407                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 408                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 409                 t!(fs::create_dir(&gcc_lld_dst_dir));
 410                 for flavor in ["ld", "ld64"] {
 411                     let exe_name = exe(flavor, compiler.host);
 412                     builder
 413                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 414                 }
 415             }
 416
 417             // Man pages
 418             t!(fs::create_dir_all(image.join("share/man/man1")));
 419             let man_src = builder.src.join("src/doc/man");
 420             let man_dst = image.join("share/man/man1");
 421
 422             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 423             // to hardlink, and we don't want to edit the source templates
 424             for file_entry in builder.read_dir(&man_src) {
 425                 let page_src = file_entry.path();
 426                 let page_dst = man_dst.join(file_entry.file_name());
 427                 let src_text = t!(std::fs::read_to_string(&page_src));
 428                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 429                 t!(std::fs::write(&page_dst, &new_text));
 430                 t!(fs::copy(&page_src, &page_dst));
 431             }
 432
 433             // Debugger scripts
 434             builder
 435                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 436
 437             // Misc license info
 438             let cp = |file: &str| {
 439                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 440             };
 441             cp("COPYRIGHT");
 442             cp("LICENSE-APACHE");
 443             cp("LICENSE-MIT");
 444             cp("README.md");
 445         }
 446     }
 447 }
 448
 449 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 450 pub struct DebuggerScripts {
 451     pub sysroot: Interned<PathBuf>,
 452     pub host: TargetSelection,
 453 }
 454
 455 impl Step for DebuggerScripts {
 456     type Output = ();
 457
 458     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 459         run.path("src/lldb_batchmode.py")
 460     }
 461
 462     fn make_run(run: RunConfig<'_>) {
 463         run.builder.ensure(DebuggerScripts {
 464             sysroot: run
 465                 .builder
 466                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 467             host: run.target,
 468         });
 469     }
 470
 471     /// Copies debugger scripts for `target` into the `sysroot` specified.
 472     fn run(self, builder: &Builder<'_>) {
 473         let host = self.host;
 474         let sysroot = self.sysroot;
 475         let dst = sysroot.join("lib/rustlib/etc");
 476         t!(fs::create_dir_all(&dst));
 477         let cp_debugger_script = |file: &str| {
 478             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 479         };
 480         if host.contains("windows-msvc") {
 481             // windbg debugger scripts
 482             builder.install(
 483                 &builder.src.join("src/etc/rust-windbg.cmd"),
 484                 &sysroot.join("bin"),
 485                 0o755,
 486             );
 487
 488             cp_debugger_script("natvis/intrinsic.natvis");
 489             cp_debugger_script("natvis/liballoc.natvis");
 490             cp_debugger_script("natvis/libcore.natvis");
 491             cp_debugger_script("natvis/libstd.natvis");
 492         } else {
 493             cp_debugger_script("rust_types.py");
 494
 495             // gdb debugger scripts
 496             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 497             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 498
 499             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 500             cp_debugger_script("gdb_lookup.py");
 501             cp_debugger_script("gdb_providers.py");
 502
 503             // lldb debugger scripts
 504             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 505
 506             cp_debugger_script("lldb_lookup.py");
 507             cp_debugger_script("lldb_providers.py");
 508             cp_debugger_script("lldb_commands")
 509         }
 510     }
 511 }
 512
 513 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 514     // The only true set of target libraries came from the build triple, so
 515     // let's reduce redundant work by only producing archives from that host.
 516     if compiler.host != builder.config.build {
 517         builder.info("\tskipping, not a build host");
 518         true
 519     } else {
 520         false
 521     }
 522 }
 523
 524 /// Copy stamped files into an image's `target/lib` directory.
 525 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 526     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 527     let self_contained_dst = dst.join("self-contained");
 528     t!(fs::create_dir_all(&dst));
 529     t!(fs::create_dir_all(&self_contained_dst));
 530     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 531         if dependency_type == DependencyType::TargetSelfContained {
 532             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 533         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 534             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: TargetSelection,
 543 }
 544
 545 impl Step for Std {
 546     type Output = Option<GeneratedTarball>;
 547     const DEFAULT: bool = true;
 548
 549     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 550         run.path("rust-std")
 551     }
 552
 553     fn make_run(run: RunConfig<'_>) {
 554         run.builder.ensure(Std {
 555             compiler: run.builder.compiler_for(
 556                 run.builder.top_stage,
 557                 run.builder.config.build,
 558                 run.target,
 559             ),
 560             target: run.target,
 561         });
 562     }
 563
 564     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 565         let compiler = self.compiler;
 566         let target = self.target;
 567
 568         if skip_host_target_lib(builder, compiler) {
 569             return None;
 570         }
 571
 572         builder.ensure(compile::Std { compiler, target });
 573
 574         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 575         tarball.include_target_in_component_name(true);
 576
 577         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 578         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 579         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 580
 581         Some(tarball.generate())
 582     }
 583 }
 584
 585 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 586 pub struct RustcDev {
 587     pub compiler: Compiler,
 588     pub target: TargetSelection,
 589 }
 590
 591 impl Step for RustcDev {
 592     type Output = Option<GeneratedTarball>;
 593     const DEFAULT: bool = true;
 594     const ONLY_HOSTS: bool = true;
 595
 596     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 597         run.path("rustc-dev")
 598     }
 599
 600     fn make_run(run: RunConfig<'_>) {
 601         run.builder.ensure(RustcDev {
 602             compiler: run.builder.compiler_for(
 603                 run.builder.top_stage,
 604                 run.builder.config.build,
 605                 run.target,
 606             ),
 607             target: run.target,
 608         });
 609     }
 610
 611     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 612         let compiler = self.compiler;
 613         let target = self.target;
 614         if skip_host_target_lib(builder, compiler) {
 615             return None;
 616         }
 617
 618         builder.ensure(compile::Rustc { compiler, target });
 619
 620         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 621
 622         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 623         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 624         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 625
 626         let src_files = &["Cargo.lock"];
 627         // This is the reduced set of paths which will become the rustc-dev component
 628         // (essentially the compiler crates and all of their path dependencies).
 629         copy_src_dirs(
 630             builder,
 631             &builder.src,
 632             &["compiler"],
 633             &[],
 634             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 635         );
 636         for file in src_files {
 637             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 638         }
 639
 640         Some(tarball.generate())
 641     }
 642 }
 643
 644 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 645 pub struct Analysis {
 646     pub compiler: Compiler,
 647     pub target: TargetSelection,
 648 }
 649
 650 impl Step for Analysis {
 651     type Output = Option<GeneratedTarball>;
 652     const DEFAULT: bool = true;
 653
 654     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 655         let default = should_build_extended_tool(&run.builder, "analysis");
 656         run.path("rust-analysis").default_condition(default)
 657     }
 658
 659     fn make_run(run: RunConfig<'_>) {
 660         run.builder.ensure(Analysis {
 661             // Find the actual compiler (handling the full bootstrap option) which
 662             // produced the save-analysis data because that data isn't copied
 663             // through the sysroot uplifting.
 664             compiler: run.builder.compiler_for(
 665                 run.builder.top_stage,
 666                 run.builder.config.build,
 667                 run.target,
 668             ),
 669             target: run.target,
 670         });
 671     }
 672
 673     /// Creates a tarball of save-analysis metadata, if available.
 674     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 675         let compiler = self.compiler;
 676         let target = self.target;
 677         if compiler.host != builder.config.build {
 678             return None;
 679         }
 680
 681         builder.ensure(compile::Std { compiler, target });
 682         let src = builder
 683             .stage_out(compiler, Mode::Std)
 684             .join(target.triple)
 685             .join(builder.cargo_dir())
 686             .join("deps")
 687             .join("save-analysis");
 688
 689         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 690         tarball.include_target_in_component_name(true);
 691         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 692         Some(tarball.generate())
 693     }
 694 }
 695
 696 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 697 /// `dst_dir`.
 698 fn copy_src_dirs(
 699     builder: &Builder<'_>,
 700     base: &Path,
 701     src_dirs: &[&str],
 702     exclude_dirs: &[&str],
 703     dst_dir: &Path,
 704 ) {
 705     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 706         let spath = match path.to_str() {
 707             Some(path) => path,
 708             None => return false,
 709         };
 710         if spath.ends_with('~') || spath.ends_with(".pyc") {
 711             return false;
 712         }
 713
 714         const LLVM_PROJECTS: &[&str] = &[
 715             "llvm-project/clang",
 716             "llvm-project\\clang",
 717             "llvm-project/libunwind",
 718             "llvm-project\\libunwind",
 719             "llvm-project/lld",
 720             "llvm-project\\lld",
 721             "llvm-project/lldb",
 722             "llvm-project\\lldb",
 723             "llvm-project/llvm",
 724             "llvm-project\\llvm",
 725             "llvm-project/compiler-rt",
 726             "llvm-project\\compiler-rt",
 727             "llvm-project/cmake",
 728             "llvm-project\\cmake",
 729         ];
 730         if spath.contains("llvm-project")
 731             && !spath.ends_with("llvm-project")
 732             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 733         {
 734             return false;
 735         }
 736
 737         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 738         if LLVM_TEST.iter().any(|path| spath.contains(path))
 739             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 740         {
 741             return false;
 742         }
 743
 744         let full_path = Path::new(dir).join(path);
 745         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 746             return false;
 747         }
 748
 749         let excludes = [
 750             "CVS",
 751             "RCS",
 752             "SCCS",
 753             ".git",
 754             ".gitignore",
 755             ".gitmodules",
 756             ".gitattributes",
 757             ".cvsignore",
 758             ".svn",
 759             ".arch-ids",
 760             "{arch}",
 761             "=RELEASE-ID",
 762             "=meta-update",
 763             "=update",
 764             ".bzr",
 765             ".bzrignore",
 766             ".bzrtags",
 767             ".hg",
 768             ".hgignore",
 769             ".hgrags",
 770             "_darcs",
 771         ];
 772         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 773     }
 774
 775     // Copy the directories using our filter
 776     for item in src_dirs {
 777         let dst = &dst_dir.join(item);
 778         t!(fs::create_dir_all(dst));
 779         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 780     }
 781 }
 782
 783 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 784 pub struct Src;
 785
 786 impl Step for Src {
 787     /// The output path of the src installer tarball
 788     type Output = GeneratedTarball;
 789     const DEFAULT: bool = true;
 790     const ONLY_HOSTS: bool = true;
 791
 792     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 793         run.path("rust-src")
 794     }
 795
 796     fn make_run(run: RunConfig<'_>) {
 797         run.builder.ensure(Src);
 798     }
 799
 800     /// Creates the `rust-src` installer component
 801     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 802         let tarball = Tarball::new_targetless(builder, "rust-src");
 803
 804         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 805         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 806         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 807         // and fix them...
 808         //
 809         // NOTE: if you update the paths here, you also should update the "virtual" path
 810         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 811         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 812
 813         let src_files = ["Cargo.lock"];
 814         // This is the reduced set of paths which will become the rust-src component
 815         // (essentially libstd and all of its path dependencies).
 816         copy_src_dirs(
 817             builder,
 818             &builder.src,
 819             &["library", "src/llvm-project/libunwind"],
 820             &[
 821                 // not needed and contains symlinks which rustup currently
 822                 // chokes on when unpacking.
 823                 "library/backtrace/crates",
 824                 // these are 30MB combined and aren't necessary for building
 825                 // the standard library.
 826                 "library/stdarch/crates/Cargo.toml",
 827                 "library/stdarch/crates/stdarch-verify",
 828                 "library/stdarch/crates/intrinsic-test",
 829             ],
 830             &dst_src,
 831         );
 832         for file in src_files.iter() {
 833             builder.copy(&builder.src.join(file), &dst_src.join(file));
 834         }
 835
 836         tarball.generate()
 837     }
 838 }
 839
 840 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 841 pub struct PlainSourceTarball;
 842
 843 impl Step for PlainSourceTarball {
 844     /// Produces the location of the tarball generated
 845     type Output = GeneratedTarball;
 846     const DEFAULT: bool = true;
 847     const ONLY_HOSTS: bool = true;
 848
 849     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 850         let builder = run.builder;
 851         run.path("rustc-src").default_condition(builder.config.rust_dist_src)
 852     }
 853
 854     fn make_run(run: RunConfig<'_>) {
 855         run.builder.ensure(PlainSourceTarball);
 856     }
 857
 858     /// Creates the plain source tarball
 859     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 860         let tarball = Tarball::new(builder, "rustc", "src");
 861         let plain_dst_src = tarball.image_dir();
 862
 863         // This is the set of root paths which will become part of the source package
 864         let src_files = [
 865             "COPYRIGHT",
 866             "LICENSE-APACHE",
 867             "LICENSE-MIT",
 868             "CONTRIBUTING.md",
 869             "README.md",
 870             "RELEASES.md",
 871             "configure",
 872             "x.py",
 873             "config.toml.example",
 874             "Cargo.toml",
 875             "Cargo.lock",
 876         ];
 877         let src_dirs = ["src", "compiler", "library"];
 878
 879         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 880
 881         // Copy the files normally
 882         for item in &src_files {
 883             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 884         }
 885
 886         // Create the version file
 887         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 888         if let Some(sha) = builder.rust_sha() {
 889             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 890         }
 891
 892         // If we're building from git sources, we need to vendor a complete distribution.
 893         if builder.rust_info.is_git() {
 894             // Vendor all Cargo dependencies
 895             let mut cmd = Command::new(&builder.initial_cargo);
 896             cmd.arg("vendor")
 897                 .arg("--sync")
 898                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 899                 .arg("--sync")
 900                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 901                 .current_dir(&plain_dst_src);
 902             builder.run(&mut cmd);
 903         }
 904
 905         tarball.bare()
 906     }
 907 }
 908
 909 // We have to run a few shell scripts, which choke quite a bit on both `\`
 910 // characters and on `C:\` paths, so normalize both of them away.
 911 pub fn sanitize_sh(path: &Path) -> String {
 912     let path = path.to_str().unwrap().replace("\\", "/");
 913     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 914
 915     fn unc_to_lfs(s: &str) -> &str {
 916         s.strip_prefix("//?/").unwrap_or(s)
 917     }
 918
 919     fn change_drive(s: &str) -> Option<String> {
 920         let mut ch = s.chars();
 921         let drive = ch.next().unwrap_or('C');
 922         if ch.next() != Some(':') {
 923             return None;
 924         }
 925         if ch.next() != Some('/') {
 926             return None;
 927         }
 928         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 929     }
 930 }
 931
 932 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 933 pub struct Cargo {
 934     pub compiler: Compiler,
 935     pub target: TargetSelection,
 936 }
 937
 938 impl Step for Cargo {
 939     type Output = Option<GeneratedTarball>;
 940     const DEFAULT: bool = true;
 941     const ONLY_HOSTS: bool = true;
 942
 943     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 944         let default = should_build_extended_tool(&run.builder, "cargo");
 945         run.path("cargo").default_condition(default)
 946     }
 947
 948     fn make_run(run: RunConfig<'_>) {
 949         run.builder.ensure(Cargo {
 950             compiler: run.builder.compiler_for(
 951                 run.builder.top_stage,
 952                 run.builder.config.build,
 953                 run.target,
 954             ),
 955             target: run.target,
 956         });
 957     }
 958
 959     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 960         let compiler = self.compiler;
 961         let target = self.target;
 962
 963         let cargo = builder.ensure(tool::Cargo { compiler, target });
 964         let src = builder.src.join("src/tools/cargo");
 965         let etc = src.join("src/etc");
 966
 967         // Prepare the image directory
 968         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 969         tarball.set_overlay(OverlayKind::Cargo);
 970
 971         tarball.add_file(&cargo, "bin", 0o755);
 972         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 973         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 974         tarball.add_dir(etc.join("man"), "share/man/man1");
 975         tarball.add_legal_and_readme_to("share/doc/cargo");
 976
 977         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
 978             let dirent = dirent.expect("read dir entry");
 979             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
 980                 tarball.add_file(&dirent.path(), "libexec", 0o755);
 981             }
 982         }
 983
 984         Some(tarball.generate())
 985     }
 986 }
 987
 988 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 989 pub struct Rls {
 990     pub compiler: Compiler,
 991     pub target: TargetSelection,
 992 }
 993
 994 impl Step for Rls {
 995     type Output = Option<GeneratedTarball>;
 996     const ONLY_HOSTS: bool = true;
 997     const DEFAULT: bool = true;
 998
 999     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1000         let default = should_build_extended_tool(&run.builder, "rls");
1001         run.path("rls").default_condition(default)
1002     }
1003
1004     fn make_run(run: RunConfig<'_>) {
1005         run.builder.ensure(Rls {
1006             compiler: run.builder.compiler_for(
1007                 run.builder.top_stage,
1008                 run.builder.config.build,
1009                 run.target,
1010             ),
1011             target: run.target,
1012         });
1013     }
1014
1015     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1016         let compiler = self.compiler;
1017         let target = self.target;
1018
1019         let rls = builder
1020             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1021             .or_else(|| {
1022                 missing_tool("RLS", builder.build.config.missing_tools);
1023                 None
1024             })?;
1025
1026         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1027         tarball.set_overlay(OverlayKind::RLS);
1028         tarball.is_preview(true);
1029         tarball.add_file(rls, "bin", 0o755);
1030         tarball.add_legal_and_readme_to("share/doc/rls");
1031         Some(tarball.generate())
1032     }
1033 }
1034
1035 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1036 pub struct RustAnalyzer {
1037     pub compiler: Compiler,
1038     pub target: TargetSelection,
1039 }
1040
1041 impl Step for RustAnalyzer {
1042     type Output = Option<GeneratedTarball>;
1043     const DEFAULT: bool = true;
1044     const ONLY_HOSTS: bool = true;
1045
1046     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1047         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1048         run.path("rust-analyzer").default_condition(default)
1049     }
1050
1051     fn make_run(run: RunConfig<'_>) {
1052         run.builder.ensure(RustAnalyzer {
1053             compiler: run.builder.compiler_for(
1054                 run.builder.top_stage,
1055                 run.builder.config.build,
1056                 run.target,
1057             ),
1058             target: run.target,
1059         });
1060     }
1061
1062     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1063         // This prevents rust-analyzer from being built for "dist" or "install"
1064         // on the stable/beta channels. It is a nightly-only tool and should
1065         // not be included.
1066         if !builder.build.unstable_features() {
1067             return None;
1068         }
1069         let compiler = self.compiler;
1070         let target = self.target;
1071
1072         if target.contains("riscv64") {
1073             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1074             // to build. See #74813 for details.
1075             return None;
1076         }
1077
1078         let rust_analyzer = builder
1079             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1080             .expect("rust-analyzer always builds");
1081
1082         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1083         tarball.set_overlay(OverlayKind::RustAnalyzer);
1084         tarball.is_preview(true);
1085         tarball.add_file(rust_analyzer, "bin", 0o755);
1086         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1087         Some(tarball.generate())
1088     }
1089 }
1090
1091 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1092 pub struct Clippy {
1093     pub compiler: Compiler,
1094     pub target: TargetSelection,
1095 }
1096
1097 impl Step for Clippy {
1098     type Output = Option<GeneratedTarball>;
1099     const DEFAULT: bool = true;
1100     const ONLY_HOSTS: bool = true;
1101
1102     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1103         let default = should_build_extended_tool(&run.builder, "clippy");
1104         run.path("clippy").default_condition(default)
1105     }
1106
1107     fn make_run(run: RunConfig<'_>) {
1108         run.builder.ensure(Clippy {
1109             compiler: run.builder.compiler_for(
1110                 run.builder.top_stage,
1111                 run.builder.config.build,
1112                 run.target,
1113             ),
1114             target: run.target,
1115         });
1116     }
1117
1118     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1119         let compiler = self.compiler;
1120         let target = self.target;
1121
1122         // Prepare the image directory
1123         // We expect clippy to build, because we've exited this step above if tool
1124         // state for clippy isn't testing.
1125         let clippy = builder
1126             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1127             .expect("clippy expected to build - essential tool");
1128         let cargoclippy = builder
1129             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1130             .expect("clippy expected to build - essential tool");
1131
1132         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1133         tarball.set_overlay(OverlayKind::Clippy);
1134         tarball.is_preview(true);
1135         tarball.add_file(clippy, "bin", 0o755);
1136         tarball.add_file(cargoclippy, "bin", 0o755);
1137         tarball.add_legal_and_readme_to("share/doc/clippy");
1138         Some(tarball.generate())
1139     }
1140 }
1141
1142 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1143 pub struct Miri {
1144     pub compiler: Compiler,
1145     pub target: TargetSelection,
1146 }
1147
1148 impl Step for Miri {
1149     type Output = Option<GeneratedTarball>;
1150     const DEFAULT: bool = true;
1151     const ONLY_HOSTS: bool = true;
1152
1153     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1154         let default = should_build_extended_tool(&run.builder, "miri");
1155         run.path("miri").default_condition(default)
1156     }
1157
1158     fn make_run(run: RunConfig<'_>) {
1159         run.builder.ensure(Miri {
1160             compiler: run.builder.compiler_for(
1161                 run.builder.top_stage,
1162                 run.builder.config.build,
1163                 run.target,
1164             ),
1165             target: run.target,
1166         });
1167     }
1168
1169     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1170         // This prevents miri from being built for "dist" or "install"
1171         // on the stable/beta channels. It is a nightly-only tool and should
1172         // not be included.
1173         if !builder.build.unstable_features() {
1174             return None;
1175         }
1176         let compiler = self.compiler;
1177         let target = self.target;
1178
1179         let miri = builder
1180             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1181             .or_else(|| {
1182                 missing_tool("miri", builder.build.config.missing_tools);
1183                 None
1184             })?;
1185         let cargomiri = builder
1186             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1187             .or_else(|| {
1188                 missing_tool("cargo miri", builder.build.config.missing_tools);
1189                 None
1190             })?;
1191
1192         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1193         tarball.set_overlay(OverlayKind::Miri);
1194         tarball.is_preview(true);
1195         tarball.add_file(miri, "bin", 0o755);
1196         tarball.add_file(cargomiri, "bin", 0o755);
1197         tarball.add_legal_and_readme_to("share/doc/miri");
1198         Some(tarball.generate())
1199     }
1200 }
1201
1202 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1203 pub struct Rustfmt {
1204     pub compiler: Compiler,
1205     pub target: TargetSelection,
1206 }
1207
1208 impl Step for Rustfmt {
1209     type Output = Option<GeneratedTarball>;
1210     const DEFAULT: bool = true;
1211     const ONLY_HOSTS: bool = true;
1212
1213     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1214         let default = should_build_extended_tool(&run.builder, "rustfmt");
1215         run.path("rustfmt").default_condition(default)
1216     }
1217
1218     fn make_run(run: RunConfig<'_>) {
1219         run.builder.ensure(Rustfmt {
1220             compiler: run.builder.compiler_for(
1221                 run.builder.top_stage,
1222                 run.builder.config.build,
1223                 run.target,
1224             ),
1225             target: run.target,
1226         });
1227     }
1228
1229     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1230         let compiler = self.compiler;
1231         let target = self.target;
1232
1233         let rustfmt = builder
1234             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1235             .or_else(|| {
1236                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1237                 None
1238             })?;
1239         let cargofmt = builder
1240             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1241             .or_else(|| {
1242                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1243                 None
1244             })?;
1245
1246         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1247         tarball.set_overlay(OverlayKind::Rustfmt);
1248         tarball.is_preview(true);
1249         tarball.add_file(rustfmt, "bin", 0o755);
1250         tarball.add_file(cargofmt, "bin", 0o755);
1251         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1252         Some(tarball.generate())
1253     }
1254 }
1255
1256 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1257 pub struct RustDemangler {
1258     pub compiler: Compiler,
1259     pub target: TargetSelection,
1260 }
1261
1262 impl Step for RustDemangler {
1263     type Output = Option<GeneratedTarball>;
1264     const DEFAULT: bool = true;
1265     const ONLY_HOSTS: bool = true;
1266
1267     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1268         // While other tools use `should_build_extended_tool` to decide whether to be run by
1269         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1270         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1271         // we run the step by default when only `extended = true`, and decide whether to actually
1272         // run it or not later.
1273         let default = run.builder.config.extended;
1274         run.path("rust-demangler").default_condition(default)
1275     }
1276
1277     fn make_run(run: RunConfig<'_>) {
1278         run.builder.ensure(RustDemangler {
1279             compiler: run.builder.compiler_for(
1280                 run.builder.top_stage,
1281                 run.builder.config.build,
1282                 run.target,
1283             ),
1284             target: run.target,
1285         });
1286     }
1287
1288     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1289         let compiler = self.compiler;
1290         let target = self.target;
1291
1292         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1293         let condition = should_build_extended_tool(builder, "rust-demangler")
1294             || builder.config.profiler_enabled(target);
1295         if builder.config.extended && !condition {
1296             return None;
1297         }
1298
1299         let rust_demangler = builder
1300             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1301             .expect("rust-demangler expected to build - in-tree tool");
1302
1303         // Prepare the image directory
1304         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1305         tarball.set_overlay(OverlayKind::RustDemangler);
1306         tarball.is_preview(true);
1307         tarball.add_file(&rust_demangler, "bin", 0o755);
1308         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1309         Some(tarball.generate())
1310     }
1311 }
1312
1313 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1314 pub struct Extended {
1315     stage: u32,
1316     host: TargetSelection,
1317     target: TargetSelection,
1318 }
1319
1320 impl Step for Extended {
1321     type Output = ();
1322     const DEFAULT: bool = true;
1323     const ONLY_HOSTS: bool = true;
1324
1325     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1326         let builder = run.builder;
1327         run.path("extended").default_condition(builder.config.extended)
1328     }
1329
1330     fn make_run(run: RunConfig<'_>) {
1331         run.builder.ensure(Extended {
1332             stage: run.builder.top_stage,
1333             host: run.builder.config.build,
1334             target: run.target,
1335         });
1336     }
1337
1338     /// Creates a combined installer for the specified target in the provided stage.
1339     fn run(self, builder: &Builder<'_>) {
1340         let target = self.target;
1341         let stage = self.stage;
1342         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1343
1344         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1345
1346         let mut tarballs = Vec::new();
1347         let mut built_tools = HashSet::new();
1348         macro_rules! add_component {
1349             ($name:expr => $step:expr) => {
1350                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1351                     tarballs.push(tarball);
1352                     built_tools.insert($name);
1353                 }
1354             };
1355         }
1356
1357         // When rust-std package split from rustc, we needed to ensure that during
1358         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1359         // the std files during uninstall. To do this ensure that rustc comes
1360         // before rust-std in the list below.
1361         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1362         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1363
1364         if target.contains("windows-gnu") {
1365             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1366         }
1367
1368         add_component!("rust-docs" => Docs { host: target });
1369         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1370         add_component!("cargo" => Cargo { compiler, target });
1371         add_component!("rustfmt" => Rustfmt { compiler, target });
1372         add_component!("rls" => Rls { compiler, target });
1373         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1374         add_component!("llvm-components" => LlvmTools { target });
1375         add_component!("clippy" => Clippy { compiler, target });
1376         add_component!("miri" => Miri { compiler, target });
1377         add_component!("analysis" => Analysis { compiler, target });
1378
1379         let etc = builder.src.join("src/etc/installer");
1380
1381         // Avoid producing tarballs during a dry run.
1382         if builder.config.dry_run {
1383             return;
1384         }
1385
1386         let tarball = Tarball::new(builder, "rust", &target.triple);
1387         let generated = tarball.combine(&tarballs);
1388
1389         let tmp = tmpdir(builder).join("combined-tarball");
1390         let work = generated.work_dir();
1391
1392         let mut license = String::new();
1393         license += &builder.read(&builder.src.join("COPYRIGHT"));
1394         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1395         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1396         license.push('\n');
1397         license.push('\n');
1398
1399         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1400         let mut rtf = rtf.to_string();
1401         rtf.push('\n');
1402         for line in license.lines() {
1403             rtf.push_str(line);
1404             rtf.push_str("\\line ");
1405         }
1406         rtf.push('}');
1407
1408         fn filter(contents: &str, marker: &str) -> String {
1409             let start = format!("tool-{}-start", marker);
1410             let end = format!("tool-{}-end", marker);
1411             let mut lines = Vec::new();
1412             let mut omitted = false;
1413             for line in contents.lines() {
1414                 if line.contains(&start) {
1415                     omitted = true;
1416                 } else if line.contains(&end) {
1417                     omitted = false;
1418                 } else if !omitted {
1419                     lines.push(line);
1420                 }
1421             }
1422
1423             lines.join("\n")
1424         }
1425
1426         let xform = |p: &Path| {
1427             let mut contents = t!(fs::read_to_string(p));
1428             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1429                 if !built_tools.contains(tool) {
1430                     contents = filter(&contents, tool);
1431                 }
1432             }
1433             let ret = tmp.join(p.file_name().unwrap());
1434             t!(fs::write(&ret, &contents));
1435             ret
1436         };
1437
1438         if target.contains("apple-darwin") {
1439             builder.info("building pkg installer");
1440             let pkg = tmp.join("pkg");
1441             let _ = fs::remove_dir_all(&pkg);
1442
1443             let pkgbuild = |component: &str| {
1444                 let mut cmd = Command::new("pkgbuild");
1445                 cmd.arg("--identifier")
1446                     .arg(format!("org.rust-lang.{}", component))
1447                     .arg("--scripts")
1448                     .arg(pkg.join(component))
1449                     .arg("--nopayload")
1450                     .arg(pkg.join(component).with_extension("pkg"));
1451                 builder.run(&mut cmd);
1452             };
1453
1454             let prepare = |name: &str| {
1455                 builder.create_dir(&pkg.join(name));
1456                 builder.cp_r(
1457                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1458                     &pkg.join(name),
1459                 );
1460                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1461                 pkgbuild(name);
1462             };
1463             prepare("rustc");
1464             prepare("cargo");
1465             prepare("rust-std");
1466             prepare("rust-analysis");
1467             prepare("clippy");
1468             for tool in &["rust-docs", "rust-demangler", "rls", "rust-analyzer", "miri"] {
1469                 if built_tools.contains(tool) {
1470                     prepare(tool);
1471                 }
1472             }
1473             // create an 'uninstall' package
1474             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1475             pkgbuild("uninstall");
1476
1477             builder.create_dir(&pkg.join("res"));
1478             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1479             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1480             let mut cmd = Command::new("productbuild");
1481             cmd.arg("--distribution")
1482                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1483                 .arg("--resources")
1484                 .arg(pkg.join("res"))
1485                 .arg(distdir(builder).join(format!(
1486                     "{}-{}.pkg",
1487                     pkgname(builder, "rust"),
1488                     target.triple
1489                 )))
1490                 .arg("--package-path")
1491                 .arg(&pkg);
1492             let _time = timeit(builder);
1493             builder.run(&mut cmd);
1494         }
1495
1496         if target.contains("windows") {
1497             let exe = tmp.join("exe");
1498             let _ = fs::remove_dir_all(&exe);
1499
1500             let prepare = |name: &str| {
1501                 builder.create_dir(&exe.join(name));
1502                 let dir = if name == "rust-std" || name == "rust-analysis" {
1503                     format!("{}-{}", name, target.triple)
1504                 } else if name == "rls" {
1505                     "rls-preview".to_string()
1506                 } else if name == "rust-analyzer" {
1507                     "rust-analyzer-preview".to_string()
1508                 } else if name == "clippy" {
1509                     "clippy-preview".to_string()
1510                 } else if name == "rust-demangler" {
1511                     "rust-demangler-preview".to_string()
1512                 } else if name == "miri" {
1513                     "miri-preview".to_string()
1514                 } else {
1515                     name.to_string()
1516                 };
1517                 builder.cp_r(
1518                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1519                     &exe.join(name),
1520                 );
1521                 builder.remove(&exe.join(name).join("manifest.in"));
1522             };
1523             prepare("rustc");
1524             prepare("cargo");
1525             prepare("rust-analysis");
1526             prepare("rust-docs");
1527             prepare("rust-std");
1528             prepare("clippy");
1529             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1530                 if built_tools.contains(tool) {
1531                     prepare(tool);
1532                 }
1533             }
1534             if target.contains("windows-gnu") {
1535                 prepare("rust-mingw");
1536             }
1537
1538             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1539
1540             // Generate msi installer
1541             let wix_path = env::var_os("WIX")
1542                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1543             let wix = PathBuf::from(wix_path);
1544             let heat = wix.join("bin/heat.exe");
1545             let candle = wix.join("bin/candle.exe");
1546             let light = wix.join("bin/light.exe");
1547
1548             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1549             builder.run(
1550                 Command::new(&heat)
1551                     .current_dir(&exe)
1552                     .arg("dir")
1553                     .arg("rustc")
1554                     .args(&heat_flags)
1555                     .arg("-cg")
1556                     .arg("RustcGroup")
1557                     .arg("-dr")
1558                     .arg("Rustc")
1559                     .arg("-var")
1560                     .arg("var.RustcDir")
1561                     .arg("-out")
1562                     .arg(exe.join("RustcGroup.wxs")),
1563             );
1564             builder.run(
1565                 Command::new(&heat)
1566                     .current_dir(&exe)
1567                     .arg("dir")
1568                     .arg("rust-docs")
1569                     .args(&heat_flags)
1570                     .arg("-cg")
1571                     .arg("DocsGroup")
1572                     .arg("-dr")
1573                     .arg("Docs")
1574                     .arg("-var")
1575                     .arg("var.DocsDir")
1576                     .arg("-out")
1577                     .arg(exe.join("DocsGroup.wxs"))
1578                     .arg("-t")
1579                     .arg(etc.join("msi/squash-components.xsl")),
1580             );
1581             builder.run(
1582                 Command::new(&heat)
1583                     .current_dir(&exe)
1584                     .arg("dir")
1585                     .arg("cargo")
1586                     .args(&heat_flags)
1587                     .arg("-cg")
1588                     .arg("CargoGroup")
1589                     .arg("-dr")
1590                     .arg("Cargo")
1591                     .arg("-var")
1592                     .arg("var.CargoDir")
1593                     .arg("-out")
1594                     .arg(exe.join("CargoGroup.wxs"))
1595                     .arg("-t")
1596                     .arg(etc.join("msi/remove-duplicates.xsl")),
1597             );
1598             builder.run(
1599                 Command::new(&heat)
1600                     .current_dir(&exe)
1601                     .arg("dir")
1602                     .arg("rust-std")
1603                     .args(&heat_flags)
1604                     .arg("-cg")
1605                     .arg("StdGroup")
1606                     .arg("-dr")
1607                     .arg("Std")
1608                     .arg("-var")
1609                     .arg("var.StdDir")
1610                     .arg("-out")
1611                     .arg(exe.join("StdGroup.wxs")),
1612             );
1613             if built_tools.contains("rls") {
1614                 builder.run(
1615                     Command::new(&heat)
1616                         .current_dir(&exe)
1617                         .arg("dir")
1618                         .arg("rls")
1619                         .args(&heat_flags)
1620                         .arg("-cg")
1621                         .arg("RlsGroup")
1622                         .arg("-dr")
1623                         .arg("Rls")
1624                         .arg("-var")
1625                         .arg("var.RlsDir")
1626                         .arg("-out")
1627                         .arg(exe.join("RlsGroup.wxs"))
1628                         .arg("-t")
1629                         .arg(etc.join("msi/remove-duplicates.xsl")),
1630                 );
1631             }
1632             if built_tools.contains("rust-analyzer") {
1633                 builder.run(
1634                     Command::new(&heat)
1635                         .current_dir(&exe)
1636                         .arg("dir")
1637                         .arg("rust-analyzer")
1638                         .args(&heat_flags)
1639                         .arg("-cg")
1640                         .arg("RustAnalyzerGroup")
1641                         .arg("-dr")
1642                         .arg("RustAnalyzer")
1643                         .arg("-var")
1644                         .arg("var.RustAnalyzerDir")
1645                         .arg("-out")
1646                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1647                         .arg("-t")
1648                         .arg(etc.join("msi/remove-duplicates.xsl")),
1649                 );
1650             }
1651             builder.run(
1652                 Command::new(&heat)
1653                     .current_dir(&exe)
1654                     .arg("dir")
1655                     .arg("clippy")
1656                     .args(&heat_flags)
1657                     .arg("-cg")
1658                     .arg("ClippyGroup")
1659                     .arg("-dr")
1660                     .arg("Clippy")
1661                     .arg("-var")
1662                     .arg("var.ClippyDir")
1663                     .arg("-out")
1664                     .arg(exe.join("ClippyGroup.wxs"))
1665                     .arg("-t")
1666                     .arg(etc.join("msi/remove-duplicates.xsl")),
1667             );
1668             if built_tools.contains("rust-demangler") {
1669                 builder.run(
1670                     Command::new(&heat)
1671                         .current_dir(&exe)
1672                         .arg("dir")
1673                         .arg("rust-demangler")
1674                         .args(&heat_flags)
1675                         .arg("-cg")
1676                         .arg("RustDemanglerGroup")
1677                         .arg("-dr")
1678                         .arg("RustDemangler")
1679                         .arg("-var")
1680                         .arg("var.RustDemanglerDir")
1681                         .arg("-out")
1682                         .arg(exe.join("RustDemanglerGroup.wxs"))
1683                         .arg("-t")
1684                         .arg(etc.join("msi/remove-duplicates.xsl")),
1685                 );
1686             }
1687             if built_tools.contains("miri") {
1688                 builder.run(
1689                     Command::new(&heat)
1690                         .current_dir(&exe)
1691                         .arg("dir")
1692                         .arg("miri")
1693                         .args(&heat_flags)
1694                         .arg("-cg")
1695                         .arg("MiriGroup")
1696                         .arg("-dr")
1697                         .arg("Miri")
1698                         .arg("-var")
1699                         .arg("var.MiriDir")
1700                         .arg("-out")
1701                         .arg(exe.join("MiriGroup.wxs"))
1702                         .arg("-t")
1703                         .arg(etc.join("msi/remove-duplicates.xsl")),
1704                 );
1705             }
1706             builder.run(
1707                 Command::new(&heat)
1708                     .current_dir(&exe)
1709                     .arg("dir")
1710                     .arg("rust-analysis")
1711                     .args(&heat_flags)
1712                     .arg("-cg")
1713                     .arg("AnalysisGroup")
1714                     .arg("-dr")
1715                     .arg("Analysis")
1716                     .arg("-var")
1717                     .arg("var.AnalysisDir")
1718                     .arg("-out")
1719                     .arg(exe.join("AnalysisGroup.wxs"))
1720                     .arg("-t")
1721                     .arg(etc.join("msi/remove-duplicates.xsl")),
1722             );
1723             if target.contains("windows-gnu") {
1724                 builder.run(
1725                     Command::new(&heat)
1726                         .current_dir(&exe)
1727                         .arg("dir")
1728                         .arg("rust-mingw")
1729                         .args(&heat_flags)
1730                         .arg("-cg")
1731                         .arg("GccGroup")
1732                         .arg("-dr")
1733                         .arg("Gcc")
1734                         .arg("-var")
1735                         .arg("var.GccDir")
1736                         .arg("-out")
1737                         .arg(exe.join("GccGroup.wxs")),
1738                 );
1739             }
1740
1741             let candle = |input: &Path| {
1742                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1743                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1744                 let mut cmd = Command::new(&candle);
1745                 cmd.current_dir(&exe)
1746                     .arg("-nologo")
1747                     .arg("-dRustcDir=rustc")
1748                     .arg("-dDocsDir=rust-docs")
1749                     .arg("-dCargoDir=cargo")
1750                     .arg("-dStdDir=rust-std")
1751                     .arg("-dAnalysisDir=rust-analysis")
1752                     .arg("-dClippyDir=clippy")
1753                     .arg("-arch")
1754                     .arg(&arch)
1755                     .arg("-out")
1756                     .arg(&output)
1757                     .arg(&input);
1758                 add_env(builder, &mut cmd, target);
1759
1760                 if built_tools.contains("rust-demangler") {
1761                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1762                 }
1763                 if built_tools.contains("rls") {
1764                     cmd.arg("-dRlsDir=rls");
1765                 }
1766                 if built_tools.contains("rust-analyzer") {
1767                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1768                 }
1769                 if built_tools.contains("miri") {
1770                     cmd.arg("-dMiriDir=miri");
1771                 }
1772                 if target.contains("windows-gnu") {
1773                     cmd.arg("-dGccDir=rust-mingw");
1774                 }
1775                 builder.run(&mut cmd);
1776             };
1777             candle(&xform(&etc.join("msi/rust.wxs")));
1778             candle(&etc.join("msi/ui.wxs"));
1779             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1780             candle("RustcGroup.wxs".as_ref());
1781             candle("DocsGroup.wxs".as_ref());
1782             candle("CargoGroup.wxs".as_ref());
1783             candle("StdGroup.wxs".as_ref());
1784             candle("ClippyGroup.wxs".as_ref());
1785             if built_tools.contains("rust-demangler") {
1786                 candle("RustDemanglerGroup.wxs".as_ref());
1787             }
1788             if built_tools.contains("rls") {
1789                 candle("RlsGroup.wxs".as_ref());
1790             }
1791             if built_tools.contains("rust-analyzer") {
1792                 candle("RustAnalyzerGroup.wxs".as_ref());
1793             }
1794             if built_tools.contains("miri") {
1795                 candle("MiriGroup.wxs".as_ref());
1796             }
1797             candle("AnalysisGroup.wxs".as_ref());
1798
1799             if target.contains("windows-gnu") {
1800                 candle("GccGroup.wxs".as_ref());
1801             }
1802
1803             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1804             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1805             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1806
1807             builder.info(&format!("building `msi` installer with {:?}", light));
1808             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1809             let mut cmd = Command::new(&light);
1810             cmd.arg("-nologo")
1811                 .arg("-ext")
1812                 .arg("WixUIExtension")
1813                 .arg("-ext")
1814                 .arg("WixUtilExtension")
1815                 .arg("-out")
1816                 .arg(exe.join(&filename))
1817                 .arg("rust.wixobj")
1818                 .arg("ui.wixobj")
1819                 .arg("rustwelcomedlg.wixobj")
1820                 .arg("RustcGroup.wixobj")
1821                 .arg("DocsGroup.wixobj")
1822                 .arg("CargoGroup.wixobj")
1823                 .arg("StdGroup.wixobj")
1824                 .arg("AnalysisGroup.wixobj")
1825                 .arg("ClippyGroup.wixobj")
1826                 .current_dir(&exe);
1827
1828             if built_tools.contains("rls") {
1829                 cmd.arg("RlsGroup.wixobj");
1830             }
1831             if built_tools.contains("rust-analyzer") {
1832                 cmd.arg("RustAnalyzerGroup.wixobj");
1833             }
1834             if built_tools.contains("rust-demangler") {
1835                 cmd.arg("RustDemanglerGroup.wixobj");
1836             }
1837             if built_tools.contains("miri") {
1838                 cmd.arg("MiriGroup.wixobj");
1839             }
1840
1841             if target.contains("windows-gnu") {
1842                 cmd.arg("GccGroup.wixobj");
1843             }
1844             // ICE57 wrongly complains about the shortcuts
1845             cmd.arg("-sice:ICE57");
1846
1847             let _time = timeit(builder);
1848             builder.run(&mut cmd);
1849
1850             if !builder.config.dry_run {
1851                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1852             }
1853         }
1854     }
1855 }
1856
1857 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1858     let mut parts = builder.version.split('.');
1859     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1860         .env("CFG_RELEASE_NUM", &builder.version)
1861         .env("CFG_RELEASE", builder.rust_release())
1862         .env("CFG_VER_MAJOR", parts.next().unwrap())
1863         .env("CFG_VER_MINOR", parts.next().unwrap())
1864         .env("CFG_VER_PATCH", parts.next().unwrap())
1865         .env("CFG_VER_BUILD", "0") // just needed to build
1866         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1867         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1868         .env("CFG_BUILD", target.triple)
1869         .env("CFG_CHANNEL", &builder.config.channel);
1870
1871     if target.contains("windows-gnu") {
1872         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1873     } else {
1874         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1875     }
1876 }
1877
1878 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1879 ///
1880
1881 /// Returns whether the files were actually copied.
1882 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1883     if let Some(config) = builder.config.target_config.get(&target) {
1884         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1885             // If the LLVM was externally provided, then we don't currently copy
1886             // artifacts into the sysroot. This is not necessarily the right
1887             // choice (in particular, it will require the LLVM dylib to be in
1888             // the linker's load path at runtime), but the common use case for
1889             // external LLVMs is distribution provided LLVMs, and in that case
1890             // they're usually in the standard search path (e.g., /usr/lib) and
1891             // copying them here is going to cause problems as we may end up
1892             // with the wrong files and isn't what distributions want.
1893             //
1894             // This behavior may be revisited in the future though.
1895             //
1896             // If the LLVM is coming from ourselves (just from CI) though, we
1897             // still want to install it, as it otherwise won't be available.
1898             return false;
1899         }
1900     }
1901
1902     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1903     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1904     // clear why this is the case, though. llvm-config will emit the versioned
1905     // paths and we don't want those in the sysroot (as we're expecting
1906     // unversioned paths).
1907     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1908         let src_libdir = builder.llvm_out(target).join("lib");
1909         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1910         if llvm_dylib_path.exists() {
1911             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1912         }
1913         !builder.config.dry_run
1914     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1915         let mut cmd = Command::new(llvm_config);
1916         cmd.arg("--libfiles");
1917         builder.verbose(&format!("running {:?}", cmd));
1918         let files = output(&mut cmd);
1919         let build_llvm_out = &builder.llvm_out(builder.config.build);
1920         let target_llvm_out = &builder.llvm_out(target);
1921         for file in files.trim_end().split(' ') {
1922             // If we're not using a custom LLVM, make sure we package for the target.
1923             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1924                 target_llvm_out.join(relative_path)
1925             } else {
1926                 PathBuf::from(file)
1927             };
1928             builder.install(&file, dst_libdir, 0o644);
1929         }
1930         !builder.config.dry_run
1931     } else {
1932         false
1933     }
1934 }
1935
1936 /// Maybe add libLLVM.so to the target lib-dir for linking.
1937 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1938     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1939     // We do not need to copy LLVM files into the sysroot if it is not
1940     // dynamically linked; it is already included into librustc_llvm
1941     // statically.
1942     if builder.config.llvm_link_shared {
1943         maybe_install_llvm(builder, target, &dst_libdir);
1944     }
1945 }
1946
1947 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1948 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1949     let dst_libdir =
1950         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1951     // We do not need to copy LLVM files into the sysroot if it is not
1952     // dynamically linked; it is already included into librustc_llvm
1953     // statically.
1954     if builder.config.llvm_link_shared {
1955         maybe_install_llvm(builder, target, &dst_libdir);
1956     }
1957 }
1958
1959 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1960 pub struct LlvmTools {
1961     pub target: TargetSelection,
1962 }
1963
1964 impl Step for LlvmTools {
1965     type Output = Option<GeneratedTarball>;
1966     const ONLY_HOSTS: bool = true;
1967     const DEFAULT: bool = true;
1968
1969     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1970         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1971         run.path("llvm-tools").default_condition(default)
1972     }
1973
1974     fn make_run(run: RunConfig<'_>) {
1975         run.builder.ensure(LlvmTools { target: run.target });
1976     }
1977
1978     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1979         let target = self.target;
1980
1981         /* run only if llvm-config isn't used */
1982         if let Some(config) = builder.config.target_config.get(&target) {
1983             if let Some(ref _s) = config.llvm_config {
1984                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1985                 return None;
1986             }
1987         }
1988
1989         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1990         tarball.set_overlay(OverlayKind::LLVM);
1991         tarball.is_preview(true);
1992
1993         // Prepare the image directory
1994         let src_bindir = builder.llvm_out(target).join("bin");
1995         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1996         for tool in LLVM_TOOLS {
1997             let exe = src_bindir.join(exe(tool, target));
1998             tarball.add_file(&exe, &dst_bindir, 0o755);
1999         }
2000
2001         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2002         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2003         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2004         // compiler libraries.
2005         maybe_install_llvm_target(builder, target, tarball.image_dir());
2006
2007         Some(tarball.generate())
2008     }
2009 }
2010
2011 // Tarball intended for internal consumption to ease rustc/std development.
2012 //
2013 // Should not be considered stable by end users.
2014 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2015 pub struct RustDev {
2016     pub target: TargetSelection,
2017 }
2018
2019 impl Step for RustDev {
2020     type Output = Option<GeneratedTarball>;
2021     const DEFAULT: bool = true;
2022     const ONLY_HOSTS: bool = true;
2023
2024     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2025         run.path("rust-dev")
2026     }
2027
2028     fn make_run(run: RunConfig<'_>) {
2029         run.builder.ensure(RustDev { target: run.target });
2030     }
2031
2032     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2033         let target = self.target;
2034
2035         /* run only if llvm-config isn't used */
2036         if let Some(config) = builder.config.target_config.get(&target) {
2037             if let Some(ref _s) = config.llvm_config {
2038                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2039                 return None;
2040             }
2041         }
2042
2043         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2044         tarball.set_overlay(OverlayKind::LLVM);
2045
2046         let src_bindir = builder.llvm_out(target).join("bin");
2047         for bin in &[
2048             "llvm-config",
2049             "llvm-ar",
2050             "llvm-objdump",
2051             "llvm-profdata",
2052             "llvm-bcanalyzer",
2053             "llvm-cov",
2054             "llvm-dwp",
2055             "llvm-nm",
2056         ] {
2057             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2058         }
2059
2060         // We don't build LLD on some platforms, so only add it if it exists
2061         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2062         if lld_path.exists() {
2063             tarball.add_file(lld_path, "bin", 0o755);
2064         }
2065
2066         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2067
2068         // Copy the include directory as well; needed mostly to build
2069         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2070         // just broadly useful to be able to link against the bundled LLVM.
2071         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2072
2073         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2074         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2075         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2076         // compiler libraries.
2077         let dst_libdir = tarball.image_dir().join("lib");
2078         maybe_install_llvm(builder, target, &dst_libdir);
2079         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2080         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2081
2082         Some(tarball.generate())
2083     }
2084 }
2085
2086 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2087 /// release process to avoid cloning the monorepo and building stuff.
2088 ///
2089 /// Should not be considered stable by end users.
2090 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2091 pub struct BuildManifest {
2092     pub target: TargetSelection,
2093 }
2094
2095 impl Step for BuildManifest {
2096     type Output = GeneratedTarball;
2097     const DEFAULT: bool = false;
2098     const ONLY_HOSTS: bool = true;
2099
2100     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2101         run.path("build-manifest")
2102     }
2103
2104     fn make_run(run: RunConfig<'_>) {
2105         run.builder.ensure(BuildManifest { target: run.target });
2106     }
2107
2108     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2109         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2110
2111         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2112         tarball.add_file(&build_manifest, "bin", 0o755);
2113         tarball.generate()
2114     }
2115 }
2116
2117 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2118 ///
2119 /// Currently this is the PGO profile data.
2120 ///
2121 /// Should not be considered stable by end users.
2122 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2123 pub struct ReproducibleArtifacts {
2124     pub target: TargetSelection,
2125 }
2126
2127 impl Step for ReproducibleArtifacts {
2128     type Output = Option<GeneratedTarball>;
2129     const DEFAULT: bool = true;
2130     const ONLY_HOSTS: bool = true;
2131
2132     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2133         run.path("reproducible-artifacts")
2134     }
2135
2136     fn make_run(run: RunConfig<'_>) {
2137         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2138     }
2139
2140     fn run(self, builder: &Builder<'_>) -> Self::Output {
2141         let mut added_anything = false;
2142         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2143         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2144             tarball.add_file(path, ".", 0o644);
2145             added_anything = true;
2146         }
2147         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2148             tarball.add_file(path, ".", 0o644);
2149             added_anything = true;
2150         }
2151         if added_anything { Some(tarball.generate()) } else { None }
2152     }
2153 }