src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("src/doc").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         run.path("src/librustc")
 100     }
 101
 102     fn make_run(run: RunConfig<'_>) {
 103         run.builder.ensure(RustcDocs { host: run.target });
 104     }
 105
 106     /// Builds the `rustc-docs` installer component.
 107     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 108         let host = self.host;
 109         if !builder.config.compiler_docs {
 110             return None;
 111         }
 112         builder.default_doc(&[]);
 113
 114         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 115         tarball.set_product_name("Rustc Documentation");
 116         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 117         Some(tarball.generate())
 118     }
 119 }
 120
 121 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 122     let mut found = Vec::with_capacity(files.len());
 123
 124     for file in files {
 125         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 126
 127         if let Some(file_path) = file_path {
 128             found.push(file_path);
 129         } else {
 130             panic!("Could not find '{}' in {:?}", file, path);
 131         }
 132     }
 133
 134     found
 135 }
 136
 137 fn make_win_dist(
 138     rust_root: &Path,
 139     plat_root: &Path,
 140     target: TargetSelection,
 141     builder: &Builder<'_>,
 142 ) {
 143     //Ask gcc where it keeps its stuff
 144     let mut cmd = Command::new(builder.cc(target));
 145     cmd.arg("-print-search-dirs");
 146     let gcc_out = output(&mut cmd);
 147
 148     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 149     let mut lib_path = Vec::new();
 150
 151     for line in gcc_out.lines() {
 152         let idx = line.find(':').unwrap();
 153         let key = &line[..idx];
 154         let trim_chars: &[_] = &[' ', '='];
 155         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 156
 157         if key == "programs" {
 158             bin_path.extend(value);
 159         } else if key == "libraries" {
 160             lib_path.extend(value);
 161         }
 162     }
 163
 164     let compiler = if target == "i686-pc-windows-gnu" {
 165         "i686-w64-mingw32-gcc.exe"
 166     } else if target == "x86_64-pc-windows-gnu" {
 167         "x86_64-w64-mingw32-gcc.exe"
 168     } else {
 169         "gcc.exe"
 170     };
 171     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 172     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 173     if target.starts_with("i686-") {
 174         rustc_dlls.push("libgcc_s_dw2-1.dll");
 175     } else {
 176         rustc_dlls.push("libgcc_s_seh-1.dll");
 177     }
 178
 179     let target_libs = [
 180         //MinGW libs
 181         "libgcc.a",
 182         "libgcc_eh.a",
 183         "libgcc_s.a",
 184         "libm.a",
 185         "libmingw32.a",
 186         "libmingwex.a",
 187         "libstdc++.a",
 188         "libiconv.a",
 189         "libmoldname.a",
 190         "libpthread.a",
 191         //Windows import libs
 192         "libadvapi32.a",
 193         "libbcrypt.a",
 194         "libcomctl32.a",
 195         "libcomdlg32.a",
 196         "libcredui.a",
 197         "libcrypt32.a",
 198         "libdbghelp.a",
 199         "libgdi32.a",
 200         "libimagehlp.a",
 201         "libiphlpapi.a",
 202         "libkernel32.a",
 203         "libmsimg32.a",
 204         "libmsvcrt.a",
 205         "libodbc32.a",
 206         "libole32.a",
 207         "liboleaut32.a",
 208         "libopengl32.a",
 209         "libpsapi.a",
 210         "librpcrt4.a",
 211         "libsecur32.a",
 212         "libsetupapi.a",
 213         "libshell32.a",
 214         "libsynchronization.a",
 215         "libuser32.a",
 216         "libuserenv.a",
 217         "libuuid.a",
 218         "libwinhttp.a",
 219         "libwinmm.a",
 220         "libwinspool.a",
 221         "libws2_32.a",
 222         "libwsock32.a",
 223     ];
 224
 225     //Find mingw artifacts we want to bundle
 226     let target_tools = find_files(&target_tools, &bin_path);
 227     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 228     let target_libs = find_files(&target_libs, &lib_path);
 229
 230     // Copy runtime dlls next to rustc.exe
 231     let dist_bin_dir = rust_root.join("bin/");
 232     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 233     for src in rustc_dlls {
 234         builder.copy_to_folder(&src, &dist_bin_dir);
 235     }
 236
 237     //Copy platform tools to platform-specific bin directory
 238     let target_bin_dir = plat_root
 239         .join("lib")
 240         .join("rustlib")
 241         .join(target.triple)
 242         .join("bin")
 243         .join("self-contained");
 244     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 245     for src in target_tools {
 246         builder.copy_to_folder(&src, &target_bin_dir);
 247     }
 248
 249     // Warn windows-gnu users that the bundled GCC cannot compile C files
 250     builder.create(
 251         &target_bin_dir.join("GCC-WARNING.txt"),
 252         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 253          used as a linker. In order to be able to compile projects containing C code use \
 254          the GCC provided by MinGW or Cygwin.",
 255     );
 256
 257     //Copy platform libs to platform-specific lib directory
 258     let target_lib_dir = plat_root
 259         .join("lib")
 260         .join("rustlib")
 261         .join(target.triple)
 262         .join("lib")
 263         .join("self-contained");
 264     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 265     for src in target_libs {
 266         builder.copy_to_folder(&src, &target_lib_dir);
 267     }
 268 }
 269
 270 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 271 pub struct Mingw {
 272     pub host: TargetSelection,
 273 }
 274
 275 impl Step for Mingw {
 276     type Output = Option<GeneratedTarball>;
 277     const DEFAULT: bool = true;
 278
 279     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 280         run.never()
 281     }
 282
 283     fn make_run(run: RunConfig<'_>) {
 284         run.builder.ensure(Mingw { host: run.target });
 285     }
 286
 287     /// Builds the `rust-mingw` installer component.
 288     ///
 289     /// This contains all the bits and pieces to run the MinGW Windows targets
 290     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 291     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 292         let host = self.host;
 293         if !host.contains("pc-windows-gnu") {
 294             return None;
 295         }
 296
 297         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 298         tarball.set_product_name("Rust MinGW");
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 305
 306         Some(tarball.generate())
 307     }
 308 }
 309
 310 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 311 pub struct Rustc {
 312     pub compiler: Compiler,
 313 }
 314
 315 impl Step for Rustc {
 316     type Output = GeneratedTarball;
 317     const DEFAULT: bool = true;
 318     const ONLY_HOSTS: bool = true;
 319
 320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 321         run.path("src/librustc")
 322     }
 323
 324     fn make_run(run: RunConfig<'_>) {
 325         run.builder
 326             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 327     }
 328
 329     /// Creates the `rustc` installer component.
 330     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 331         let compiler = self.compiler;
 332         let host = self.compiler.host;
 333
 334         let tarball = Tarball::new(builder, "rustc", &host.triple);
 335
 336         // Prepare the rustc "image", what will actually end up getting installed
 337         prepare_image(builder, compiler, tarball.image_dir());
 338
 339         // On MinGW we've got a few runtime DLL dependencies that we need to
 340         // include. The first argument to this script is where to put these DLLs
 341         // (the image we're creating), and the second argument is a junk directory
 342         // to ignore all other MinGW stuff the script creates.
 343         //
 344         // On 32-bit MinGW we're always including a DLL which needs some extra
 345         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 346         // anything requiring us to distribute a license, but it's likely the
 347         // install will *also* include the rust-mingw package, which also needs
 348         // licenses, so to be safe we just include it here in all MinGW packages.
 349         if host.contains("pc-windows-gnu") {
 350             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 351             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 352         }
 353
 354         return tarball.generate();
 355
 356         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 357             let host = compiler.host;
 358             let src = builder.sysroot(compiler);
 359
 360             // Copy rustc/rustdoc binaries
 361             t!(fs::create_dir_all(image.join("bin")));
 362             builder.cp_r(&src.join("bin"), &image.join("bin"));
 363
 364             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 365
 366             let libdir_relative = builder.libdir_relative(compiler);
 367
 368             // Copy runtime DLLs needed by the compiler
 369             if libdir_relative.to_str() != Some("bin") {
 370                 let libdir = builder.rustc_libdir(compiler);
 371                 for entry in builder.read_dir(&libdir) {
 372                     let name = entry.file_name();
 373                     if let Some(s) = name.to_str() {
 374                         if is_dylib(s) {
 375                             // Don't use custom libdir here because ^lib/ will be resolved again
 376                             // with installer
 377                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 378                         }
 379                     }
 380                 }
 381             }
 382
 383             // Copy over the codegen backends
 384             let backends_src = builder.sysroot_codegen_backends(compiler);
 385             let backends_rel = backends_src
 386                 .strip_prefix(&src)
 387                 .unwrap()
 388                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 389                 .unwrap();
 390             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 391             let backends_dst = image.join("lib").join(&backends_rel);
 392
 393             t!(fs::create_dir_all(&backends_dst));
 394             builder.cp_r(&backends_src, &backends_dst);
 395
 396             // Copy libLLVM.so to the lib dir as well, if needed. While not
 397             // technically needed by rustc itself it's needed by lots of other
 398             // components like the llvm tools and LLD. LLD is included below and
 399             // tools/LLDB come later, so let's just throw it in the rustc
 400             // component for now.
 401             maybe_install_llvm_runtime(builder, host, image);
 402
 403             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 405             t!(fs::create_dir_all(&dst_dir));
 406
 407             // Copy over lld if it's there
 408             if builder.config.lld_enabled {
 409                 let rust_lld = exe("rust-lld", compiler.host);
 410                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 411                 // for `-Z gcc-ld=lld`
 412                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 413                 t!(fs::create_dir(&gcc_lld_dir));
 414                 builder.copy(&src_dir.join(&rust_lld), &gcc_lld_dir.join(exe("ld", compiler.host)));
 415             }
 416
 417             // Copy over llvm-dwp if it's there
 418             let exe = exe("rust-llvm-dwp", compiler.host);
 419             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 420
 421             // Man pages
 422             t!(fs::create_dir_all(image.join("share/man/man1")));
 423             let man_src = builder.src.join("src/doc/man");
 424             let man_dst = image.join("share/man/man1");
 425
 426             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 427             let time = env::var("SOURCE_DATE_EPOCH")
 428                 .map(|timestamp| {
 429                     let epoch = timestamp
 430                         .parse()
 431                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 432                         .unwrap();
 433
 434                     time::at(Timespec::new(epoch, 0))
 435                 })
 436                 .unwrap_or_else(|_| time::now());
 437
 438             let month_year = t!(time::strftime("%B %Y", &time));
 439             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 440             // to hardlink, and we don't want to edit the source templates
 441             for file_entry in builder.read_dir(&man_src) {
 442                 let page_src = file_entry.path();
 443                 let page_dst = man_dst.join(file_entry.file_name());
 444                 t!(fs::copy(&page_src, &page_dst));
 445                 // template in month/year and version number
 446                 builder.replace_in_file(
 447                     &page_dst,
 448                     &[
 449                         ("<INSERT DATE HERE>", &month_year),
 450                         ("<INSERT VERSION HERE>", &builder.version),
 451                     ],
 452                 );
 453             }
 454
 455             // Debugger scripts
 456             builder
 457                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 458
 459             // Misc license info
 460             let cp = |file: &str| {
 461                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 462             };
 463             cp("COPYRIGHT");
 464             cp("LICENSE-APACHE");
 465             cp("LICENSE-MIT");
 466             cp("README.md");
 467         }
 468     }
 469 }
 470
 471 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 472 pub struct DebuggerScripts {
 473     pub sysroot: Interned<PathBuf>,
 474     pub host: TargetSelection,
 475 }
 476
 477 impl Step for DebuggerScripts {
 478     type Output = ();
 479
 480     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 481         run.path("src/lldb_batchmode.py")
 482     }
 483
 484     fn make_run(run: RunConfig<'_>) {
 485         run.builder.ensure(DebuggerScripts {
 486             sysroot: run
 487                 .builder
 488                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 489             host: run.target,
 490         });
 491     }
 492
 493     /// Copies debugger scripts for `target` into the `sysroot` specified.
 494     fn run(self, builder: &Builder<'_>) {
 495         let host = self.host;
 496         let sysroot = self.sysroot;
 497         let dst = sysroot.join("lib/rustlib/etc");
 498         t!(fs::create_dir_all(&dst));
 499         let cp_debugger_script = |file: &str| {
 500             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 501         };
 502         if host.contains("windows-msvc") {
 503             // windbg debugger scripts
 504             builder.install(
 505                 &builder.src.join("src/etc/rust-windbg.cmd"),
 506                 &sysroot.join("bin"),
 507                 0o755,
 508             );
 509
 510             cp_debugger_script("natvis/intrinsic.natvis");
 511             cp_debugger_script("natvis/liballoc.natvis");
 512             cp_debugger_script("natvis/libcore.natvis");
 513             cp_debugger_script("natvis/libstd.natvis");
 514         } else {
 515             cp_debugger_script("rust_types.py");
 516
 517             // gdb debugger scripts
 518             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 519             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 520
 521             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 522             cp_debugger_script("gdb_lookup.py");
 523             cp_debugger_script("gdb_providers.py");
 524
 525             // lldb debugger scripts
 526             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 527
 528             cp_debugger_script("lldb_lookup.py");
 529             cp_debugger_script("lldb_providers.py");
 530             cp_debugger_script("lldb_commands")
 531         }
 532     }
 533 }
 534
 535 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 536     // The only true set of target libraries came from the build triple, so
 537     // let's reduce redundant work by only producing archives from that host.
 538     if compiler.host != builder.config.build {
 539         builder.info("\tskipping, not a build host");
 540         true
 541     } else {
 542         false
 543     }
 544 }
 545
 546 /// Copy stamped files into an image's `target/lib` directory.
 547 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 548     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 549     let self_contained_dst = dst.join("self-contained");
 550     t!(fs::create_dir_all(&dst));
 551     t!(fs::create_dir_all(&self_contained_dst));
 552     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 553         if dependency_type == DependencyType::TargetSelfContained {
 554             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 555         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 556             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 557         }
 558     }
 559 }
 560
 561 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 562 pub struct Std {
 563     pub compiler: Compiler,
 564     pub target: TargetSelection,
 565 }
 566
 567 impl Step for Std {
 568     type Output = Option<GeneratedTarball>;
 569     const DEFAULT: bool = true;
 570
 571     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 572         run.path("library/std")
 573     }
 574
 575     fn make_run(run: RunConfig<'_>) {
 576         run.builder.ensure(Std {
 577             compiler: run.builder.compiler_for(
 578                 run.builder.top_stage,
 579                 run.builder.config.build,
 580                 run.target,
 581             ),
 582             target: run.target,
 583         });
 584     }
 585
 586     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 587         let compiler = self.compiler;
 588         let target = self.target;
 589
 590         if skip_host_target_lib(builder, compiler) {
 591             return None;
 592         }
 593
 594         builder.ensure(compile::Std { compiler, target });
 595
 596         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 597         tarball.include_target_in_component_name(true);
 598
 599         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 600         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 601         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 602
 603         Some(tarball.generate())
 604     }
 605 }
 606
 607 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 608 pub struct RustcDev {
 609     pub compiler: Compiler,
 610     pub target: TargetSelection,
 611 }
 612
 613 impl Step for RustcDev {
 614     type Output = Option<GeneratedTarball>;
 615     const DEFAULT: bool = true;
 616     const ONLY_HOSTS: bool = true;
 617
 618     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 619         run.path("rustc-dev")
 620     }
 621
 622     fn make_run(run: RunConfig<'_>) {
 623         run.builder.ensure(RustcDev {
 624             compiler: run.builder.compiler_for(
 625                 run.builder.top_stage,
 626                 run.builder.config.build,
 627                 run.target,
 628             ),
 629             target: run.target,
 630         });
 631     }
 632
 633     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 634         let compiler = self.compiler;
 635         let target = self.target;
 636         if skip_host_target_lib(builder, compiler) {
 637             return None;
 638         }
 639
 640         builder.ensure(compile::Rustc { compiler, target });
 641
 642         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 643
 644         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 645         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 646         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 647
 648         let src_files = &["Cargo.lock"];
 649         // This is the reduced set of paths which will become the rustc-dev component
 650         // (essentially the compiler crates and all of their path dependencies).
 651         copy_src_dirs(
 652             builder,
 653             &builder.src,
 654             &["compiler"],
 655             &[],
 656             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 657         );
 658         // This particular crate is used as a build dependency of the above.
 659         copy_src_dirs(
 660             builder,
 661             &builder.src,
 662             &["src/build_helper"],
 663             &[],
 664             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 665         );
 666         for file in src_files {
 667             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 668         }
 669
 670         Some(tarball.generate())
 671     }
 672 }
 673
 674 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 675 pub struct Analysis {
 676     pub compiler: Compiler,
 677     pub target: TargetSelection,
 678 }
 679
 680 impl Step for Analysis {
 681     type Output = Option<GeneratedTarball>;
 682     const DEFAULT: bool = true;
 683
 684     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 685         let default = should_build_extended_tool(&run.builder, "analysis");
 686         run.path("analysis").default_condition(default)
 687     }
 688
 689     fn make_run(run: RunConfig<'_>) {
 690         run.builder.ensure(Analysis {
 691             // Find the actual compiler (handling the full bootstrap option) which
 692             // produced the save-analysis data because that data isn't copied
 693             // through the sysroot uplifting.
 694             compiler: run.builder.compiler_for(
 695                 run.builder.top_stage,
 696                 run.builder.config.build,
 697                 run.target,
 698             ),
 699             target: run.target,
 700         });
 701     }
 702
 703     /// Creates a tarball of save-analysis metadata, if available.
 704     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 705         let compiler = self.compiler;
 706         let target = self.target;
 707         if compiler.host != builder.config.build {
 708             return None;
 709         }
 710
 711         builder.ensure(compile::Std { compiler, target });
 712         let src = builder
 713             .stage_out(compiler, Mode::Std)
 714             .join(target.triple)
 715             .join(builder.cargo_dir())
 716             .join("deps")
 717             .join("save-analysis");
 718
 719         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 720         tarball.include_target_in_component_name(true);
 721         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 722         Some(tarball.generate())
 723     }
 724 }
 725
 726 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 727 /// `dst_dir`.
 728 fn copy_src_dirs(
 729     builder: &Builder<'_>,
 730     base: &Path,
 731     src_dirs: &[&str],
 732     exclude_dirs: &[&str],
 733     dst_dir: &Path,
 734 ) {
 735     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 736         let spath = match path.to_str() {
 737             Some(path) => path,
 738             None => return false,
 739         };
 740         if spath.ends_with('~') || spath.ends_with(".pyc") {
 741             return false;
 742         }
 743
 744         const LLVM_PROJECTS: &[&str] = &[
 745             "llvm-project/clang",
 746             "llvm-project\\clang",
 747             "llvm-project/libunwind",
 748             "llvm-project\\libunwind",
 749             "llvm-project/lld",
 750             "llvm-project\\lld",
 751             "llvm-project/lldb",
 752             "llvm-project\\lldb",
 753             "llvm-project/llvm",
 754             "llvm-project\\llvm",
 755             "llvm-project/compiler-rt",
 756             "llvm-project\\compiler-rt",
 757         ];
 758         if spath.contains("llvm-project")
 759             && !spath.ends_with("llvm-project")
 760             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 761         {
 762             return false;
 763         }
 764
 765         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 766         if LLVM_TEST.iter().any(|path| spath.contains(path))
 767             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 768         {
 769             return false;
 770         }
 771
 772         let full_path = Path::new(dir).join(path);
 773         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 774             return false;
 775         }
 776
 777         let excludes = [
 778             "CVS",
 779             "RCS",
 780             "SCCS",
 781             ".git",
 782             ".gitignore",
 783             ".gitmodules",
 784             ".gitattributes",
 785             ".cvsignore",
 786             ".svn",
 787             ".arch-ids",
 788             "{arch}",
 789             "=RELEASE-ID",
 790             "=meta-update",
 791             "=update",
 792             ".bzr",
 793             ".bzrignore",
 794             ".bzrtags",
 795             ".hg",
 796             ".hgignore",
 797             ".hgrags",
 798             "_darcs",
 799         ];
 800         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 801     }
 802
 803     // Copy the directories using our filter
 804     for item in src_dirs {
 805         let dst = &dst_dir.join(item);
 806         t!(fs::create_dir_all(dst));
 807         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 808     }
 809 }
 810
 811 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 812 pub struct Src;
 813
 814 impl Step for Src {
 815     /// The output path of the src installer tarball
 816     type Output = GeneratedTarball;
 817     const DEFAULT: bool = true;
 818     const ONLY_HOSTS: bool = true;
 819
 820     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 821         run.path("src")
 822     }
 823
 824     fn make_run(run: RunConfig<'_>) {
 825         run.builder.ensure(Src);
 826     }
 827
 828     /// Creates the `rust-src` installer component
 829     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 830         let tarball = Tarball::new_targetless(builder, "rust-src");
 831
 832         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 833         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 834         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 835         // and fix them...
 836         //
 837         // NOTE: if you update the paths here, you also should update the "virtual" path
 838         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 839         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 840
 841         let src_files = ["Cargo.lock"];
 842         // This is the reduced set of paths which will become the rust-src component
 843         // (essentially libstd and all of its path dependencies).
 844         copy_src_dirs(
 845             builder,
 846             &builder.src,
 847             &["library", "src/llvm-project/libunwind"],
 848             &[
 849                 // not needed and contains symlinks which rustup currently
 850                 // chokes on when unpacking.
 851                 "library/backtrace/crates",
 852             ],
 853             &dst_src,
 854         );
 855         for file in src_files.iter() {
 856             builder.copy(&builder.src.join(file), &dst_src.join(file));
 857         }
 858
 859         tarball.generate()
 860     }
 861 }
 862
 863 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 864 pub struct PlainSourceTarball;
 865
 866 impl Step for PlainSourceTarball {
 867     /// Produces the location of the tarball generated
 868     type Output = GeneratedTarball;
 869     const DEFAULT: bool = true;
 870     const ONLY_HOSTS: bool = true;
 871
 872     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 873         let builder = run.builder;
 874         run.path("src").default_condition(builder.config.rust_dist_src)
 875     }
 876
 877     fn make_run(run: RunConfig<'_>) {
 878         run.builder.ensure(PlainSourceTarball);
 879     }
 880
 881     /// Creates the plain source tarball
 882     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 883         let tarball = Tarball::new(builder, "rustc", "src");
 884         let plain_dst_src = tarball.image_dir();
 885
 886         // This is the set of root paths which will become part of the source package
 887         let src_files = [
 888             "COPYRIGHT",
 889             "LICENSE-APACHE",
 890             "LICENSE-MIT",
 891             "CONTRIBUTING.md",
 892             "README.md",
 893             "RELEASES.md",
 894             "configure",
 895             "x.py",
 896             "config.toml.example",
 897             "Cargo.toml",
 898             "Cargo.lock",
 899         ];
 900         let src_dirs = ["src", "compiler", "library"];
 901
 902         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 903
 904         // Copy the files normally
 905         for item in &src_files {
 906             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 907         }
 908
 909         // Create the version file
 910         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 911         if let Some(sha) = builder.rust_sha() {
 912             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 913         }
 914
 915         // If we're building from git sources, we need to vendor a complete distribution.
 916         if builder.rust_info.is_git() {
 917             // Vendor all Cargo dependencies
 918             let mut cmd = Command::new(&builder.initial_cargo);
 919             cmd.arg("vendor")
 920                 .arg("--sync")
 921                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 922                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 923                 .current_dir(&plain_dst_src);
 924             builder.run(&mut cmd);
 925         }
 926
 927         tarball.bare()
 928     }
 929 }
 930
 931 // We have to run a few shell scripts, which choke quite a bit on both `\`
 932 // characters and on `C:\` paths, so normalize both of them away.
 933 pub fn sanitize_sh(path: &Path) -> String {
 934     let path = path.to_str().unwrap().replace("\\", "/");
 935     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 936
 937     fn unc_to_lfs(s: &str) -> &str {
 938         s.strip_prefix("//?/").unwrap_or(s)
 939     }
 940
 941     fn change_drive(s: &str) -> Option<String> {
 942         let mut ch = s.chars();
 943         let drive = ch.next().unwrap_or('C');
 944         if ch.next() != Some(':') {
 945             return None;
 946         }
 947         if ch.next() != Some('/') {
 948             return None;
 949         }
 950         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 951     }
 952 }
 953
 954 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 955 pub struct Cargo {
 956     pub compiler: Compiler,
 957     pub target: TargetSelection,
 958 }
 959
 960 impl Step for Cargo {
 961     type Output = Option<GeneratedTarball>;
 962     const DEFAULT: bool = true;
 963     const ONLY_HOSTS: bool = true;
 964
 965     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 966         let default = should_build_extended_tool(&run.builder, "cargo");
 967         run.path("cargo").default_condition(default)
 968     }
 969
 970     fn make_run(run: RunConfig<'_>) {
 971         run.builder.ensure(Cargo {
 972             compiler: run.builder.compiler_for(
 973                 run.builder.top_stage,
 974                 run.builder.config.build,
 975                 run.target,
 976             ),
 977             target: run.target,
 978         });
 979     }
 980
 981     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 982         let compiler = self.compiler;
 983         let target = self.target;
 984
 985         let cargo = builder.ensure(tool::Cargo { compiler, target });
 986         let src = builder.src.join("src/tools/cargo");
 987         let etc = src.join("src/etc");
 988
 989         // Prepare the image directory
 990         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 991         tarball.set_overlay(OverlayKind::Cargo);
 992
 993         tarball.add_file(&cargo, "bin", 0o755);
 994         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 995         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 996         tarball.add_dir(etc.join("man"), "share/man/man1");
 997         tarball.add_legal_and_readme_to("share/doc/cargo");
 998
 999         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1000             let dirent = dirent.expect("read dir entry");
1001             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1002                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1003             }
1004         }
1005
1006         Some(tarball.generate())
1007     }
1008 }
1009
1010 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1011 pub struct Rls {
1012     pub compiler: Compiler,
1013     pub target: TargetSelection,
1014 }
1015
1016 impl Step for Rls {
1017     type Output = Option<GeneratedTarball>;
1018     const ONLY_HOSTS: bool = true;
1019     const DEFAULT: bool = true;
1020
1021     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1022         let default = should_build_extended_tool(&run.builder, "rls");
1023         run.path("rls").default_condition(default)
1024     }
1025
1026     fn make_run(run: RunConfig<'_>) {
1027         run.builder.ensure(Rls {
1028             compiler: run.builder.compiler_for(
1029                 run.builder.top_stage,
1030                 run.builder.config.build,
1031                 run.target,
1032             ),
1033             target: run.target,
1034         });
1035     }
1036
1037     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1038         let compiler = self.compiler;
1039         let target = self.target;
1040
1041         let rls = builder
1042             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1043             .or_else(|| {
1044                 missing_tool("RLS", builder.build.config.missing_tools);
1045                 None
1046             })?;
1047
1048         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1049         tarball.set_overlay(OverlayKind::RLS);
1050         tarball.is_preview(true);
1051         tarball.add_file(rls, "bin", 0o755);
1052         tarball.add_legal_and_readme_to("share/doc/rls");
1053         Some(tarball.generate())
1054     }
1055 }
1056
1057 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1058 pub struct RustAnalyzer {
1059     pub compiler: Compiler,
1060     pub target: TargetSelection,
1061 }
1062
1063 impl Step for RustAnalyzer {
1064     type Output = Option<GeneratedTarball>;
1065     const DEFAULT: bool = true;
1066     const ONLY_HOSTS: bool = true;
1067
1068     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1069         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1070         run.path("rust-analyzer").default_condition(default)
1071     }
1072
1073     fn make_run(run: RunConfig<'_>) {
1074         run.builder.ensure(RustAnalyzer {
1075             compiler: run.builder.compiler_for(
1076                 run.builder.top_stage,
1077                 run.builder.config.build,
1078                 run.target,
1079             ),
1080             target: run.target,
1081         });
1082     }
1083
1084     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1085         // This prevents rust-analyzer from being built for "dist" or "install"
1086         // on the stable/beta channels. It is a nightly-only tool and should
1087         // not be included.
1088         if !builder.build.unstable_features() {
1089             return None;
1090         }
1091         let compiler = self.compiler;
1092         let target = self.target;
1093
1094         if target.contains("riscv64") {
1095             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1096             // to build. See #74813 for details.
1097             return None;
1098         }
1099
1100         let rust_analyzer = builder
1101             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1102             .expect("rust-analyzer always builds");
1103
1104         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1105         tarball.set_overlay(OverlayKind::RustAnalyzer);
1106         tarball.is_preview(true);
1107         tarball.add_file(rust_analyzer, "bin", 0o755);
1108         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1109         Some(tarball.generate())
1110     }
1111 }
1112
1113 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1114 pub struct Clippy {
1115     pub compiler: Compiler,
1116     pub target: TargetSelection,
1117 }
1118
1119 impl Step for Clippy {
1120     type Output = Option<GeneratedTarball>;
1121     const DEFAULT: bool = true;
1122     const ONLY_HOSTS: bool = true;
1123
1124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1125         let default = should_build_extended_tool(&run.builder, "clippy");
1126         run.path("clippy").default_condition(default)
1127     }
1128
1129     fn make_run(run: RunConfig<'_>) {
1130         run.builder.ensure(Clippy {
1131             compiler: run.builder.compiler_for(
1132                 run.builder.top_stage,
1133                 run.builder.config.build,
1134                 run.target,
1135             ),
1136             target: run.target,
1137         });
1138     }
1139
1140     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1141         let compiler = self.compiler;
1142         let target = self.target;
1143
1144         // Prepare the image directory
1145         // We expect clippy to build, because we've exited this step above if tool
1146         // state for clippy isn't testing.
1147         let clippy = builder
1148             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1149             .expect("clippy expected to build - essential tool");
1150         let cargoclippy = builder
1151             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1152             .expect("clippy expected to build - essential tool");
1153
1154         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1155         tarball.set_overlay(OverlayKind::Clippy);
1156         tarball.is_preview(true);
1157         tarball.add_file(clippy, "bin", 0o755);
1158         tarball.add_file(cargoclippy, "bin", 0o755);
1159         tarball.add_legal_and_readme_to("share/doc/clippy");
1160         Some(tarball.generate())
1161     }
1162 }
1163
1164 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1165 pub struct Miri {
1166     pub compiler: Compiler,
1167     pub target: TargetSelection,
1168 }
1169
1170 impl Step for Miri {
1171     type Output = Option<GeneratedTarball>;
1172     const DEFAULT: bool = true;
1173     const ONLY_HOSTS: bool = true;
1174
1175     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1176         let default = should_build_extended_tool(&run.builder, "miri");
1177         run.path("miri").default_condition(default)
1178     }
1179
1180     fn make_run(run: RunConfig<'_>) {
1181         run.builder.ensure(Miri {
1182             compiler: run.builder.compiler_for(
1183                 run.builder.top_stage,
1184                 run.builder.config.build,
1185                 run.target,
1186             ),
1187             target: run.target,
1188         });
1189     }
1190
1191     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1192         // This prevents miri from being built for "dist" or "install"
1193         // on the stable/beta channels. It is a nightly-only tool and should
1194         // not be included.
1195         if !builder.build.unstable_features() {
1196             return None;
1197         }
1198         let compiler = self.compiler;
1199         let target = self.target;
1200
1201         let miri = builder
1202             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1203             .or_else(|| {
1204                 missing_tool("miri", builder.build.config.missing_tools);
1205                 None
1206             })?;
1207         let cargomiri = builder
1208             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1209             .or_else(|| {
1210                 missing_tool("cargo miri", builder.build.config.missing_tools);
1211                 None
1212             })?;
1213
1214         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1215         tarball.set_overlay(OverlayKind::Miri);
1216         tarball.is_preview(true);
1217         tarball.add_file(miri, "bin", 0o755);
1218         tarball.add_file(cargomiri, "bin", 0o755);
1219         tarball.add_legal_and_readme_to("share/doc/miri");
1220         Some(tarball.generate())
1221     }
1222 }
1223
1224 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1225 pub struct Rustfmt {
1226     pub compiler: Compiler,
1227     pub target: TargetSelection,
1228 }
1229
1230 impl Step for Rustfmt {
1231     type Output = Option<GeneratedTarball>;
1232     const DEFAULT: bool = true;
1233     const ONLY_HOSTS: bool = true;
1234
1235     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1236         let default = should_build_extended_tool(&run.builder, "rustfmt");
1237         run.path("rustfmt").default_condition(default)
1238     }
1239
1240     fn make_run(run: RunConfig<'_>) {
1241         run.builder.ensure(Rustfmt {
1242             compiler: run.builder.compiler_for(
1243                 run.builder.top_stage,
1244                 run.builder.config.build,
1245                 run.target,
1246             ),
1247             target: run.target,
1248         });
1249     }
1250
1251     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1252         let compiler = self.compiler;
1253         let target = self.target;
1254
1255         let rustfmt = builder
1256             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1257             .or_else(|| {
1258                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1259                 None
1260             })?;
1261         let cargofmt = builder
1262             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1263             .or_else(|| {
1264                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1265                 None
1266             })?;
1267
1268         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1269         tarball.set_overlay(OverlayKind::Rustfmt);
1270         tarball.is_preview(true);
1271         tarball.add_file(rustfmt, "bin", 0o755);
1272         tarball.add_file(cargofmt, "bin", 0o755);
1273         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1274         Some(tarball.generate())
1275     }
1276 }
1277
1278 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1279 pub struct RustDemangler {
1280     pub compiler: Compiler,
1281     pub target: TargetSelection,
1282 }
1283
1284 impl Step for RustDemangler {
1285     type Output = Option<GeneratedTarball>;
1286     const DEFAULT: bool = true;
1287     const ONLY_HOSTS: bool = true;
1288
1289     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1290         // While other tools use `should_build_extended_tool` to decide whether to be run by
1291         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1292         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1293         // we run the step by default when only `extended = true`, and decide whether to actually
1294         // run it or not later.
1295         let default = run.builder.config.extended;
1296         run.path("rust-demangler").default_condition(default)
1297     }
1298
1299     fn make_run(run: RunConfig<'_>) {
1300         run.builder.ensure(RustDemangler {
1301             compiler: run.builder.compiler_for(
1302                 run.builder.top_stage,
1303                 run.builder.config.build,
1304                 run.target,
1305             ),
1306             target: run.target,
1307         });
1308     }
1309
1310     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1311         let compiler = self.compiler;
1312         let target = self.target;
1313
1314         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1315         let condition = should_build_extended_tool(builder, "rust-demangler")
1316             || builder.config.profiler_enabled(target);
1317         if builder.config.extended && !condition {
1318             return None;
1319         }
1320
1321         let rust_demangler = builder
1322             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1323             .expect("rust-demangler expected to build - in-tree tool");
1324
1325         // Prepare the image directory
1326         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1327         tarball.set_overlay(OverlayKind::RustDemangler);
1328         tarball.is_preview(true);
1329         tarball.add_file(&rust_demangler, "bin", 0o755);
1330         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1331         Some(tarball.generate())
1332     }
1333 }
1334
1335 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1336 pub struct Extended {
1337     stage: u32,
1338     host: TargetSelection,
1339     target: TargetSelection,
1340 }
1341
1342 impl Step for Extended {
1343     type Output = ();
1344     const DEFAULT: bool = true;
1345     const ONLY_HOSTS: bool = true;
1346
1347     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1348         let builder = run.builder;
1349         run.path("extended").default_condition(builder.config.extended)
1350     }
1351
1352     fn make_run(run: RunConfig<'_>) {
1353         run.builder.ensure(Extended {
1354             stage: run.builder.top_stage,
1355             host: run.builder.config.build,
1356             target: run.target,
1357         });
1358     }
1359
1360     /// Creates a combined installer for the specified target in the provided stage.
1361     fn run(self, builder: &Builder<'_>) {
1362         let target = self.target;
1363         let stage = self.stage;
1364         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1365
1366         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1367
1368         let mut tarballs = Vec::new();
1369         let mut built_tools = HashSet::new();
1370         macro_rules! add_component {
1371             ($name:expr => $step:expr) => {
1372                 if let Some(tarball) = builder.ensure_if_default($step) {
1373                     tarballs.push(tarball);
1374                     built_tools.insert($name);
1375                 }
1376             };
1377         }
1378
1379         // When rust-std package split from rustc, we needed to ensure that during
1380         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1381         // the std files during uninstall. To do this ensure that rustc comes
1382         // before rust-std in the list below.
1383         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1384         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1385
1386         if target.contains("windows-gnu") {
1387             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1388         }
1389
1390         add_component!("rust-docs" => Docs { host: target });
1391         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1392         add_component!("cargo" => Cargo { compiler, target });
1393         add_component!("rustfmt" => Rustfmt { compiler, target });
1394         add_component!("rls" => Rls { compiler, target });
1395         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1396         add_component!("llvm-components" => LlvmTools { target });
1397         add_component!("clippy" => Clippy { compiler, target });
1398         add_component!("miri" => Miri { compiler, target });
1399         add_component!("analysis" => Analysis { compiler, target });
1400
1401         let etc = builder.src.join("src/etc/installer");
1402
1403         // Avoid producing tarballs during a dry run.
1404         if builder.config.dry_run {
1405             return;
1406         }
1407
1408         let tarball = Tarball::new(builder, "rust", &target.triple);
1409         let generated = tarball.combine(&tarballs);
1410
1411         let tmp = tmpdir(builder).join("combined-tarball");
1412         let work = generated.work_dir();
1413
1414         let mut license = String::new();
1415         license += &builder.read(&builder.src.join("COPYRIGHT"));
1416         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1417         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1418         license.push('\n');
1419         license.push('\n');
1420
1421         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1422         let mut rtf = rtf.to_string();
1423         rtf.push('\n');
1424         for line in license.lines() {
1425             rtf.push_str(line);
1426             rtf.push_str("\\line ");
1427         }
1428         rtf.push('}');
1429
1430         fn filter(contents: &str, marker: &str) -> String {
1431             let start = format!("tool-{}-start", marker);
1432             let end = format!("tool-{}-end", marker);
1433             let mut lines = Vec::new();
1434             let mut omitted = false;
1435             for line in contents.lines() {
1436                 if line.contains(&start) {
1437                     omitted = true;
1438                 } else if line.contains(&end) {
1439                     omitted = false;
1440                 } else if !omitted {
1441                     lines.push(line);
1442                 }
1443             }
1444
1445             lines.join("\n")
1446         }
1447
1448         let xform = |p: &Path| {
1449             let mut contents = t!(fs::read_to_string(p));
1450             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1451                 if !built_tools.contains(tool) {
1452                     contents = filter(&contents, tool);
1453                 }
1454             }
1455             let ret = tmp.join(p.file_name().unwrap());
1456             t!(fs::write(&ret, &contents));
1457             ret
1458         };
1459
1460         if target.contains("apple-darwin") {
1461             builder.info("building pkg installer");
1462             let pkg = tmp.join("pkg");
1463             let _ = fs::remove_dir_all(&pkg);
1464
1465             let pkgbuild = |component: &str| {
1466                 let mut cmd = Command::new("pkgbuild");
1467                 cmd.arg("--identifier")
1468                     .arg(format!("org.rust-lang.{}", component))
1469                     .arg("--scripts")
1470                     .arg(pkg.join(component))
1471                     .arg("--nopayload")
1472                     .arg(pkg.join(component).with_extension("pkg"));
1473                 builder.run(&mut cmd);
1474             };
1475
1476             let prepare = |name: &str| {
1477                 builder.create_dir(&pkg.join(name));
1478                 builder.cp_r(
1479                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1480                     &pkg.join(name),
1481                 );
1482                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1483                 pkgbuild(name);
1484             };
1485             prepare("rustc");
1486             prepare("cargo");
1487             prepare("rust-docs");
1488             prepare("rust-std");
1489             prepare("rust-analysis");
1490             prepare("clippy");
1491             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1492                 if built_tools.contains(tool) {
1493                     prepare(tool);
1494                 }
1495             }
1496             // create an 'uninstall' package
1497             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1498             pkgbuild("uninstall");
1499
1500             builder.create_dir(&pkg.join("res"));
1501             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1502             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1503             let mut cmd = Command::new("productbuild");
1504             cmd.arg("--distribution")
1505                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1506                 .arg("--resources")
1507                 .arg(pkg.join("res"))
1508                 .arg(distdir(builder).join(format!(
1509                     "{}-{}.pkg",
1510                     pkgname(builder, "rust"),
1511                     target.triple
1512                 )))
1513                 .arg("--package-path")
1514                 .arg(&pkg);
1515             let _time = timeit(builder);
1516             builder.run(&mut cmd);
1517         }
1518
1519         if target.contains("windows") {
1520             let exe = tmp.join("exe");
1521             let _ = fs::remove_dir_all(&exe);
1522
1523             let prepare = |name: &str| {
1524                 builder.create_dir(&exe.join(name));
1525                 let dir = if name == "rust-std" || name == "rust-analysis" {
1526                     format!("{}-{}", name, target.triple)
1527                 } else if name == "rls" {
1528                     "rls-preview".to_string()
1529                 } else if name == "rust-analyzer" {
1530                     "rust-analyzer-preview".to_string()
1531                 } else if name == "clippy" {
1532                     "clippy-preview".to_string()
1533                 } else if name == "rust-demangler" {
1534                     "rust-demangler-preview".to_string()
1535                 } else if name == "miri" {
1536                     "miri-preview".to_string()
1537                 } else {
1538                     name.to_string()
1539                 };
1540                 builder.cp_r(
1541                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1542                     &exe.join(name),
1543                 );
1544                 builder.remove(&exe.join(name).join("manifest.in"));
1545             };
1546             prepare("rustc");
1547             prepare("cargo");
1548             prepare("rust-analysis");
1549             prepare("rust-docs");
1550             prepare("rust-std");
1551             prepare("clippy");
1552             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1553                 if built_tools.contains(tool) {
1554                     prepare(tool);
1555                 }
1556             }
1557             if target.contains("windows-gnu") {
1558                 prepare("rust-mingw");
1559             }
1560
1561             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1562
1563             // Generate msi installer
1564             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1565             let heat = wix.join("bin/heat.exe");
1566             let candle = wix.join("bin/candle.exe");
1567             let light = wix.join("bin/light.exe");
1568
1569             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1570             builder.run(
1571                 Command::new(&heat)
1572                     .current_dir(&exe)
1573                     .arg("dir")
1574                     .arg("rustc")
1575                     .args(&heat_flags)
1576                     .arg("-cg")
1577                     .arg("RustcGroup")
1578                     .arg("-dr")
1579                     .arg("Rustc")
1580                     .arg("-var")
1581                     .arg("var.RustcDir")
1582                     .arg("-out")
1583                     .arg(exe.join("RustcGroup.wxs")),
1584             );
1585             builder.run(
1586                 Command::new(&heat)
1587                     .current_dir(&exe)
1588                     .arg("dir")
1589                     .arg("rust-docs")
1590                     .args(&heat_flags)
1591                     .arg("-cg")
1592                     .arg("DocsGroup")
1593                     .arg("-dr")
1594                     .arg("Docs")
1595                     .arg("-var")
1596                     .arg("var.DocsDir")
1597                     .arg("-out")
1598                     .arg(exe.join("DocsGroup.wxs"))
1599                     .arg("-t")
1600                     .arg(etc.join("msi/squash-components.xsl")),
1601             );
1602             builder.run(
1603                 Command::new(&heat)
1604                     .current_dir(&exe)
1605                     .arg("dir")
1606                     .arg("cargo")
1607                     .args(&heat_flags)
1608                     .arg("-cg")
1609                     .arg("CargoGroup")
1610                     .arg("-dr")
1611                     .arg("Cargo")
1612                     .arg("-var")
1613                     .arg("var.CargoDir")
1614                     .arg("-out")
1615                     .arg(exe.join("CargoGroup.wxs"))
1616                     .arg("-t")
1617                     .arg(etc.join("msi/remove-duplicates.xsl")),
1618             );
1619             builder.run(
1620                 Command::new(&heat)
1621                     .current_dir(&exe)
1622                     .arg("dir")
1623                     .arg("rust-std")
1624                     .args(&heat_flags)
1625                     .arg("-cg")
1626                     .arg("StdGroup")
1627                     .arg("-dr")
1628                     .arg("Std")
1629                     .arg("-var")
1630                     .arg("var.StdDir")
1631                     .arg("-out")
1632                     .arg(exe.join("StdGroup.wxs")),
1633             );
1634             if built_tools.contains("rls") {
1635                 builder.run(
1636                     Command::new(&heat)
1637                         .current_dir(&exe)
1638                         .arg("dir")
1639                         .arg("rls")
1640                         .args(&heat_flags)
1641                         .arg("-cg")
1642                         .arg("RlsGroup")
1643                         .arg("-dr")
1644                         .arg("Rls")
1645                         .arg("-var")
1646                         .arg("var.RlsDir")
1647                         .arg("-out")
1648                         .arg(exe.join("RlsGroup.wxs"))
1649                         .arg("-t")
1650                         .arg(etc.join("msi/remove-duplicates.xsl")),
1651                 );
1652             }
1653             if built_tools.contains("rust-analyzer") {
1654                 builder.run(
1655                     Command::new(&heat)
1656                         .current_dir(&exe)
1657                         .arg("dir")
1658                         .arg("rust-analyzer")
1659                         .args(&heat_flags)
1660                         .arg("-cg")
1661                         .arg("RustAnalyzerGroup")
1662                         .arg("-dr")
1663                         .arg("RustAnalyzer")
1664                         .arg("-var")
1665                         .arg("var.RustAnalyzerDir")
1666                         .arg("-out")
1667                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1668                         .arg("-t")
1669                         .arg(etc.join("msi/remove-duplicates.xsl")),
1670                 );
1671             }
1672             builder.run(
1673                 Command::new(&heat)
1674                     .current_dir(&exe)
1675                     .arg("dir")
1676                     .arg("clippy")
1677                     .args(&heat_flags)
1678                     .arg("-cg")
1679                     .arg("ClippyGroup")
1680                     .arg("-dr")
1681                     .arg("Clippy")
1682                     .arg("-var")
1683                     .arg("var.ClippyDir")
1684                     .arg("-out")
1685                     .arg(exe.join("ClippyGroup.wxs"))
1686                     .arg("-t")
1687                     .arg(etc.join("msi/remove-duplicates.xsl")),
1688             );
1689             if built_tools.contains("rust-demangler") {
1690                 builder.run(
1691                     Command::new(&heat)
1692                         .current_dir(&exe)
1693                         .arg("dir")
1694                         .arg("rust-demangler")
1695                         .args(&heat_flags)
1696                         .arg("-cg")
1697                         .arg("RustDemanglerGroup")
1698                         .arg("-dr")
1699                         .arg("RustDemangler")
1700                         .arg("-var")
1701                         .arg("var.RustDemanglerDir")
1702                         .arg("-out")
1703                         .arg(exe.join("RustDemanglerGroup.wxs"))
1704                         .arg("-t")
1705                         .arg(etc.join("msi/remove-duplicates.xsl")),
1706                 );
1707             }
1708             if built_tools.contains("miri") {
1709                 builder.run(
1710                     Command::new(&heat)
1711                         .current_dir(&exe)
1712                         .arg("dir")
1713                         .arg("miri")
1714                         .args(&heat_flags)
1715                         .arg("-cg")
1716                         .arg("MiriGroup")
1717                         .arg("-dr")
1718                         .arg("Miri")
1719                         .arg("-var")
1720                         .arg("var.MiriDir")
1721                         .arg("-out")
1722                         .arg(exe.join("MiriGroup.wxs"))
1723                         .arg("-t")
1724                         .arg(etc.join("msi/remove-duplicates.xsl")),
1725                 );
1726             }
1727             builder.run(
1728                 Command::new(&heat)
1729                     .current_dir(&exe)
1730                     .arg("dir")
1731                     .arg("rust-analysis")
1732                     .args(&heat_flags)
1733                     .arg("-cg")
1734                     .arg("AnalysisGroup")
1735                     .arg("-dr")
1736                     .arg("Analysis")
1737                     .arg("-var")
1738                     .arg("var.AnalysisDir")
1739                     .arg("-out")
1740                     .arg(exe.join("AnalysisGroup.wxs"))
1741                     .arg("-t")
1742                     .arg(etc.join("msi/remove-duplicates.xsl")),
1743             );
1744             if target.contains("windows-gnu") {
1745                 builder.run(
1746                     Command::new(&heat)
1747                         .current_dir(&exe)
1748                         .arg("dir")
1749                         .arg("rust-mingw")
1750                         .args(&heat_flags)
1751                         .arg("-cg")
1752                         .arg("GccGroup")
1753                         .arg("-dr")
1754                         .arg("Gcc")
1755                         .arg("-var")
1756                         .arg("var.GccDir")
1757                         .arg("-out")
1758                         .arg(exe.join("GccGroup.wxs")),
1759                 );
1760             }
1761
1762             let candle = |input: &Path| {
1763                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1764                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1765                 let mut cmd = Command::new(&candle);
1766                 cmd.current_dir(&exe)
1767                     .arg("-nologo")
1768                     .arg("-dRustcDir=rustc")
1769                     .arg("-dDocsDir=rust-docs")
1770                     .arg("-dCargoDir=cargo")
1771                     .arg("-dStdDir=rust-std")
1772                     .arg("-dAnalysisDir=rust-analysis")
1773                     .arg("-dClippyDir=clippy")
1774                     .arg("-arch")
1775                     .arg(&arch)
1776                     .arg("-out")
1777                     .arg(&output)
1778                     .arg(&input);
1779                 add_env(builder, &mut cmd, target);
1780
1781                 if built_tools.contains("rust-demangler") {
1782                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1783                 }
1784                 if built_tools.contains("rls") {
1785                     cmd.arg("-dRlsDir=rls");
1786                 }
1787                 if built_tools.contains("rust-analyzer") {
1788                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1789                 }
1790                 if built_tools.contains("miri") {
1791                     cmd.arg("-dMiriDir=miri");
1792                 }
1793                 if target.contains("windows-gnu") {
1794                     cmd.arg("-dGccDir=rust-mingw");
1795                 }
1796                 builder.run(&mut cmd);
1797             };
1798             candle(&xform(&etc.join("msi/rust.wxs")));
1799             candle(&etc.join("msi/ui.wxs"));
1800             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1801             candle("RustcGroup.wxs".as_ref());
1802             candle("DocsGroup.wxs".as_ref());
1803             candle("CargoGroup.wxs".as_ref());
1804             candle("StdGroup.wxs".as_ref());
1805             candle("ClippyGroup.wxs".as_ref());
1806             if built_tools.contains("rust-demangler") {
1807                 candle("RustDemanglerGroup.wxs".as_ref());
1808             }
1809             if built_tools.contains("rls") {
1810                 candle("RlsGroup.wxs".as_ref());
1811             }
1812             if built_tools.contains("rust-analyzer") {
1813                 candle("RustAnalyzerGroup.wxs".as_ref());
1814             }
1815             if built_tools.contains("miri") {
1816                 candle("MiriGroup.wxs".as_ref());
1817             }
1818             candle("AnalysisGroup.wxs".as_ref());
1819
1820             if target.contains("windows-gnu") {
1821                 candle("GccGroup.wxs".as_ref());
1822             }
1823
1824             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1825             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1826             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1827
1828             builder.info(&format!("building `msi` installer with {:?}", light));
1829             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1830             let mut cmd = Command::new(&light);
1831             cmd.arg("-nologo")
1832                 .arg("-ext")
1833                 .arg("WixUIExtension")
1834                 .arg("-ext")
1835                 .arg("WixUtilExtension")
1836                 .arg("-out")
1837                 .arg(exe.join(&filename))
1838                 .arg("rust.wixobj")
1839                 .arg("ui.wixobj")
1840                 .arg("rustwelcomedlg.wixobj")
1841                 .arg("RustcGroup.wixobj")
1842                 .arg("DocsGroup.wixobj")
1843                 .arg("CargoGroup.wixobj")
1844                 .arg("StdGroup.wixobj")
1845                 .arg("AnalysisGroup.wixobj")
1846                 .arg("ClippyGroup.wixobj")
1847                 .current_dir(&exe);
1848
1849             if built_tools.contains("rls") {
1850                 cmd.arg("RlsGroup.wixobj");
1851             }
1852             if built_tools.contains("rust-analyzer") {
1853                 cmd.arg("RustAnalyzerGroup.wixobj");
1854             }
1855             if built_tools.contains("rust-demangler") {
1856                 cmd.arg("RustDemanglerGroup.wixobj");
1857             }
1858             if built_tools.contains("miri") {
1859                 cmd.arg("MiriGroup.wixobj");
1860             }
1861
1862             if target.contains("windows-gnu") {
1863                 cmd.arg("GccGroup.wixobj");
1864             }
1865             // ICE57 wrongly complains about the shortcuts
1866             cmd.arg("-sice:ICE57");
1867
1868             let _time = timeit(builder);
1869             builder.run(&mut cmd);
1870
1871             if !builder.config.dry_run {
1872                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1873             }
1874         }
1875     }
1876 }
1877
1878 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1879     let mut parts = builder.version.split('.');
1880     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1881         .env("CFG_RELEASE_NUM", &builder.version)
1882         .env("CFG_RELEASE", builder.rust_release())
1883         .env("CFG_VER_MAJOR", parts.next().unwrap())
1884         .env("CFG_VER_MINOR", parts.next().unwrap())
1885         .env("CFG_VER_PATCH", parts.next().unwrap())
1886         .env("CFG_VER_BUILD", "0") // just needed to build
1887         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1888         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1889         .env("CFG_BUILD", target.triple)
1890         .env("CFG_CHANNEL", &builder.config.channel);
1891
1892     if target.contains("windows-gnu") {
1893         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1894     } else {
1895         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1896     }
1897
1898     if target.contains("x86_64") {
1899         cmd.env("CFG_PLATFORM", "x64");
1900     } else {
1901         cmd.env("CFG_PLATFORM", "x86");
1902     }
1903 }
1904
1905 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1906 ///
1907
1908 /// Returns whether the files were actually copied.
1909 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1910     if let Some(config) = builder.config.target_config.get(&target) {
1911         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1912             // If the LLVM was externally provided, then we don't currently copy
1913             // artifacts into the sysroot. This is not necessarily the right
1914             // choice (in particular, it will require the LLVM dylib to be in
1915             // the linker's load path at runtime), but the common use case for
1916             // external LLVMs is distribution provided LLVMs, and in that case
1917             // they're usually in the standard search path (e.g., /usr/lib) and
1918             // copying them here is going to cause problems as we may end up
1919             // with the wrong files and isn't what distributions want.
1920             //
1921             // This behavior may be revisited in the future though.
1922             //
1923             // If the LLVM is coming from ourselves (just from CI) though, we
1924             // still want to install it, as it otherwise won't be available.
1925             return false;
1926         }
1927     }
1928
1929     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1930     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1931     // clear why this is the case, though. llvm-config will emit the versioned
1932     // paths and we don't want those in the sysroot (as we're expecting
1933     // unversioned paths).
1934     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1935         let src_libdir = builder.llvm_out(target).join("lib");
1936         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1937         if llvm_dylib_path.exists() {
1938             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1939         }
1940         !builder.config.dry_run
1941     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1942         let mut cmd = Command::new(llvm_config);
1943         cmd.arg("--libfiles");
1944         builder.verbose(&format!("running {:?}", cmd));
1945         let files = output(&mut cmd);
1946         let build_llvm_out = &builder.llvm_out(builder.config.build);
1947         let target_llvm_out = &builder.llvm_out(target);
1948         for file in files.trim_end().split(' ') {
1949             // If we're not using a custom LLVM, make sure we package for the target.
1950             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1951                 target_llvm_out.join(relative_path)
1952             } else {
1953                 PathBuf::from(file)
1954             };
1955             builder.install(&file, dst_libdir, 0o644);
1956         }
1957         !builder.config.dry_run
1958     } else {
1959         false
1960     }
1961 }
1962
1963 /// Maybe add libLLVM.so to the target lib-dir for linking.
1964 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1965     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1966     // We do not need to copy LLVM files into the sysroot if it is not
1967     // dynamically linked; it is already included into librustc_llvm
1968     // statically.
1969     if builder.config.llvm_link_shared {
1970         maybe_install_llvm(builder, target, &dst_libdir);
1971     }
1972 }
1973
1974 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1975 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1976     let dst_libdir =
1977         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1978     // We do not need to copy LLVM files into the sysroot if it is not
1979     // dynamically linked; it is already included into librustc_llvm
1980     // statically.
1981     if builder.config.llvm_link_shared {
1982         maybe_install_llvm(builder, target, &dst_libdir);
1983     }
1984 }
1985
1986 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1987 pub struct LlvmTools {
1988     pub target: TargetSelection,
1989 }
1990
1991 impl Step for LlvmTools {
1992     type Output = Option<GeneratedTarball>;
1993     const ONLY_HOSTS: bool = true;
1994     const DEFAULT: bool = true;
1995
1996     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1997         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1998         run.path("llvm-tools").default_condition(default)
1999     }
2000
2001     fn make_run(run: RunConfig<'_>) {
2002         run.builder.ensure(LlvmTools { target: run.target });
2003     }
2004
2005     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2006         let target = self.target;
2007
2008         /* run only if llvm-config isn't used */
2009         if let Some(config) = builder.config.target_config.get(&target) {
2010             if let Some(ref _s) = config.llvm_config {
2011                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2012                 return None;
2013             }
2014         }
2015
2016         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2017         tarball.set_overlay(OverlayKind::LLVM);
2018         tarball.is_preview(true);
2019
2020         // Prepare the image directory
2021         let src_bindir = builder.llvm_out(target).join("bin");
2022         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2023         for tool in LLVM_TOOLS {
2024             let exe = src_bindir.join(exe(tool, target));
2025             tarball.add_file(&exe, &dst_bindir, 0o755);
2026         }
2027
2028         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2029         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2030         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2031         // compiler libraries.
2032         maybe_install_llvm_target(builder, target, tarball.image_dir());
2033
2034         Some(tarball.generate())
2035     }
2036 }
2037
2038 // Tarball intended for internal consumption to ease rustc/std development.
2039 //
2040 // Should not be considered stable by end users.
2041 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2042 pub struct RustDev {
2043     pub target: TargetSelection,
2044 }
2045
2046 impl Step for RustDev {
2047     type Output = Option<GeneratedTarball>;
2048     const DEFAULT: bool = true;
2049     const ONLY_HOSTS: bool = true;
2050
2051     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2052         run.path("rust-dev")
2053     }
2054
2055     fn make_run(run: RunConfig<'_>) {
2056         run.builder.ensure(RustDev { target: run.target });
2057     }
2058
2059     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2060         let target = self.target;
2061
2062         /* run only if llvm-config isn't used */
2063         if let Some(config) = builder.config.target_config.get(&target) {
2064             if let Some(ref _s) = config.llvm_config {
2065                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2066                 return None;
2067             }
2068         }
2069
2070         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2071         tarball.set_overlay(OverlayKind::LLVM);
2072
2073         let src_bindir = builder.llvm_out(target).join("bin");
2074         for bin in &[
2075             "llvm-config",
2076             "llvm-ar",
2077             "llvm-objdump",
2078             "llvm-profdata",
2079             "llvm-bcanalyzer",
2080             "llvm-cov",
2081             "llvm-dwp",
2082         ] {
2083             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2084         }
2085         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2086
2087         // Copy the include directory as well; needed mostly to build
2088         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2089         // just broadly useful to be able to link against the bundled LLVM.
2090         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2091
2092         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2093         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2094         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2095         // compiler libraries.
2096         let dst_libdir = tarball.image_dir().join("lib");
2097         maybe_install_llvm(builder, target, &dst_libdir);
2098         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2099         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2100
2101         Some(tarball.generate())
2102     }
2103 }
2104
2105 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2106 /// release process to avoid cloning the monorepo and building stuff.
2107 ///
2108 /// Should not be considered stable by end users.
2109 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2110 pub struct BuildManifest {
2111     pub target: TargetSelection,
2112 }
2113
2114 impl Step for BuildManifest {
2115     type Output = GeneratedTarball;
2116     const DEFAULT: bool = false;
2117     const ONLY_HOSTS: bool = true;
2118
2119     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2120         run.path("src/tools/build-manifest")
2121     }
2122
2123     fn make_run(run: RunConfig<'_>) {
2124         run.builder.ensure(BuildManifest { target: run.target });
2125     }
2126
2127     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2128         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2129
2130         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2131         tarball.add_file(&build_manifest, "bin", 0o755);
2132         tarball.generate()
2133     }
2134 }
2135
2136 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2137 ///
2138 /// Currently this is the PGO profile data.
2139 ///
2140 /// Should not be considered stable by end users.
2141 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2142 pub struct ReproducibleArtifacts {
2143     pub target: TargetSelection,
2144 }
2145
2146 impl Step for ReproducibleArtifacts {
2147     type Output = Option<GeneratedTarball>;
2148     const DEFAULT: bool = true;
2149     const ONLY_HOSTS: bool = true;
2150
2151     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2152         run.path("reproducible")
2153     }
2154
2155     fn make_run(run: RunConfig<'_>) {
2156         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2157     }
2158
2159     fn run(self, builder: &Builder<'_>) -> Self::Output {
2160         let mut added_anything = false;
2161         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2162         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2163             tarball.add_file(path, ".", 0o644);
2164             added_anything = true;
2165         }
2166         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2167             tarball.add_file(path, ".", 0o644);
2168             added_anything = true;
2169         }
2170         if added_anything { Some(tarball.generate()) } else { None }
2171     }
2172 }