src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use build_helper::{output, t};
  18
  19 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  20 use crate::cache::{Interned, INTERNER};
  21 use crate::compile;
  22 use crate::config::TargetSelection;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27 use time::{self, Timespec};
  28
  29 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  30     format!("{}-{}", component, builder.rust_package_vers())
  31 }
  32
  33 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  34     builder.out.join("dist")
  35 }
  36
  37 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  38     builder.out.join("tmp/dist")
  39 }
  40
  41 fn missing_tool(tool_name: &str, skip: bool) {
  42     if skip {
  43         println!("Unable to build {}, skipping dist", tool_name)
  44     } else {
  45         panic!("Unable to build {}", tool_name)
  46     }
  47 }
  48
  49 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  50     if !builder.config.extended {
  51         return false;
  52     }
  53     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  54 }
  55
  56 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  57 pub struct Docs {
  58     pub host: TargetSelection,
  59 }
  60
  61 impl Step for Docs {
  62     type Output = Option<GeneratedTarball>;
  63     const DEFAULT: bool = true;
  64
  65     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  66         let default = run.builder.config.docs;
  67         run.path("src/doc").default_condition(default)
  68     }
  69
  70     fn make_run(run: RunConfig<'_>) {
  71         run.builder.ensure(Docs { host: run.target });
  72     }
  73
  74     /// Builds the `rust-docs` installer component.
  75     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  76         let host = self.host;
  77         builder.default_doc(&[]);
  78
  79         let dest = "share/doc/rust/html";
  80
  81         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  82         tarball.set_product_name("Rust Documentation");
  83         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  84         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  85         Some(tarball.generate())
  86     }
  87 }
  88
  89 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  90 pub struct RustcDocs {
  91     pub host: TargetSelection,
  92 }
  93
  94 impl Step for RustcDocs {
  95     type Output = Option<GeneratedTarball>;
  96     const DEFAULT: bool = true;
  97
  98     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  99         run.path("src/librustc")
 100     }
 101
 102     fn make_run(run: RunConfig<'_>) {
 103         run.builder.ensure(RustcDocs { host: run.target });
 104     }
 105
 106     /// Builds the `rustc-docs` installer component.
 107     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 108         let host = self.host;
 109         if !builder.config.compiler_docs {
 110             return None;
 111         }
 112         builder.default_doc(&[]);
 113
 114         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 115         tarball.set_product_name("Rustc Documentation");
 116         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 117         Some(tarball.generate())
 118     }
 119 }
 120
 121 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 122     let mut found = Vec::with_capacity(files.len());
 123
 124     for file in files {
 125         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 126
 127         if let Some(file_path) = file_path {
 128             found.push(file_path);
 129         } else {
 130             panic!("Could not find '{}' in {:?}", file, path);
 131         }
 132     }
 133
 134     found
 135 }
 136
 137 fn make_win_dist(
 138     rust_root: &Path,
 139     plat_root: &Path,
 140     target: TargetSelection,
 141     builder: &Builder<'_>,
 142 ) {
 143     //Ask gcc where it keeps its stuff
 144     let mut cmd = Command::new(builder.cc(target));
 145     cmd.arg("-print-search-dirs");
 146     let gcc_out = output(&mut cmd);
 147
 148     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 149     let mut lib_path = Vec::new();
 150
 151     for line in gcc_out.lines() {
 152         let idx = line.find(':').unwrap();
 153         let key = &line[..idx];
 154         let trim_chars: &[_] = &[' ', '='];
 155         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 156
 157         if key == "programs" {
 158             bin_path.extend(value);
 159         } else if key == "libraries" {
 160             lib_path.extend(value);
 161         }
 162     }
 163
 164     let compiler = if target == "i686-pc-windows-gnu" {
 165         "i686-w64-mingw32-gcc.exe"
 166     } else if target == "x86_64-pc-windows-gnu" {
 167         "x86_64-w64-mingw32-gcc.exe"
 168     } else {
 169         "gcc.exe"
 170     };
 171     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 172     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 173     if target.starts_with("i686-") {
 174         rustc_dlls.push("libgcc_s_dw2-1.dll");
 175     } else {
 176         rustc_dlls.push("libgcc_s_seh-1.dll");
 177     }
 178
 179     let target_libs = [
 180         //MinGW libs
 181         "libgcc.a",
 182         "libgcc_eh.a",
 183         "libgcc_s.a",
 184         "libm.a",
 185         "libmingw32.a",
 186         "libmingwex.a",
 187         "libstdc++.a",
 188         "libiconv.a",
 189         "libmoldname.a",
 190         "libpthread.a",
 191         //Windows import libs
 192         "libadvapi32.a",
 193         "libbcrypt.a",
 194         "libcomctl32.a",
 195         "libcomdlg32.a",
 196         "libcredui.a",
 197         "libcrypt32.a",
 198         "libdbghelp.a",
 199         "libgdi32.a",
 200         "libimagehlp.a",
 201         "libiphlpapi.a",
 202         "libkernel32.a",
 203         "libmsimg32.a",
 204         "libmsvcrt.a",
 205         "libodbc32.a",
 206         "libole32.a",
 207         "liboleaut32.a",
 208         "libopengl32.a",
 209         "libpsapi.a",
 210         "librpcrt4.a",
 211         "libsecur32.a",
 212         "libsetupapi.a",
 213         "libshell32.a",
 214         "libsynchronization.a",
 215         "libuser32.a",
 216         "libuserenv.a",
 217         "libuuid.a",
 218         "libwinhttp.a",
 219         "libwinmm.a",
 220         "libwinspool.a",
 221         "libws2_32.a",
 222         "libwsock32.a",
 223     ];
 224
 225     //Find mingw artifacts we want to bundle
 226     let target_tools = find_files(&target_tools, &bin_path);
 227     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 228     let target_libs = find_files(&target_libs, &lib_path);
 229
 230     // Copy runtime dlls next to rustc.exe
 231     let dist_bin_dir = rust_root.join("bin/");
 232     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 233     for src in rustc_dlls {
 234         builder.copy_to_folder(&src, &dist_bin_dir);
 235     }
 236
 237     //Copy platform tools to platform-specific bin directory
 238     let target_bin_dir = plat_root
 239         .join("lib")
 240         .join("rustlib")
 241         .join(target.triple)
 242         .join("bin")
 243         .join("self-contained");
 244     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 245     for src in target_tools {
 246         builder.copy_to_folder(&src, &target_bin_dir);
 247     }
 248
 249     // Warn windows-gnu users that the bundled GCC cannot compile C files
 250     builder.create(
 251         &target_bin_dir.join("GCC-WARNING.txt"),
 252         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 253          used as a linker. In order to be able to compile projects containing C code use \
 254          the GCC provided by MinGW or Cygwin.",
 255     );
 256
 257     //Copy platform libs to platform-specific lib directory
 258     let target_lib_dir = plat_root
 259         .join("lib")
 260         .join("rustlib")
 261         .join(target.triple)
 262         .join("lib")
 263         .join("self-contained");
 264     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 265     for src in target_libs {
 266         builder.copy_to_folder(&src, &target_lib_dir);
 267     }
 268 }
 269
 270 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 271 pub struct Mingw {
 272     pub host: TargetSelection,
 273 }
 274
 275 impl Step for Mingw {
 276     type Output = Option<GeneratedTarball>;
 277     const DEFAULT: bool = true;
 278
 279     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 280         run.never()
 281     }
 282
 283     fn make_run(run: RunConfig<'_>) {
 284         run.builder.ensure(Mingw { host: run.target });
 285     }
 286
 287     /// Builds the `rust-mingw` installer component.
 288     ///
 289     /// This contains all the bits and pieces to run the MinGW Windows targets
 290     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 291     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 292         let host = self.host;
 293         if !host.contains("pc-windows-gnu") {
 294             return None;
 295         }
 296
 297         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 298         tarball.set_product_name("Rust MinGW");
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 305
 306         Some(tarball.generate())
 307     }
 308 }
 309
 310 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 311 pub struct Rustc {
 312     pub compiler: Compiler,
 313 }
 314
 315 impl Step for Rustc {
 316     type Output = GeneratedTarball;
 317     const DEFAULT: bool = true;
 318     const ONLY_HOSTS: bool = true;
 319
 320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 321         run.path("src/librustc")
 322     }
 323
 324     fn make_run(run: RunConfig<'_>) {
 325         run.builder
 326             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 327     }
 328
 329     /// Creates the `rustc` installer component.
 330     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 331         let compiler = self.compiler;
 332         let host = self.compiler.host;
 333
 334         let tarball = Tarball::new(builder, "rustc", &host.triple);
 335
 336         // Prepare the rustc "image", what will actually end up getting installed
 337         prepare_image(builder, compiler, tarball.image_dir());
 338
 339         // On MinGW we've got a few runtime DLL dependencies that we need to
 340         // include. The first argument to this script is where to put these DLLs
 341         // (the image we're creating), and the second argument is a junk directory
 342         // to ignore all other MinGW stuff the script creates.
 343         //
 344         // On 32-bit MinGW we're always including a DLL which needs some extra
 345         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 346         // anything requiring us to distribute a license, but it's likely the
 347         // install will *also* include the rust-mingw package, which also needs
 348         // licenses, so to be safe we just include it here in all MinGW packages.
 349         if host.contains("pc-windows-gnu") {
 350             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 351             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 352         }
 353
 354         return tarball.generate();
 355
 356         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 357             let host = compiler.host;
 358             let src = builder.sysroot(compiler);
 359
 360             // Copy rustc/rustdoc binaries
 361             t!(fs::create_dir_all(image.join("bin")));
 362             builder.cp_r(&src.join("bin"), &image.join("bin"));
 363
 364             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 365
 366             let libdir_relative = builder.libdir_relative(compiler);
 367
 368             // Copy runtime DLLs needed by the compiler
 369             if libdir_relative.to_str() != Some("bin") {
 370                 let libdir = builder.rustc_libdir(compiler);
 371                 for entry in builder.read_dir(&libdir) {
 372                     let name = entry.file_name();
 373                     if let Some(s) = name.to_str() {
 374                         if is_dylib(s) {
 375                             // Don't use custom libdir here because ^lib/ will be resolved again
 376                             // with installer
 377                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 378                         }
 379                     }
 380                 }
 381             }
 382
 383             // Copy over the codegen backends
 384             let backends_src = builder.sysroot_codegen_backends(compiler);
 385             let backends_rel = backends_src
 386                 .strip_prefix(&src)
 387                 .unwrap()
 388                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 389                 .unwrap();
 390             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 391             let backends_dst = image.join("lib").join(&backends_rel);
 392
 393             t!(fs::create_dir_all(&backends_dst));
 394             builder.cp_r(&backends_src, &backends_dst);
 395
 396             // Copy libLLVM.so to the lib dir as well, if needed. While not
 397             // technically needed by rustc itself it's needed by lots of other
 398             // components like the llvm tools and LLD. LLD is included below and
 399             // tools/LLDB come later, so let's just throw it in the rustc
 400             // component for now.
 401             maybe_install_llvm_runtime(builder, host, image);
 402
 403             let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 404             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 405             t!(fs::create_dir_all(&dst_dir));
 406
 407             // Copy over lld if it's there
 408             if builder.config.lld_enabled {
 409                 let rust_lld = exe("rust-lld", compiler.host);
 410                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 411                 // for `-Z gcc-ld=lld`
 412                 let gcc_lld_dir = dst_dir.join("gcc-ld");
 413                 t!(fs::create_dir(&gcc_lld_dir));
 414                 builder.copy(&src_dir.join(&rust_lld), &gcc_lld_dir.join(exe("ld", compiler.host)));
 415                 builder
 416                     .copy(&src_dir.join(&rust_lld), &gcc_lld_dir.join(exe("ld64", compiler.host)));
 417             }
 418
 419             // Copy over llvm-dwp if it's there
 420             let exe = exe("rust-llvm-dwp", compiler.host);
 421             builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe));
 422
 423             // Man pages
 424             t!(fs::create_dir_all(image.join("share/man/man1")));
 425             let man_src = builder.src.join("src/doc/man");
 426             let man_dst = image.join("share/man/man1");
 427
 428             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 429             let time = env::var("SOURCE_DATE_EPOCH")
 430                 .map(|timestamp| {
 431                     let epoch = timestamp
 432                         .parse()
 433                         .map_err(|err| format!("could not parse SOURCE_DATE_EPOCH: {}", err))
 434                         .unwrap();
 435
 436                     time::at(Timespec::new(epoch, 0))
 437                 })
 438                 .unwrap_or_else(|_| time::now());
 439
 440             let month_year = t!(time::strftime("%B %Y", &time));
 441             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 442             // to hardlink, and we don't want to edit the source templates
 443             for file_entry in builder.read_dir(&man_src) {
 444                 let page_src = file_entry.path();
 445                 let page_dst = man_dst.join(file_entry.file_name());
 446                 t!(fs::copy(&page_src, &page_dst));
 447                 // template in month/year and version number
 448                 builder.replace_in_file(
 449                     &page_dst,
 450                     &[
 451                         ("<INSERT DATE HERE>", &month_year),
 452                         ("<INSERT VERSION HERE>", &builder.version),
 453                     ],
 454                 );
 455             }
 456
 457             // Debugger scripts
 458             builder
 459                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 460
 461             // Misc license info
 462             let cp = |file: &str| {
 463                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 464             };
 465             cp("COPYRIGHT");
 466             cp("LICENSE-APACHE");
 467             cp("LICENSE-MIT");
 468             cp("README.md");
 469         }
 470     }
 471 }
 472
 473 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 474 pub struct DebuggerScripts {
 475     pub sysroot: Interned<PathBuf>,
 476     pub host: TargetSelection,
 477 }
 478
 479 impl Step for DebuggerScripts {
 480     type Output = ();
 481
 482     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 483         run.path("src/lldb_batchmode.py")
 484     }
 485
 486     fn make_run(run: RunConfig<'_>) {
 487         run.builder.ensure(DebuggerScripts {
 488             sysroot: run
 489                 .builder
 490                 .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())),
 491             host: run.target,
 492         });
 493     }
 494
 495     /// Copies debugger scripts for `target` into the `sysroot` specified.
 496     fn run(self, builder: &Builder<'_>) {
 497         let host = self.host;
 498         let sysroot = self.sysroot;
 499         let dst = sysroot.join("lib/rustlib/etc");
 500         t!(fs::create_dir_all(&dst));
 501         let cp_debugger_script = |file: &str| {
 502             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 503         };
 504         if host.contains("windows-msvc") {
 505             // windbg debugger scripts
 506             builder.install(
 507                 &builder.src.join("src/etc/rust-windbg.cmd"),
 508                 &sysroot.join("bin"),
 509                 0o755,
 510             );
 511
 512             cp_debugger_script("natvis/intrinsic.natvis");
 513             cp_debugger_script("natvis/liballoc.natvis");
 514             cp_debugger_script("natvis/libcore.natvis");
 515             cp_debugger_script("natvis/libstd.natvis");
 516         } else {
 517             cp_debugger_script("rust_types.py");
 518
 519             // gdb debugger scripts
 520             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 521             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 522
 523             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 524             cp_debugger_script("gdb_lookup.py");
 525             cp_debugger_script("gdb_providers.py");
 526
 527             // lldb debugger scripts
 528             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 529
 530             cp_debugger_script("lldb_lookup.py");
 531             cp_debugger_script("lldb_providers.py");
 532             cp_debugger_script("lldb_commands")
 533         }
 534     }
 535 }
 536
 537 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 538     // The only true set of target libraries came from the build triple, so
 539     // let's reduce redundant work by only producing archives from that host.
 540     if compiler.host != builder.config.build {
 541         builder.info("\tskipping, not a build host");
 542         true
 543     } else {
 544         false
 545     }
 546 }
 547
 548 /// Copy stamped files into an image's `target/lib` directory.
 549 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 550     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 551     let self_contained_dst = dst.join("self-contained");
 552     t!(fs::create_dir_all(&dst));
 553     t!(fs::create_dir_all(&self_contained_dst));
 554     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 555         if dependency_type == DependencyType::TargetSelfContained {
 556             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 557         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 558             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 559         }
 560     }
 561 }
 562
 563 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 564 pub struct Std {
 565     pub compiler: Compiler,
 566     pub target: TargetSelection,
 567 }
 568
 569 impl Step for Std {
 570     type Output = Option<GeneratedTarball>;
 571     const DEFAULT: bool = true;
 572
 573     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 574         run.path("library/std")
 575     }
 576
 577     fn make_run(run: RunConfig<'_>) {
 578         run.builder.ensure(Std {
 579             compiler: run.builder.compiler_for(
 580                 run.builder.top_stage,
 581                 run.builder.config.build,
 582                 run.target,
 583             ),
 584             target: run.target,
 585         });
 586     }
 587
 588     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 589         let compiler = self.compiler;
 590         let target = self.target;
 591
 592         if skip_host_target_lib(builder, compiler) {
 593             return None;
 594         }
 595
 596         builder.ensure(compile::Std { compiler, target });
 597
 598         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 599         tarball.include_target_in_component_name(true);
 600
 601         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 602         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 603         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 604
 605         Some(tarball.generate())
 606     }
 607 }
 608
 609 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 610 pub struct RustcDev {
 611     pub compiler: Compiler,
 612     pub target: TargetSelection,
 613 }
 614
 615 impl Step for RustcDev {
 616     type Output = Option<GeneratedTarball>;
 617     const DEFAULT: bool = true;
 618     const ONLY_HOSTS: bool = true;
 619
 620     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 621         run.path("rustc-dev")
 622     }
 623
 624     fn make_run(run: RunConfig<'_>) {
 625         run.builder.ensure(RustcDev {
 626             compiler: run.builder.compiler_for(
 627                 run.builder.top_stage,
 628                 run.builder.config.build,
 629                 run.target,
 630             ),
 631             target: run.target,
 632         });
 633     }
 634
 635     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 636         let compiler = self.compiler;
 637         let target = self.target;
 638         if skip_host_target_lib(builder, compiler) {
 639             return None;
 640         }
 641
 642         builder.ensure(compile::Rustc { compiler, target });
 643
 644         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 645
 646         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 647         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 648         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 649
 650         let src_files = &["Cargo.lock"];
 651         // This is the reduced set of paths which will become the rustc-dev component
 652         // (essentially the compiler crates and all of their path dependencies).
 653         copy_src_dirs(
 654             builder,
 655             &builder.src,
 656             &["compiler"],
 657             &[],
 658             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 659         );
 660         // This particular crate is used as a build dependency of the above.
 661         copy_src_dirs(
 662             builder,
 663             &builder.src,
 664             &["src/build_helper"],
 665             &[],
 666             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 667         );
 668         for file in src_files {
 669             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 670         }
 671
 672         Some(tarball.generate())
 673     }
 674 }
 675
 676 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 677 pub struct Analysis {
 678     pub compiler: Compiler,
 679     pub target: TargetSelection,
 680 }
 681
 682 impl Step for Analysis {
 683     type Output = Option<GeneratedTarball>;
 684     const DEFAULT: bool = true;
 685
 686     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 687         let default = should_build_extended_tool(&run.builder, "analysis");
 688         run.path("analysis").default_condition(default)
 689     }
 690
 691     fn make_run(run: RunConfig<'_>) {
 692         run.builder.ensure(Analysis {
 693             // Find the actual compiler (handling the full bootstrap option) which
 694             // produced the save-analysis data because that data isn't copied
 695             // through the sysroot uplifting.
 696             compiler: run.builder.compiler_for(
 697                 run.builder.top_stage,
 698                 run.builder.config.build,
 699                 run.target,
 700             ),
 701             target: run.target,
 702         });
 703     }
 704
 705     /// Creates a tarball of save-analysis metadata, if available.
 706     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 707         let compiler = self.compiler;
 708         let target = self.target;
 709         if compiler.host != builder.config.build {
 710             return None;
 711         }
 712
 713         builder.ensure(compile::Std { compiler, target });
 714         let src = builder
 715             .stage_out(compiler, Mode::Std)
 716             .join(target.triple)
 717             .join(builder.cargo_dir())
 718             .join("deps")
 719             .join("save-analysis");
 720
 721         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 722         tarball.include_target_in_component_name(true);
 723         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 724         Some(tarball.generate())
 725     }
 726 }
 727
 728 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 729 /// `dst_dir`.
 730 fn copy_src_dirs(
 731     builder: &Builder<'_>,
 732     base: &Path,
 733     src_dirs: &[&str],
 734     exclude_dirs: &[&str],
 735     dst_dir: &Path,
 736 ) {
 737     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 738         let spath = match path.to_str() {
 739             Some(path) => path,
 740             None => return false,
 741         };
 742         if spath.ends_with('~') || spath.ends_with(".pyc") {
 743             return false;
 744         }
 745
 746         const LLVM_PROJECTS: &[&str] = &[
 747             "llvm-project/clang",
 748             "llvm-project\\clang",
 749             "llvm-project/libunwind",
 750             "llvm-project\\libunwind",
 751             "llvm-project/lld",
 752             "llvm-project\\lld",
 753             "llvm-project/lldb",
 754             "llvm-project\\lldb",
 755             "llvm-project/llvm",
 756             "llvm-project\\llvm",
 757             "llvm-project/compiler-rt",
 758             "llvm-project\\compiler-rt",
 759         ];
 760         if spath.contains("llvm-project")
 761             && !spath.ends_with("llvm-project")
 762             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 763         {
 764             return false;
 765         }
 766
 767         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 768         if LLVM_TEST.iter().any(|path| spath.contains(path))
 769             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 770         {
 771             return false;
 772         }
 773
 774         let full_path = Path::new(dir).join(path);
 775         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 776             return false;
 777         }
 778
 779         let excludes = [
 780             "CVS",
 781             "RCS",
 782             "SCCS",
 783             ".git",
 784             ".gitignore",
 785             ".gitmodules",
 786             ".gitattributes",
 787             ".cvsignore",
 788             ".svn",
 789             ".arch-ids",
 790             "{arch}",
 791             "=RELEASE-ID",
 792             "=meta-update",
 793             "=update",
 794             ".bzr",
 795             ".bzrignore",
 796             ".bzrtags",
 797             ".hg",
 798             ".hgignore",
 799             ".hgrags",
 800             "_darcs",
 801         ];
 802         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 803     }
 804
 805     // Copy the directories using our filter
 806     for item in src_dirs {
 807         let dst = &dst_dir.join(item);
 808         t!(fs::create_dir_all(dst));
 809         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 810     }
 811 }
 812
 813 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 814 pub struct Src;
 815
 816 impl Step for Src {
 817     /// The output path of the src installer tarball
 818     type Output = GeneratedTarball;
 819     const DEFAULT: bool = true;
 820     const ONLY_HOSTS: bool = true;
 821
 822     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 823         run.path("src")
 824     }
 825
 826     fn make_run(run: RunConfig<'_>) {
 827         run.builder.ensure(Src);
 828     }
 829
 830     /// Creates the `rust-src` installer component
 831     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 832         let tarball = Tarball::new_targetless(builder, "rust-src");
 833
 834         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 835         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 836         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 837         // and fix them...
 838         //
 839         // NOTE: if you update the paths here, you also should update the "virtual" path
 840         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 841         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 842
 843         let src_files = ["Cargo.lock"];
 844         // This is the reduced set of paths which will become the rust-src component
 845         // (essentially libstd and all of its path dependencies).
 846         copy_src_dirs(
 847             builder,
 848             &builder.src,
 849             &["library", "src/llvm-project/libunwind"],
 850             &[
 851                 // not needed and contains symlinks which rustup currently
 852                 // chokes on when unpacking.
 853                 "library/backtrace/crates",
 854             ],
 855             &dst_src,
 856         );
 857         for file in src_files.iter() {
 858             builder.copy(&builder.src.join(file), &dst_src.join(file));
 859         }
 860
 861         tarball.generate()
 862     }
 863 }
 864
 865 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 866 pub struct PlainSourceTarball;
 867
 868 impl Step for PlainSourceTarball {
 869     /// Produces the location of the tarball generated
 870     type Output = GeneratedTarball;
 871     const DEFAULT: bool = true;
 872     const ONLY_HOSTS: bool = true;
 873
 874     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 875         let builder = run.builder;
 876         run.path("src").default_condition(builder.config.rust_dist_src)
 877     }
 878
 879     fn make_run(run: RunConfig<'_>) {
 880         run.builder.ensure(PlainSourceTarball);
 881     }
 882
 883     /// Creates the plain source tarball
 884     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 885         let tarball = Tarball::new(builder, "rustc", "src");
 886         let plain_dst_src = tarball.image_dir();
 887
 888         // This is the set of root paths which will become part of the source package
 889         let src_files = [
 890             "COPYRIGHT",
 891             "LICENSE-APACHE",
 892             "LICENSE-MIT",
 893             "CONTRIBUTING.md",
 894             "README.md",
 895             "RELEASES.md",
 896             "configure",
 897             "x.py",
 898             "config.toml.example",
 899             "Cargo.toml",
 900             "Cargo.lock",
 901         ];
 902         let src_dirs = ["src", "compiler", "library"];
 903
 904         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 905
 906         // Copy the files normally
 907         for item in &src_files {
 908             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 909         }
 910
 911         // Create the version file
 912         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 913         if let Some(sha) = builder.rust_sha() {
 914             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 915         }
 916
 917         // If we're building from git sources, we need to vendor a complete distribution.
 918         if builder.rust_info.is_git() {
 919             // Vendor all Cargo dependencies
 920             let mut cmd = Command::new(&builder.initial_cargo);
 921             cmd.arg("vendor")
 922                 .arg("--sync")
 923                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 924                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 925                 .current_dir(&plain_dst_src);
 926             builder.run(&mut cmd);
 927         }
 928
 929         tarball.bare()
 930     }
 931 }
 932
 933 // We have to run a few shell scripts, which choke quite a bit on both `\`
 934 // characters and on `C:\` paths, so normalize both of them away.
 935 pub fn sanitize_sh(path: &Path) -> String {
 936     let path = path.to_str().unwrap().replace("\\", "/");
 937     return change_drive(unc_to_lfs(&path)).unwrap_or(path);
 938
 939     fn unc_to_lfs(s: &str) -> &str {
 940         s.strip_prefix("//?/").unwrap_or(s)
 941     }
 942
 943     fn change_drive(s: &str) -> Option<String> {
 944         let mut ch = s.chars();
 945         let drive = ch.next().unwrap_or('C');
 946         if ch.next() != Some(':') {
 947             return None;
 948         }
 949         if ch.next() != Some('/') {
 950             return None;
 951         }
 952         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 953     }
 954 }
 955
 956 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 957 pub struct Cargo {
 958     pub compiler: Compiler,
 959     pub target: TargetSelection,
 960 }
 961
 962 impl Step for Cargo {
 963     type Output = Option<GeneratedTarball>;
 964     const DEFAULT: bool = true;
 965     const ONLY_HOSTS: bool = true;
 966
 967     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 968         let default = should_build_extended_tool(&run.builder, "cargo");
 969         run.path("cargo").default_condition(default)
 970     }
 971
 972     fn make_run(run: RunConfig<'_>) {
 973         run.builder.ensure(Cargo {
 974             compiler: run.builder.compiler_for(
 975                 run.builder.top_stage,
 976                 run.builder.config.build,
 977                 run.target,
 978             ),
 979             target: run.target,
 980         });
 981     }
 982
 983     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 984         let compiler = self.compiler;
 985         let target = self.target;
 986
 987         let cargo = builder.ensure(tool::Cargo { compiler, target });
 988         let src = builder.src.join("src/tools/cargo");
 989         let etc = src.join("src/etc");
 990
 991         // Prepare the image directory
 992         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 993         tarball.set_overlay(OverlayKind::Cargo);
 994
 995         tarball.add_file(&cargo, "bin", 0o755);
 996         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
 997         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
 998         tarball.add_dir(etc.join("man"), "share/man/man1");
 999         tarball.add_legal_and_readme_to("share/doc/cargo");
1000
1001         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1002             let dirent = dirent.expect("read dir entry");
1003             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1004                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1005             }
1006         }
1007
1008         Some(tarball.generate())
1009     }
1010 }
1011
1012 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1013 pub struct Rls {
1014     pub compiler: Compiler,
1015     pub target: TargetSelection,
1016 }
1017
1018 impl Step for Rls {
1019     type Output = Option<GeneratedTarball>;
1020     const ONLY_HOSTS: bool = true;
1021     const DEFAULT: bool = true;
1022
1023     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1024         let default = should_build_extended_tool(&run.builder, "rls");
1025         run.path("rls").default_condition(default)
1026     }
1027
1028     fn make_run(run: RunConfig<'_>) {
1029         run.builder.ensure(Rls {
1030             compiler: run.builder.compiler_for(
1031                 run.builder.top_stage,
1032                 run.builder.config.build,
1033                 run.target,
1034             ),
1035             target: run.target,
1036         });
1037     }
1038
1039     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1040         let compiler = self.compiler;
1041         let target = self.target;
1042
1043         let rls = builder
1044             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1045             .or_else(|| {
1046                 missing_tool("RLS", builder.build.config.missing_tools);
1047                 None
1048             })?;
1049
1050         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1051         tarball.set_overlay(OverlayKind::RLS);
1052         tarball.is_preview(true);
1053         tarball.add_file(rls, "bin", 0o755);
1054         tarball.add_legal_and_readme_to("share/doc/rls");
1055         Some(tarball.generate())
1056     }
1057 }
1058
1059 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1060 pub struct RustAnalyzer {
1061     pub compiler: Compiler,
1062     pub target: TargetSelection,
1063 }
1064
1065 impl Step for RustAnalyzer {
1066     type Output = Option<GeneratedTarball>;
1067     const DEFAULT: bool = true;
1068     const ONLY_HOSTS: bool = true;
1069
1070     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1071         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1072         run.path("rust-analyzer").default_condition(default)
1073     }
1074
1075     fn make_run(run: RunConfig<'_>) {
1076         run.builder.ensure(RustAnalyzer {
1077             compiler: run.builder.compiler_for(
1078                 run.builder.top_stage,
1079                 run.builder.config.build,
1080                 run.target,
1081             ),
1082             target: run.target,
1083         });
1084     }
1085
1086     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1087         // This prevents rust-analyzer from being built for "dist" or "install"
1088         // on the stable/beta channels. It is a nightly-only tool and should
1089         // not be included.
1090         if !builder.build.unstable_features() {
1091             return None;
1092         }
1093         let compiler = self.compiler;
1094         let target = self.target;
1095
1096         if target.contains("riscv64") {
1097             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1098             // to build. See #74813 for details.
1099             return None;
1100         }
1101
1102         let rust_analyzer = builder
1103             .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
1104             .expect("rust-analyzer always builds");
1105
1106         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1107         tarball.set_overlay(OverlayKind::RustAnalyzer);
1108         tarball.is_preview(true);
1109         tarball.add_file(rust_analyzer, "bin", 0o755);
1110         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1111         Some(tarball.generate())
1112     }
1113 }
1114
1115 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1116 pub struct Clippy {
1117     pub compiler: Compiler,
1118     pub target: TargetSelection,
1119 }
1120
1121 impl Step for Clippy {
1122     type Output = Option<GeneratedTarball>;
1123     const DEFAULT: bool = true;
1124     const ONLY_HOSTS: bool = true;
1125
1126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1127         let default = should_build_extended_tool(&run.builder, "clippy");
1128         run.path("clippy").default_condition(default)
1129     }
1130
1131     fn make_run(run: RunConfig<'_>) {
1132         run.builder.ensure(Clippy {
1133             compiler: run.builder.compiler_for(
1134                 run.builder.top_stage,
1135                 run.builder.config.build,
1136                 run.target,
1137             ),
1138             target: run.target,
1139         });
1140     }
1141
1142     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1143         let compiler = self.compiler;
1144         let target = self.target;
1145
1146         // Prepare the image directory
1147         // We expect clippy to build, because we've exited this step above if tool
1148         // state for clippy isn't testing.
1149         let clippy = builder
1150             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1151             .expect("clippy expected to build - essential tool");
1152         let cargoclippy = builder
1153             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1154             .expect("clippy expected to build - essential tool");
1155
1156         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1157         tarball.set_overlay(OverlayKind::Clippy);
1158         tarball.is_preview(true);
1159         tarball.add_file(clippy, "bin", 0o755);
1160         tarball.add_file(cargoclippy, "bin", 0o755);
1161         tarball.add_legal_and_readme_to("share/doc/clippy");
1162         Some(tarball.generate())
1163     }
1164 }
1165
1166 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1167 pub struct Miri {
1168     pub compiler: Compiler,
1169     pub target: TargetSelection,
1170 }
1171
1172 impl Step for Miri {
1173     type Output = Option<GeneratedTarball>;
1174     const DEFAULT: bool = true;
1175     const ONLY_HOSTS: bool = true;
1176
1177     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1178         let default = should_build_extended_tool(&run.builder, "miri");
1179         run.path("miri").default_condition(default)
1180     }
1181
1182     fn make_run(run: RunConfig<'_>) {
1183         run.builder.ensure(Miri {
1184             compiler: run.builder.compiler_for(
1185                 run.builder.top_stage,
1186                 run.builder.config.build,
1187                 run.target,
1188             ),
1189             target: run.target,
1190         });
1191     }
1192
1193     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1194         // This prevents miri from being built for "dist" or "install"
1195         // on the stable/beta channels. It is a nightly-only tool and should
1196         // not be included.
1197         if !builder.build.unstable_features() {
1198             return None;
1199         }
1200         let compiler = self.compiler;
1201         let target = self.target;
1202
1203         let miri = builder
1204             .ensure(tool::Miri { compiler, target, extra_features: Vec::new() })
1205             .or_else(|| {
1206                 missing_tool("miri", builder.build.config.missing_tools);
1207                 None
1208             })?;
1209         let cargomiri = builder
1210             .ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })
1211             .or_else(|| {
1212                 missing_tool("cargo miri", builder.build.config.missing_tools);
1213                 None
1214             })?;
1215
1216         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1217         tarball.set_overlay(OverlayKind::Miri);
1218         tarball.is_preview(true);
1219         tarball.add_file(miri, "bin", 0o755);
1220         tarball.add_file(cargomiri, "bin", 0o755);
1221         tarball.add_legal_and_readme_to("share/doc/miri");
1222         Some(tarball.generate())
1223     }
1224 }
1225
1226 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1227 pub struct Rustfmt {
1228     pub compiler: Compiler,
1229     pub target: TargetSelection,
1230 }
1231
1232 impl Step for Rustfmt {
1233     type Output = Option<GeneratedTarball>;
1234     const DEFAULT: bool = true;
1235     const ONLY_HOSTS: bool = true;
1236
1237     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1238         let default = should_build_extended_tool(&run.builder, "rustfmt");
1239         run.path("rustfmt").default_condition(default)
1240     }
1241
1242     fn make_run(run: RunConfig<'_>) {
1243         run.builder.ensure(Rustfmt {
1244             compiler: run.builder.compiler_for(
1245                 run.builder.top_stage,
1246                 run.builder.config.build,
1247                 run.target,
1248             ),
1249             target: run.target,
1250         });
1251     }
1252
1253     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1254         let compiler = self.compiler;
1255         let target = self.target;
1256
1257         let rustfmt = builder
1258             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1259             .or_else(|| {
1260                 missing_tool("Rustfmt", builder.build.config.missing_tools);
1261                 None
1262             })?;
1263         let cargofmt = builder
1264             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1265             .or_else(|| {
1266                 missing_tool("Cargofmt", builder.build.config.missing_tools);
1267                 None
1268             })?;
1269
1270         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1271         tarball.set_overlay(OverlayKind::Rustfmt);
1272         tarball.is_preview(true);
1273         tarball.add_file(rustfmt, "bin", 0o755);
1274         tarball.add_file(cargofmt, "bin", 0o755);
1275         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1276         Some(tarball.generate())
1277     }
1278 }
1279
1280 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1281 pub struct RustDemangler {
1282     pub compiler: Compiler,
1283     pub target: TargetSelection,
1284 }
1285
1286 impl Step for RustDemangler {
1287     type Output = Option<GeneratedTarball>;
1288     const DEFAULT: bool = true;
1289     const ONLY_HOSTS: bool = true;
1290
1291     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1292         // While other tools use `should_build_extended_tool` to decide whether to be run by
1293         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1294         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1295         // we run the step by default when only `extended = true`, and decide whether to actually
1296         // run it or not later.
1297         let default = run.builder.config.extended;
1298         run.path("rust-demangler").default_condition(default)
1299     }
1300
1301     fn make_run(run: RunConfig<'_>) {
1302         run.builder.ensure(RustDemangler {
1303             compiler: run.builder.compiler_for(
1304                 run.builder.top_stage,
1305                 run.builder.config.build,
1306                 run.target,
1307             ),
1308             target: run.target,
1309         });
1310     }
1311
1312     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1313         let compiler = self.compiler;
1314         let target = self.target;
1315
1316         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1317         let condition = should_build_extended_tool(builder, "rust-demangler")
1318             || builder.config.profiler_enabled(target);
1319         if builder.config.extended && !condition {
1320             return None;
1321         }
1322
1323         let rust_demangler = builder
1324             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1325             .expect("rust-demangler expected to build - in-tree tool");
1326
1327         // Prepare the image directory
1328         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1329         tarball.set_overlay(OverlayKind::RustDemangler);
1330         tarball.is_preview(true);
1331         tarball.add_file(&rust_demangler, "bin", 0o755);
1332         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1333         Some(tarball.generate())
1334     }
1335 }
1336
1337 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1338 pub struct Extended {
1339     stage: u32,
1340     host: TargetSelection,
1341     target: TargetSelection,
1342 }
1343
1344 impl Step for Extended {
1345     type Output = ();
1346     const DEFAULT: bool = true;
1347     const ONLY_HOSTS: bool = true;
1348
1349     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1350         let builder = run.builder;
1351         run.path("extended").default_condition(builder.config.extended)
1352     }
1353
1354     fn make_run(run: RunConfig<'_>) {
1355         run.builder.ensure(Extended {
1356             stage: run.builder.top_stage,
1357             host: run.builder.config.build,
1358             target: run.target,
1359         });
1360     }
1361
1362     /// Creates a combined installer for the specified target in the provided stage.
1363     fn run(self, builder: &Builder<'_>) {
1364         let target = self.target;
1365         let stage = self.stage;
1366         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1367
1368         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1369
1370         let mut tarballs = Vec::new();
1371         let mut built_tools = HashSet::new();
1372         macro_rules! add_component {
1373             ($name:expr => $step:expr) => {
1374                 if let Some(tarball) = builder.ensure_if_default($step) {
1375                     tarballs.push(tarball);
1376                     built_tools.insert($name);
1377                 }
1378             };
1379         }
1380
1381         // When rust-std package split from rustc, we needed to ensure that during
1382         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1383         // the std files during uninstall. To do this ensure that rustc comes
1384         // before rust-std in the list below.
1385         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1386         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1387
1388         if target.contains("windows-gnu") {
1389             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1390         }
1391
1392         add_component!("rust-docs" => Docs { host: target });
1393         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1394         add_component!("cargo" => Cargo { compiler, target });
1395         add_component!("rustfmt" => Rustfmt { compiler, target });
1396         add_component!("rls" => Rls { compiler, target });
1397         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1398         add_component!("llvm-components" => LlvmTools { target });
1399         add_component!("clippy" => Clippy { compiler, target });
1400         add_component!("miri" => Miri { compiler, target });
1401         add_component!("analysis" => Analysis { compiler, target });
1402
1403         let etc = builder.src.join("src/etc/installer");
1404
1405         // Avoid producing tarballs during a dry run.
1406         if builder.config.dry_run {
1407             return;
1408         }
1409
1410         let tarball = Tarball::new(builder, "rust", &target.triple);
1411         let generated = tarball.combine(&tarballs);
1412
1413         let tmp = tmpdir(builder).join("combined-tarball");
1414         let work = generated.work_dir();
1415
1416         let mut license = String::new();
1417         license += &builder.read(&builder.src.join("COPYRIGHT"));
1418         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1419         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1420         license.push('\n');
1421         license.push('\n');
1422
1423         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1424         let mut rtf = rtf.to_string();
1425         rtf.push('\n');
1426         for line in license.lines() {
1427             rtf.push_str(line);
1428             rtf.push_str("\\line ");
1429         }
1430         rtf.push('}');
1431
1432         fn filter(contents: &str, marker: &str) -> String {
1433             let start = format!("tool-{}-start", marker);
1434             let end = format!("tool-{}-end", marker);
1435             let mut lines = Vec::new();
1436             let mut omitted = false;
1437             for line in contents.lines() {
1438                 if line.contains(&start) {
1439                     omitted = true;
1440                 } else if line.contains(&end) {
1441                     omitted = false;
1442                 } else if !omitted {
1443                     lines.push(line);
1444                 }
1445             }
1446
1447             lines.join("\n")
1448         }
1449
1450         let xform = |p: &Path| {
1451             let mut contents = t!(fs::read_to_string(p));
1452             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] {
1453                 if !built_tools.contains(tool) {
1454                     contents = filter(&contents, tool);
1455                 }
1456             }
1457             let ret = tmp.join(p.file_name().unwrap());
1458             t!(fs::write(&ret, &contents));
1459             ret
1460         };
1461
1462         if target.contains("apple-darwin") {
1463             builder.info("building pkg installer");
1464             let pkg = tmp.join("pkg");
1465             let _ = fs::remove_dir_all(&pkg);
1466
1467             let pkgbuild = |component: &str| {
1468                 let mut cmd = Command::new("pkgbuild");
1469                 cmd.arg("--identifier")
1470                     .arg(format!("org.rust-lang.{}", component))
1471                     .arg("--scripts")
1472                     .arg(pkg.join(component))
1473                     .arg("--nopayload")
1474                     .arg(pkg.join(component).with_extension("pkg"));
1475                 builder.run(&mut cmd);
1476             };
1477
1478             let prepare = |name: &str| {
1479                 builder.create_dir(&pkg.join(name));
1480                 builder.cp_r(
1481                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1482                     &pkg.join(name),
1483                 );
1484                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1485                 pkgbuild(name);
1486             };
1487             prepare("rustc");
1488             prepare("cargo");
1489             prepare("rust-docs");
1490             prepare("rust-std");
1491             prepare("rust-analysis");
1492             prepare("clippy");
1493             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1494                 if built_tools.contains(tool) {
1495                     prepare(tool);
1496                 }
1497             }
1498             // create an 'uninstall' package
1499             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1500             pkgbuild("uninstall");
1501
1502             builder.create_dir(&pkg.join("res"));
1503             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1504             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1505             let mut cmd = Command::new("productbuild");
1506             cmd.arg("--distribution")
1507                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1508                 .arg("--resources")
1509                 .arg(pkg.join("res"))
1510                 .arg(distdir(builder).join(format!(
1511                     "{}-{}.pkg",
1512                     pkgname(builder, "rust"),
1513                     target.triple
1514                 )))
1515                 .arg("--package-path")
1516                 .arg(&pkg);
1517             let _time = timeit(builder);
1518             builder.run(&mut cmd);
1519         }
1520
1521         if target.contains("windows") {
1522             let exe = tmp.join("exe");
1523             let _ = fs::remove_dir_all(&exe);
1524
1525             let prepare = |name: &str| {
1526                 builder.create_dir(&exe.join(name));
1527                 let dir = if name == "rust-std" || name == "rust-analysis" {
1528                     format!("{}-{}", name, target.triple)
1529                 } else if name == "rls" {
1530                     "rls-preview".to_string()
1531                 } else if name == "rust-analyzer" {
1532                     "rust-analyzer-preview".to_string()
1533                 } else if name == "clippy" {
1534                     "clippy-preview".to_string()
1535                 } else if name == "rust-demangler" {
1536                     "rust-demangler-preview".to_string()
1537                 } else if name == "miri" {
1538                     "miri-preview".to_string()
1539                 } else {
1540                     name.to_string()
1541                 };
1542                 builder.cp_r(
1543                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1544                     &exe.join(name),
1545                 );
1546                 builder.remove(&exe.join(name).join("manifest.in"));
1547             };
1548             prepare("rustc");
1549             prepare("cargo");
1550             prepare("rust-analysis");
1551             prepare("rust-docs");
1552             prepare("rust-std");
1553             prepare("clippy");
1554             for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] {
1555                 if built_tools.contains(tool) {
1556                     prepare(tool);
1557                 }
1558             }
1559             if target.contains("windows-gnu") {
1560                 prepare("rust-mingw");
1561             }
1562
1563             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1564
1565             // Generate msi installer
1566             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1567             let heat = wix.join("bin/heat.exe");
1568             let candle = wix.join("bin/candle.exe");
1569             let light = wix.join("bin/light.exe");
1570
1571             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1572             builder.run(
1573                 Command::new(&heat)
1574                     .current_dir(&exe)
1575                     .arg("dir")
1576                     .arg("rustc")
1577                     .args(&heat_flags)
1578                     .arg("-cg")
1579                     .arg("RustcGroup")
1580                     .arg("-dr")
1581                     .arg("Rustc")
1582                     .arg("-var")
1583                     .arg("var.RustcDir")
1584                     .arg("-out")
1585                     .arg(exe.join("RustcGroup.wxs")),
1586             );
1587             builder.run(
1588                 Command::new(&heat)
1589                     .current_dir(&exe)
1590                     .arg("dir")
1591                     .arg("rust-docs")
1592                     .args(&heat_flags)
1593                     .arg("-cg")
1594                     .arg("DocsGroup")
1595                     .arg("-dr")
1596                     .arg("Docs")
1597                     .arg("-var")
1598                     .arg("var.DocsDir")
1599                     .arg("-out")
1600                     .arg(exe.join("DocsGroup.wxs"))
1601                     .arg("-t")
1602                     .arg(etc.join("msi/squash-components.xsl")),
1603             );
1604             builder.run(
1605                 Command::new(&heat)
1606                     .current_dir(&exe)
1607                     .arg("dir")
1608                     .arg("cargo")
1609                     .args(&heat_flags)
1610                     .arg("-cg")
1611                     .arg("CargoGroup")
1612                     .arg("-dr")
1613                     .arg("Cargo")
1614                     .arg("-var")
1615                     .arg("var.CargoDir")
1616                     .arg("-out")
1617                     .arg(exe.join("CargoGroup.wxs"))
1618                     .arg("-t")
1619                     .arg(etc.join("msi/remove-duplicates.xsl")),
1620             );
1621             builder.run(
1622                 Command::new(&heat)
1623                     .current_dir(&exe)
1624                     .arg("dir")
1625                     .arg("rust-std")
1626                     .args(&heat_flags)
1627                     .arg("-cg")
1628                     .arg("StdGroup")
1629                     .arg("-dr")
1630                     .arg("Std")
1631                     .arg("-var")
1632                     .arg("var.StdDir")
1633                     .arg("-out")
1634                     .arg(exe.join("StdGroup.wxs")),
1635             );
1636             if built_tools.contains("rls") {
1637                 builder.run(
1638                     Command::new(&heat)
1639                         .current_dir(&exe)
1640                         .arg("dir")
1641                         .arg("rls")
1642                         .args(&heat_flags)
1643                         .arg("-cg")
1644                         .arg("RlsGroup")
1645                         .arg("-dr")
1646                         .arg("Rls")
1647                         .arg("-var")
1648                         .arg("var.RlsDir")
1649                         .arg("-out")
1650                         .arg(exe.join("RlsGroup.wxs"))
1651                         .arg("-t")
1652                         .arg(etc.join("msi/remove-duplicates.xsl")),
1653                 );
1654             }
1655             if built_tools.contains("rust-analyzer") {
1656                 builder.run(
1657                     Command::new(&heat)
1658                         .current_dir(&exe)
1659                         .arg("dir")
1660                         .arg("rust-analyzer")
1661                         .args(&heat_flags)
1662                         .arg("-cg")
1663                         .arg("RustAnalyzerGroup")
1664                         .arg("-dr")
1665                         .arg("RustAnalyzer")
1666                         .arg("-var")
1667                         .arg("var.RustAnalyzerDir")
1668                         .arg("-out")
1669                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1670                         .arg("-t")
1671                         .arg(etc.join("msi/remove-duplicates.xsl")),
1672                 );
1673             }
1674             builder.run(
1675                 Command::new(&heat)
1676                     .current_dir(&exe)
1677                     .arg("dir")
1678                     .arg("clippy")
1679                     .args(&heat_flags)
1680                     .arg("-cg")
1681                     .arg("ClippyGroup")
1682                     .arg("-dr")
1683                     .arg("Clippy")
1684                     .arg("-var")
1685                     .arg("var.ClippyDir")
1686                     .arg("-out")
1687                     .arg(exe.join("ClippyGroup.wxs"))
1688                     .arg("-t")
1689                     .arg(etc.join("msi/remove-duplicates.xsl")),
1690             );
1691             if built_tools.contains("rust-demangler") {
1692                 builder.run(
1693                     Command::new(&heat)
1694                         .current_dir(&exe)
1695                         .arg("dir")
1696                         .arg("rust-demangler")
1697                         .args(&heat_flags)
1698                         .arg("-cg")
1699                         .arg("RustDemanglerGroup")
1700                         .arg("-dr")
1701                         .arg("RustDemangler")
1702                         .arg("-var")
1703                         .arg("var.RustDemanglerDir")
1704                         .arg("-out")
1705                         .arg(exe.join("RustDemanglerGroup.wxs"))
1706                         .arg("-t")
1707                         .arg(etc.join("msi/remove-duplicates.xsl")),
1708                 );
1709             }
1710             if built_tools.contains("miri") {
1711                 builder.run(
1712                     Command::new(&heat)
1713                         .current_dir(&exe)
1714                         .arg("dir")
1715                         .arg("miri")
1716                         .args(&heat_flags)
1717                         .arg("-cg")
1718                         .arg("MiriGroup")
1719                         .arg("-dr")
1720                         .arg("Miri")
1721                         .arg("-var")
1722                         .arg("var.MiriDir")
1723                         .arg("-out")
1724                         .arg(exe.join("MiriGroup.wxs"))
1725                         .arg("-t")
1726                         .arg(etc.join("msi/remove-duplicates.xsl")),
1727                 );
1728             }
1729             builder.run(
1730                 Command::new(&heat)
1731                     .current_dir(&exe)
1732                     .arg("dir")
1733                     .arg("rust-analysis")
1734                     .args(&heat_flags)
1735                     .arg("-cg")
1736                     .arg("AnalysisGroup")
1737                     .arg("-dr")
1738                     .arg("Analysis")
1739                     .arg("-var")
1740                     .arg("var.AnalysisDir")
1741                     .arg("-out")
1742                     .arg(exe.join("AnalysisGroup.wxs"))
1743                     .arg("-t")
1744                     .arg(etc.join("msi/remove-duplicates.xsl")),
1745             );
1746             if target.contains("windows-gnu") {
1747                 builder.run(
1748                     Command::new(&heat)
1749                         .current_dir(&exe)
1750                         .arg("dir")
1751                         .arg("rust-mingw")
1752                         .args(&heat_flags)
1753                         .arg("-cg")
1754                         .arg("GccGroup")
1755                         .arg("-dr")
1756                         .arg("Gcc")
1757                         .arg("-var")
1758                         .arg("var.GccDir")
1759                         .arg("-out")
1760                         .arg(exe.join("GccGroup.wxs")),
1761                 );
1762             }
1763
1764             let candle = |input: &Path| {
1765                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1766                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1767                 let mut cmd = Command::new(&candle);
1768                 cmd.current_dir(&exe)
1769                     .arg("-nologo")
1770                     .arg("-dRustcDir=rustc")
1771                     .arg("-dDocsDir=rust-docs")
1772                     .arg("-dCargoDir=cargo")
1773                     .arg("-dStdDir=rust-std")
1774                     .arg("-dAnalysisDir=rust-analysis")
1775                     .arg("-dClippyDir=clippy")
1776                     .arg("-arch")
1777                     .arg(&arch)
1778                     .arg("-out")
1779                     .arg(&output)
1780                     .arg(&input);
1781                 add_env(builder, &mut cmd, target);
1782
1783                 if built_tools.contains("rust-demangler") {
1784                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1785                 }
1786                 if built_tools.contains("rls") {
1787                     cmd.arg("-dRlsDir=rls");
1788                 }
1789                 if built_tools.contains("rust-analyzer") {
1790                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1791                 }
1792                 if built_tools.contains("miri") {
1793                     cmd.arg("-dMiriDir=miri");
1794                 }
1795                 if target.contains("windows-gnu") {
1796                     cmd.arg("-dGccDir=rust-mingw");
1797                 }
1798                 builder.run(&mut cmd);
1799             };
1800             candle(&xform(&etc.join("msi/rust.wxs")));
1801             candle(&etc.join("msi/ui.wxs"));
1802             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1803             candle("RustcGroup.wxs".as_ref());
1804             candle("DocsGroup.wxs".as_ref());
1805             candle("CargoGroup.wxs".as_ref());
1806             candle("StdGroup.wxs".as_ref());
1807             candle("ClippyGroup.wxs".as_ref());
1808             if built_tools.contains("rust-demangler") {
1809                 candle("RustDemanglerGroup.wxs".as_ref());
1810             }
1811             if built_tools.contains("rls") {
1812                 candle("RlsGroup.wxs".as_ref());
1813             }
1814             if built_tools.contains("rust-analyzer") {
1815                 candle("RustAnalyzerGroup.wxs".as_ref());
1816             }
1817             if built_tools.contains("miri") {
1818                 candle("MiriGroup.wxs".as_ref());
1819             }
1820             candle("AnalysisGroup.wxs".as_ref());
1821
1822             if target.contains("windows-gnu") {
1823                 candle("GccGroup.wxs".as_ref());
1824             }
1825
1826             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1827             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1828             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1829
1830             builder.info(&format!("building `msi` installer with {:?}", light));
1831             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1832             let mut cmd = Command::new(&light);
1833             cmd.arg("-nologo")
1834                 .arg("-ext")
1835                 .arg("WixUIExtension")
1836                 .arg("-ext")
1837                 .arg("WixUtilExtension")
1838                 .arg("-out")
1839                 .arg(exe.join(&filename))
1840                 .arg("rust.wixobj")
1841                 .arg("ui.wixobj")
1842                 .arg("rustwelcomedlg.wixobj")
1843                 .arg("RustcGroup.wixobj")
1844                 .arg("DocsGroup.wixobj")
1845                 .arg("CargoGroup.wixobj")
1846                 .arg("StdGroup.wixobj")
1847                 .arg("AnalysisGroup.wixobj")
1848                 .arg("ClippyGroup.wixobj")
1849                 .current_dir(&exe);
1850
1851             if built_tools.contains("rls") {
1852                 cmd.arg("RlsGroup.wixobj");
1853             }
1854             if built_tools.contains("rust-analyzer") {
1855                 cmd.arg("RustAnalyzerGroup.wixobj");
1856             }
1857             if built_tools.contains("rust-demangler") {
1858                 cmd.arg("RustDemanglerGroup.wixobj");
1859             }
1860             if built_tools.contains("miri") {
1861                 cmd.arg("MiriGroup.wixobj");
1862             }
1863
1864             if target.contains("windows-gnu") {
1865                 cmd.arg("GccGroup.wixobj");
1866             }
1867             // ICE57 wrongly complains about the shortcuts
1868             cmd.arg("-sice:ICE57");
1869
1870             let _time = timeit(builder);
1871             builder.run(&mut cmd);
1872
1873             if !builder.config.dry_run {
1874                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1875             }
1876         }
1877     }
1878 }
1879
1880 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1881     let mut parts = builder.version.split('.');
1882     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1883         .env("CFG_RELEASE_NUM", &builder.version)
1884         .env("CFG_RELEASE", builder.rust_release())
1885         .env("CFG_VER_MAJOR", parts.next().unwrap())
1886         .env("CFG_VER_MINOR", parts.next().unwrap())
1887         .env("CFG_VER_PATCH", parts.next().unwrap())
1888         .env("CFG_VER_BUILD", "0") // just needed to build
1889         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1890         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1891         .env("CFG_BUILD", target.triple)
1892         .env("CFG_CHANNEL", &builder.config.channel);
1893
1894     if target.contains("windows-gnu") {
1895         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1896     } else {
1897         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1898     }
1899
1900     if target.contains("x86_64") {
1901         cmd.env("CFG_PLATFORM", "x64");
1902     } else {
1903         cmd.env("CFG_PLATFORM", "x86");
1904     }
1905 }
1906
1907 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1908 ///
1909
1910 /// Returns whether the files were actually copied.
1911 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1912     if let Some(config) = builder.config.target_config.get(&target) {
1913         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1914             // If the LLVM was externally provided, then we don't currently copy
1915             // artifacts into the sysroot. This is not necessarily the right
1916             // choice (in particular, it will require the LLVM dylib to be in
1917             // the linker's load path at runtime), but the common use case for
1918             // external LLVMs is distribution provided LLVMs, and in that case
1919             // they're usually in the standard search path (e.g., /usr/lib) and
1920             // copying them here is going to cause problems as we may end up
1921             // with the wrong files and isn't what distributions want.
1922             //
1923             // This behavior may be revisited in the future though.
1924             //
1925             // If the LLVM is coming from ourselves (just from CI) though, we
1926             // still want to install it, as it otherwise won't be available.
1927             return false;
1928         }
1929     }
1930
1931     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1932     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1933     // clear why this is the case, though. llvm-config will emit the versioned
1934     // paths and we don't want those in the sysroot (as we're expecting
1935     // unversioned paths).
1936     if target.contains("apple-darwin") && builder.config.llvm_link_shared {
1937         let src_libdir = builder.llvm_out(target).join("lib");
1938         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1939         if llvm_dylib_path.exists() {
1940             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1941         }
1942         !builder.config.dry_run
1943     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1944         let mut cmd = Command::new(llvm_config);
1945         cmd.arg("--libfiles");
1946         builder.verbose(&format!("running {:?}", cmd));
1947         let files = output(&mut cmd);
1948         let build_llvm_out = &builder.llvm_out(builder.config.build);
1949         let target_llvm_out = &builder.llvm_out(target);
1950         for file in files.trim_end().split(' ') {
1951             // If we're not using a custom LLVM, make sure we package for the target.
1952             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1953                 target_llvm_out.join(relative_path)
1954             } else {
1955                 PathBuf::from(file)
1956             };
1957             builder.install(&file, dst_libdir, 0o644);
1958         }
1959         !builder.config.dry_run
1960     } else {
1961         false
1962     }
1963 }
1964
1965 /// Maybe add libLLVM.so to the target lib-dir for linking.
1966 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1967     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1968     // We do not need to copy LLVM files into the sysroot if it is not
1969     // dynamically linked; it is already included into librustc_llvm
1970     // statically.
1971     if builder.config.llvm_link_shared {
1972         maybe_install_llvm(builder, target, &dst_libdir);
1973     }
1974 }
1975
1976 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1977 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1978     let dst_libdir =
1979         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1980     // We do not need to copy LLVM files into the sysroot if it is not
1981     // dynamically linked; it is already included into librustc_llvm
1982     // statically.
1983     if builder.config.llvm_link_shared {
1984         maybe_install_llvm(builder, target, &dst_libdir);
1985     }
1986 }
1987
1988 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1989 pub struct LlvmTools {
1990     pub target: TargetSelection,
1991 }
1992
1993 impl Step for LlvmTools {
1994     type Output = Option<GeneratedTarball>;
1995     const ONLY_HOSTS: bool = true;
1996     const DEFAULT: bool = true;
1997
1998     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1999         let default = should_build_extended_tool(&run.builder, "llvm-tools");
2000         run.path("llvm-tools").default_condition(default)
2001     }
2002
2003     fn make_run(run: RunConfig<'_>) {
2004         run.builder.ensure(LlvmTools { target: run.target });
2005     }
2006
2007     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2008         let target = self.target;
2009
2010         /* run only if llvm-config isn't used */
2011         if let Some(config) = builder.config.target_config.get(&target) {
2012             if let Some(ref _s) = config.llvm_config {
2013                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
2014                 return None;
2015             }
2016         }
2017
2018         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
2019         tarball.set_overlay(OverlayKind::LLVM);
2020         tarball.is_preview(true);
2021
2022         // Prepare the image directory
2023         let src_bindir = builder.llvm_out(target).join("bin");
2024         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
2025         for tool in LLVM_TOOLS {
2026             let exe = src_bindir.join(exe(tool, target));
2027             tarball.add_file(&exe, &dst_bindir, 0o755);
2028         }
2029
2030         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2031         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2032         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2033         // compiler libraries.
2034         maybe_install_llvm_target(builder, target, tarball.image_dir());
2035
2036         Some(tarball.generate())
2037     }
2038 }
2039
2040 // Tarball intended for internal consumption to ease rustc/std development.
2041 //
2042 // Should not be considered stable by end users.
2043 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2044 pub struct RustDev {
2045     pub target: TargetSelection,
2046 }
2047
2048 impl Step for RustDev {
2049     type Output = Option<GeneratedTarball>;
2050     const DEFAULT: bool = true;
2051     const ONLY_HOSTS: bool = true;
2052
2053     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2054         run.path("rust-dev")
2055     }
2056
2057     fn make_run(run: RunConfig<'_>) {
2058         run.builder.ensure(RustDev { target: run.target });
2059     }
2060
2061     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2062         let target = self.target;
2063
2064         /* run only if llvm-config isn't used */
2065         if let Some(config) = builder.config.target_config.get(&target) {
2066             if let Some(ref _s) = config.llvm_config {
2067                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2068                 return None;
2069             }
2070         }
2071
2072         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2073         tarball.set_overlay(OverlayKind::LLVM);
2074
2075         let src_bindir = builder.llvm_out(target).join("bin");
2076         for bin in &[
2077             "llvm-config",
2078             "llvm-ar",
2079             "llvm-objdump",
2080             "llvm-profdata",
2081             "llvm-bcanalyzer",
2082             "llvm-cov",
2083             "llvm-dwp",
2084         ] {
2085             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2086         }
2087         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2088
2089         // Copy the include directory as well; needed mostly to build
2090         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2091         // just broadly useful to be able to link against the bundled LLVM.
2092         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2093
2094         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2095         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2096         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2097         // compiler libraries.
2098         let dst_libdir = tarball.image_dir().join("lib");
2099         maybe_install_llvm(builder, target, &dst_libdir);
2100         let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
2101         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2102
2103         Some(tarball.generate())
2104     }
2105 }
2106
2107 /// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
2108 /// release process to avoid cloning the monorepo and building stuff.
2109 ///
2110 /// Should not be considered stable by end users.
2111 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2112 pub struct BuildManifest {
2113     pub target: TargetSelection,
2114 }
2115
2116 impl Step for BuildManifest {
2117     type Output = GeneratedTarball;
2118     const DEFAULT: bool = false;
2119     const ONLY_HOSTS: bool = true;
2120
2121     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2122         run.path("src/tools/build-manifest")
2123     }
2124
2125     fn make_run(run: RunConfig<'_>) {
2126         run.builder.ensure(BuildManifest { target: run.target });
2127     }
2128
2129     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2130         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2131
2132         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2133         tarball.add_file(&build_manifest, "bin", 0o755);
2134         tarball.generate()
2135     }
2136 }
2137
2138 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2139 ///
2140 /// Currently this is the PGO profile data.
2141 ///
2142 /// Should not be considered stable by end users.
2143 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2144 pub struct ReproducibleArtifacts {
2145     pub target: TargetSelection,
2146 }
2147
2148 impl Step for ReproducibleArtifacts {
2149     type Output = Option<GeneratedTarball>;
2150     const DEFAULT: bool = true;
2151     const ONLY_HOSTS: bool = true;
2152
2153     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2154         run.path("reproducible")
2155     }
2156
2157     fn make_run(run: RunConfig<'_>) {
2158         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2159     }
2160
2161     fn run(self, builder: &Builder<'_>) -> Self::Output {
2162         let mut added_anything = false;
2163         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2164         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2165             tarball.add_file(path, ".", 0o644);
2166             added_anything = true;
2167         }
2168         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2169             tarball.add_file(path, ".", 0o644);
2170             added_anything = true;
2171         }
2172         if added_anything { Some(tarball.generate()) } else { None }
2173     }
2174 }