src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::channel;
  20 use crate::compile;
  21 use crate::config::TargetSelection;
  22 use crate::doc::DocumentationFormat;
  23 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  24 use crate::tool::{self, Tool};
  25 use crate::util::{exe, is_dylib, output, t, timeit};
  26 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     format!("{}-{}", component, builder.rust_package_vers())
  30 }
  31
  32 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  33     builder.out.join("dist")
  34 }
  35
  36 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  37     builder.out.join("tmp/dist")
  38 }
  39
  40 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  41     if !builder.config.extended {
  42         return false;
  43     }
  44     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  45 }
  46
  47 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  48 pub struct Docs {
  49     pub host: TargetSelection,
  50 }
  51
  52 impl Step for Docs {
  53     type Output = Option<GeneratedTarball>;
  54     const DEFAULT: bool = true;
  55
  56     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  57         let default = run.builder.config.docs;
  58         run.alias("rust-docs").default_condition(default)
  59     }
  60
  61     fn make_run(run: RunConfig<'_>) {
  62         run.builder.ensure(Docs { host: run.target });
  63     }
  64
  65     /// Builds the `rust-docs` installer component.
  66     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  67         let host = self.host;
  68         builder.default_doc(&[]);
  69
  70         let dest = "share/doc/rust/html";
  71
  72         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  73         tarball.set_product_name("Rust Documentation");
  74         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  75         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  76         Some(tarball.generate())
  77     }
  78 }
  79
  80 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  81 pub struct JsonDocs {
  82     pub host: TargetSelection,
  83 }
  84
  85 impl Step for JsonDocs {
  86     type Output = Option<GeneratedTarball>;
  87     const DEFAULT: bool = true;
  88
  89     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  90         let default = run.builder.config.docs;
  91         run.alias("rust-docs-json").default_condition(default)
  92     }
  93
  94     fn make_run(run: RunConfig<'_>) {
  95         run.builder.ensure(JsonDocs { host: run.target });
  96     }
  97
  98     /// Builds the `rust-docs-json` installer component.
  99     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 100         let host = self.host;
 101         builder.ensure(crate::doc::Std {
 102             stage: builder.top_stage,
 103             target: host,
 104             format: DocumentationFormat::JSON,
 105         });
 106
 107         let dest = "share/doc/rust/json";
 108
 109         let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple);
 110         tarball.set_product_name("Rust Documentation In JSON Format");
 111         tarball.is_preview(true);
 112         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 113         Some(tarball.generate())
 114     }
 115 }
 116
 117 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 118 pub struct RustcDocs {
 119     pub host: TargetSelection,
 120 }
 121
 122 impl Step for RustcDocs {
 123     type Output = Option<GeneratedTarball>;
 124     const DEFAULT: bool = true;
 125
 126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 127         let builder = run.builder;
 128         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 129     }
 130
 131     fn make_run(run: RunConfig<'_>) {
 132         run.builder.ensure(RustcDocs { host: run.target });
 133     }
 134
 135     /// Builds the `rustc-docs` installer component.
 136     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 137         let host = self.host;
 138         builder.default_doc(&[]);
 139
 140         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 141         tarball.set_product_name("Rustc Documentation");
 142         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 143         Some(tarball.generate())
 144     }
 145 }
 146
 147 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 148     let mut found = Vec::with_capacity(files.len());
 149
 150     for file in files {
 151         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 152
 153         if let Some(file_path) = file_path {
 154             found.push(file_path);
 155         } else {
 156             panic!("Could not find '{}' in {:?}", file, path);
 157         }
 158     }
 159
 160     found
 161 }
 162
 163 fn make_win_dist(
 164     rust_root: &Path,
 165     plat_root: &Path,
 166     target: TargetSelection,
 167     builder: &Builder<'_>,
 168 ) {
 169     //Ask gcc where it keeps its stuff
 170     let mut cmd = Command::new(builder.cc(target));
 171     cmd.arg("-print-search-dirs");
 172     let gcc_out = output(&mut cmd);
 173
 174     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 175     let mut lib_path = Vec::new();
 176
 177     for line in gcc_out.lines() {
 178         let idx = line.find(':').unwrap();
 179         let key = &line[..idx];
 180         let trim_chars: &[_] = &[' ', '='];
 181         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 182
 183         if key == "programs" {
 184             bin_path.extend(value);
 185         } else if key == "libraries" {
 186             lib_path.extend(value);
 187         }
 188     }
 189
 190     let compiler = if target == "i686-pc-windows-gnu" {
 191         "i686-w64-mingw32-gcc.exe"
 192     } else if target == "x86_64-pc-windows-gnu" {
 193         "x86_64-w64-mingw32-gcc.exe"
 194     } else {
 195         "gcc.exe"
 196     };
 197     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 198     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 199     if target.starts_with("i686-") {
 200         rustc_dlls.push("libgcc_s_dw2-1.dll");
 201     } else {
 202         rustc_dlls.push("libgcc_s_seh-1.dll");
 203     }
 204
 205     let target_libs = [
 206         //MinGW libs
 207         "libgcc.a",
 208         "libgcc_eh.a",
 209         "libgcc_s.a",
 210         "libm.a",
 211         "libmingw32.a",
 212         "libmingwex.a",
 213         "libstdc++.a",
 214         "libiconv.a",
 215         "libmoldname.a",
 216         "libpthread.a",
 217         //Windows import libs
 218         "libadvapi32.a",
 219         "libbcrypt.a",
 220         "libcomctl32.a",
 221         "libcomdlg32.a",
 222         "libcredui.a",
 223         "libcrypt32.a",
 224         "libdbghelp.a",
 225         "libgdi32.a",
 226         "libimagehlp.a",
 227         "libiphlpapi.a",
 228         "libkernel32.a",
 229         "libmsimg32.a",
 230         "libmsvcrt.a",
 231         "libodbc32.a",
 232         "libole32.a",
 233         "liboleaut32.a",
 234         "libopengl32.a",
 235         "libpsapi.a",
 236         "librpcrt4.a",
 237         "libsecur32.a",
 238         "libsetupapi.a",
 239         "libshell32.a",
 240         "libsynchronization.a",
 241         "libuser32.a",
 242         "libuserenv.a",
 243         "libuuid.a",
 244         "libwinhttp.a",
 245         "libwinmm.a",
 246         "libwinspool.a",
 247         "libws2_32.a",
 248         "libwsock32.a",
 249     ];
 250
 251     //Find mingw artifacts we want to bundle
 252     let target_tools = find_files(&target_tools, &bin_path);
 253     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 254     let target_libs = find_files(&target_libs, &lib_path);
 255
 256     // Copy runtime dlls next to rustc.exe
 257     let dist_bin_dir = rust_root.join("bin/");
 258     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 259     for src in rustc_dlls {
 260         builder.copy_to_folder(&src, &dist_bin_dir);
 261     }
 262
 263     //Copy platform tools to platform-specific bin directory
 264     let target_bin_dir = plat_root
 265         .join("lib")
 266         .join("rustlib")
 267         .join(target.triple)
 268         .join("bin")
 269         .join("self-contained");
 270     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 271     for src in target_tools {
 272         builder.copy_to_folder(&src, &target_bin_dir);
 273     }
 274
 275     // Warn windows-gnu users that the bundled GCC cannot compile C files
 276     builder.create(
 277         &target_bin_dir.join("GCC-WARNING.txt"),
 278         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 279          used as a linker. In order to be able to compile projects containing C code use \
 280          the GCC provided by MinGW or Cygwin.",
 281     );
 282
 283     //Copy platform libs to platform-specific lib directory
 284     let target_lib_dir = plat_root
 285         .join("lib")
 286         .join("rustlib")
 287         .join(target.triple)
 288         .join("lib")
 289         .join("self-contained");
 290     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 291     for src in target_libs {
 292         builder.copy_to_folder(&src, &target_lib_dir);
 293     }
 294 }
 295
 296 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 297 pub struct Mingw {
 298     pub host: TargetSelection,
 299 }
 300
 301 impl Step for Mingw {
 302     type Output = Option<GeneratedTarball>;
 303     const DEFAULT: bool = true;
 304
 305     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 306         run.alias("rust-mingw")
 307     }
 308
 309     fn make_run(run: RunConfig<'_>) {
 310         run.builder.ensure(Mingw { host: run.target });
 311     }
 312
 313     /// Builds the `rust-mingw` installer component.
 314     ///
 315     /// This contains all the bits and pieces to run the MinGW Windows targets
 316     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 317     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 318         let host = self.host;
 319         if !host.ends_with("pc-windows-gnu") {
 320             return None;
 321         }
 322
 323         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 324         tarball.set_product_name("Rust MinGW");
 325
 326         // The first argument is a "temporary directory" which is just
 327         // thrown away (this contains the runtime DLLs included in the rustc package
 328         // above) and the second argument is where to place all the MinGW components
 329         // (which is what we want).
 330         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 331
 332         Some(tarball.generate())
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Rustc {
 338     pub compiler: Compiler,
 339 }
 340
 341 impl Step for Rustc {
 342     type Output = GeneratedTarball;
 343     const DEFAULT: bool = true;
 344     const ONLY_HOSTS: bool = true;
 345
 346     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 347         run.alias("rustc")
 348     }
 349
 350     fn make_run(run: RunConfig<'_>) {
 351         run.builder
 352             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 353     }
 354
 355     /// Creates the `rustc` installer component.
 356     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 357         let compiler = self.compiler;
 358         let host = self.compiler.host;
 359
 360         let tarball = Tarball::new(builder, "rustc", &host.triple);
 361
 362         // Prepare the rustc "image", what will actually end up getting installed
 363         prepare_image(builder, compiler, tarball.image_dir());
 364
 365         // On MinGW we've got a few runtime DLL dependencies that we need to
 366         // include. The first argument to this script is where to put these DLLs
 367         // (the image we're creating), and the second argument is a junk directory
 368         // to ignore all other MinGW stuff the script creates.
 369         //
 370         // On 32-bit MinGW we're always including a DLL which needs some extra
 371         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 372         // anything requiring us to distribute a license, but it's likely the
 373         // install will *also* include the rust-mingw package, which also needs
 374         // licenses, so to be safe we just include it here in all MinGW packages.
 375         if host.ends_with("pc-windows-gnu") {
 376             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 377             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 378         }
 379
 380         return tarball.generate();
 381
 382         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 383             let host = compiler.host;
 384             let src = builder.sysroot(compiler);
 385
 386             // Copy rustc/rustdoc binaries
 387             t!(fs::create_dir_all(image.join("bin")));
 388             builder.cp_r(&src.join("bin"), &image.join("bin"));
 389
 390             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 391
 392             let ra_proc_macro_srv = builder
 393                 .ensure(tool::RustAnalyzerProcMacroSrv {
 394                     compiler: builder.compiler_for(
 395                         compiler.stage,
 396                         builder.config.build,
 397                         compiler.host,
 398                     ),
 399                     target: compiler.host,
 400                 })
 401                 .expect("rust-analyzer-proc-macro-server always builds");
 402             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 403
 404             let libdir_relative = builder.libdir_relative(compiler);
 405
 406             // Copy runtime DLLs needed by the compiler
 407             if libdir_relative.to_str() != Some("bin") {
 408                 let libdir = builder.rustc_libdir(compiler);
 409                 for entry in builder.read_dir(&libdir) {
 410                     let name = entry.file_name();
 411                     if let Some(s) = name.to_str() {
 412                         if is_dylib(s) {
 413                             // Don't use custom libdir here because ^lib/ will be resolved again
 414                             // with installer
 415                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 416                         }
 417                     }
 418                 }
 419             }
 420
 421             // Copy over the codegen backends
 422             let backends_src = builder.sysroot_codegen_backends(compiler);
 423             let backends_rel = backends_src
 424                 .strip_prefix(&src)
 425                 .unwrap()
 426                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 427                 .unwrap();
 428             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 429             let backends_dst = image.join("lib").join(&backends_rel);
 430
 431             t!(fs::create_dir_all(&backends_dst));
 432             builder.cp_r(&backends_src, &backends_dst);
 433
 434             // Copy libLLVM.so to the lib dir as well, if needed. While not
 435             // technically needed by rustc itself it's needed by lots of other
 436             // components like the llvm tools and LLD. LLD is included below and
 437             // tools/LLDB come later, so let's just throw it in the rustc
 438             // component for now.
 439             maybe_install_llvm_runtime(builder, host, image);
 440
 441             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 442             t!(fs::create_dir_all(&dst_dir));
 443
 444             // Copy over lld if it's there
 445             if builder.config.lld_enabled {
 446                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 447                 let rust_lld = exe("rust-lld", compiler.host);
 448                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 449                 // for `-Z gcc-ld=lld`
 450                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 451                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 452                 t!(fs::create_dir(&gcc_lld_dst_dir));
 453                 for name in crate::LLD_FILE_NAMES {
 454                     let exe_name = exe(name, compiler.host);
 455                     builder
 456                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 457                 }
 458             }
 459
 460             // Man pages
 461             t!(fs::create_dir_all(image.join("share/man/man1")));
 462             let man_src = builder.src.join("src/doc/man");
 463             let man_dst = image.join("share/man/man1");
 464
 465             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 466             // to hardlink, and we don't want to edit the source templates
 467             for file_entry in builder.read_dir(&man_src) {
 468                 let page_src = file_entry.path();
 469                 let page_dst = man_dst.join(file_entry.file_name());
 470                 let src_text = t!(std::fs::read_to_string(&page_src));
 471                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 472                 t!(std::fs::write(&page_dst, &new_text));
 473                 t!(fs::copy(&page_src, &page_dst));
 474             }
 475
 476             // Debugger scripts
 477             builder
 478                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 479
 480             // Misc license info
 481             let cp = |file: &str| {
 482                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 483             };
 484             cp("COPYRIGHT");
 485             cp("LICENSE-APACHE");
 486             cp("LICENSE-MIT");
 487             cp("README.md");
 488         }
 489     }
 490 }
 491
 492 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 493 pub struct DebuggerScripts {
 494     pub sysroot: Interned<PathBuf>,
 495     pub host: TargetSelection,
 496 }
 497
 498 impl Step for DebuggerScripts {
 499     type Output = ();
 500
 501     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 502         run.never()
 503     }
 504
 505     /// Copies debugger scripts for `target` into the `sysroot` specified.
 506     fn run(self, builder: &Builder<'_>) {
 507         let host = self.host;
 508         let sysroot = self.sysroot;
 509         let dst = sysroot.join("lib/rustlib/etc");
 510         t!(fs::create_dir_all(&dst));
 511         let cp_debugger_script = |file: &str| {
 512             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 513         };
 514         if host.contains("windows-msvc") {
 515             // windbg debugger scripts
 516             builder.install(
 517                 &builder.src.join("src/etc/rust-windbg.cmd"),
 518                 &sysroot.join("bin"),
 519                 0o755,
 520             );
 521
 522             cp_debugger_script("natvis/intrinsic.natvis");
 523             cp_debugger_script("natvis/liballoc.natvis");
 524             cp_debugger_script("natvis/libcore.natvis");
 525             cp_debugger_script("natvis/libstd.natvis");
 526         } else {
 527             cp_debugger_script("rust_types.py");
 528
 529             // gdb debugger scripts
 530             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 531             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 532
 533             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 534             cp_debugger_script("gdb_lookup.py");
 535             cp_debugger_script("gdb_providers.py");
 536
 537             // lldb debugger scripts
 538             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 539
 540             cp_debugger_script("lldb_lookup.py");
 541             cp_debugger_script("lldb_providers.py");
 542             cp_debugger_script("lldb_commands")
 543         }
 544     }
 545 }
 546
 547 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 548     // The only true set of target libraries came from the build triple, so
 549     // let's reduce redundant work by only producing archives from that host.
 550     if compiler.host != builder.config.build {
 551         builder.info("\tskipping, not a build host");
 552         true
 553     } else {
 554         false
 555     }
 556 }
 557
 558 /// Copy stamped files into an image's `target/lib` directory.
 559 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 560     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 561     let self_contained_dst = dst.join("self-contained");
 562     t!(fs::create_dir_all(&dst));
 563     t!(fs::create_dir_all(&self_contained_dst));
 564     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 565         if dependency_type == DependencyType::TargetSelfContained {
 566             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 567         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 568             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 569         }
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct Std {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for Std {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582
 583     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 584         run.alias("rust-std")
 585     }
 586
 587     fn make_run(run: RunConfig<'_>) {
 588         run.builder.ensure(Std {
 589             compiler: run.builder.compiler_for(
 590                 run.builder.top_stage,
 591                 run.builder.config.build,
 592                 run.target,
 593             ),
 594             target: run.target,
 595         });
 596     }
 597
 598     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 599         let compiler = self.compiler;
 600         let target = self.target;
 601
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Std::new(compiler, target));
 607
 608         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 609         tarball.include_target_in_component_name(true);
 610
 611         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 612         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 613         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 614
 615         Some(tarball.generate())
 616     }
 617 }
 618
 619 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 620 pub struct RustcDev {
 621     pub compiler: Compiler,
 622     pub target: TargetSelection,
 623 }
 624
 625 impl Step for RustcDev {
 626     type Output = Option<GeneratedTarball>;
 627     const DEFAULT: bool = true;
 628     const ONLY_HOSTS: bool = true;
 629
 630     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 631         run.alias("rustc-dev")
 632     }
 633
 634     fn make_run(run: RunConfig<'_>) {
 635         run.builder.ensure(RustcDev {
 636             compiler: run.builder.compiler_for(
 637                 run.builder.top_stage,
 638                 run.builder.config.build,
 639                 run.target,
 640             ),
 641             target: run.target,
 642         });
 643     }
 644
 645     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 646         let compiler = self.compiler;
 647         let target = self.target;
 648         if skip_host_target_lib(builder, compiler) {
 649             return None;
 650         }
 651
 652         builder.ensure(compile::Rustc::new(compiler, target));
 653
 654         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 655
 656         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 657         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 658         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 659
 660         let src_files = &["Cargo.lock"];
 661         // This is the reduced set of paths which will become the rustc-dev component
 662         // (essentially the compiler crates and all of their path dependencies).
 663         copy_src_dirs(
 664             builder,
 665             &builder.src,
 666             &["compiler"],
 667             &[],
 668             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 669         );
 670         for file in src_files {
 671             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 672         }
 673
 674         Some(tarball.generate())
 675     }
 676 }
 677
 678 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 679 pub struct Analysis {
 680     pub compiler: Compiler,
 681     pub target: TargetSelection,
 682 }
 683
 684 impl Step for Analysis {
 685     type Output = Option<GeneratedTarball>;
 686     const DEFAULT: bool = true;
 687
 688     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 689         let default = should_build_extended_tool(&run.builder, "analysis");
 690         run.alias("rust-analysis").default_condition(default)
 691     }
 692
 693     fn make_run(run: RunConfig<'_>) {
 694         run.builder.ensure(Analysis {
 695             // Find the actual compiler (handling the full bootstrap option) which
 696             // produced the save-analysis data because that data isn't copied
 697             // through the sysroot uplifting.
 698             compiler: run.builder.compiler_for(
 699                 run.builder.top_stage,
 700                 run.builder.config.build,
 701                 run.target,
 702             ),
 703             target: run.target,
 704         });
 705     }
 706
 707     /// Creates a tarball of save-analysis metadata, if available.
 708     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 709         let compiler = self.compiler;
 710         let target = self.target;
 711         if compiler.host != builder.config.build {
 712             return None;
 713         }
 714
 715         builder.ensure(compile::Std::new(compiler, target));
 716         let src = builder
 717             .stage_out(compiler, Mode::Std)
 718             .join(target.triple)
 719             .join(builder.cargo_dir())
 720             .join("deps")
 721             .join("save-analysis");
 722
 723         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 724         tarball.include_target_in_component_name(true);
 725         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 726         Some(tarball.generate())
 727     }
 728 }
 729
 730 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 731 /// `dst_dir`.
 732 fn copy_src_dirs(
 733     builder: &Builder<'_>,
 734     base: &Path,
 735     src_dirs: &[&str],
 736     exclude_dirs: &[&str],
 737     dst_dir: &Path,
 738 ) {
 739     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 740         let spath = match path.to_str() {
 741             Some(path) => path,
 742             None => return false,
 743         };
 744         if spath.ends_with('~') || spath.ends_with(".pyc") {
 745             return false;
 746         }
 747
 748         const LLVM_PROJECTS: &[&str] = &[
 749             "llvm-project/clang",
 750             "llvm-project\\clang",
 751             "llvm-project/libunwind",
 752             "llvm-project\\libunwind",
 753             "llvm-project/lld",
 754             "llvm-project\\lld",
 755             "llvm-project/lldb",
 756             "llvm-project\\lldb",
 757             "llvm-project/llvm",
 758             "llvm-project\\llvm",
 759             "llvm-project/compiler-rt",
 760             "llvm-project\\compiler-rt",
 761             "llvm-project/cmake",
 762             "llvm-project\\cmake",
 763         ];
 764         if spath.contains("llvm-project")
 765             && !spath.ends_with("llvm-project")
 766             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 767         {
 768             return false;
 769         }
 770
 771         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 772         if LLVM_TEST.iter().any(|path| spath.contains(path))
 773             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 774         {
 775             return false;
 776         }
 777
 778         let full_path = Path::new(dir).join(path);
 779         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 780             return false;
 781         }
 782
 783         let excludes = [
 784             "CVS",
 785             "RCS",
 786             "SCCS",
 787             ".git",
 788             ".gitignore",
 789             ".gitmodules",
 790             ".gitattributes",
 791             ".cvsignore",
 792             ".svn",
 793             ".arch-ids",
 794             "{arch}",
 795             "=RELEASE-ID",
 796             "=meta-update",
 797             "=update",
 798             ".bzr",
 799             ".bzrignore",
 800             ".bzrtags",
 801             ".hg",
 802             ".hgignore",
 803             ".hgrags",
 804             "_darcs",
 805         ];
 806         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 807     }
 808
 809     // Copy the directories using our filter
 810     for item in src_dirs {
 811         let dst = &dst_dir.join(item);
 812         t!(fs::create_dir_all(dst));
 813         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 814     }
 815 }
 816
 817 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 818 pub struct Src;
 819
 820 impl Step for Src {
 821     /// The output path of the src installer tarball
 822     type Output = GeneratedTarball;
 823     const DEFAULT: bool = true;
 824     const ONLY_HOSTS: bool = true;
 825
 826     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 827         run.alias("rust-src")
 828     }
 829
 830     fn make_run(run: RunConfig<'_>) {
 831         run.builder.ensure(Src);
 832     }
 833
 834     /// Creates the `rust-src` installer component
 835     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 836         let tarball = Tarball::new_targetless(builder, "rust-src");
 837
 838         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 839         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 840         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 841         // and fix them...
 842         //
 843         // NOTE: if you update the paths here, you also should update the "virtual" path
 844         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 845         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 846
 847         let src_files = ["Cargo.lock"];
 848         // This is the reduced set of paths which will become the rust-src component
 849         // (essentially libstd and all of its path dependencies).
 850         copy_src_dirs(
 851             builder,
 852             &builder.src,
 853             &["library", "src/llvm-project/libunwind"],
 854             &[
 855                 // not needed and contains symlinks which rustup currently
 856                 // chokes on when unpacking.
 857                 "library/backtrace/crates",
 858                 // these are 30MB combined and aren't necessary for building
 859                 // the standard library.
 860                 "library/stdarch/Cargo.toml",
 861                 "library/stdarch/crates/stdarch-verify",
 862                 "library/stdarch/crates/intrinsic-test",
 863             ],
 864             &dst_src,
 865         );
 866         for file in src_files.iter() {
 867             builder.copy(&builder.src.join(file), &dst_src.join(file));
 868         }
 869
 870         tarball.generate()
 871     }
 872 }
 873
 874 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 875 pub struct PlainSourceTarball;
 876
 877 impl Step for PlainSourceTarball {
 878     /// Produces the location of the tarball generated
 879     type Output = GeneratedTarball;
 880     const DEFAULT: bool = true;
 881     const ONLY_HOSTS: bool = true;
 882
 883     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 884         let builder = run.builder;
 885         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 886     }
 887
 888     fn make_run(run: RunConfig<'_>) {
 889         run.builder.ensure(PlainSourceTarball);
 890     }
 891
 892     /// Creates the plain source tarball
 893     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 894         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 895         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 896         // It also contains symbolic links, unlike other any other dist tarball.
 897         // It's used for distros building rustc from source in a pre-vendored environment.
 898         let mut tarball = Tarball::new(builder, "rustc", "src");
 899         tarball.permit_symlinks(true);
 900         let plain_dst_src = tarball.image_dir();
 901
 902         // This is the set of root paths which will become part of the source package
 903         let src_files = [
 904             "COPYRIGHT",
 905             "LICENSE-APACHE",
 906             "LICENSE-MIT",
 907             "CONTRIBUTING.md",
 908             "README.md",
 909             "RELEASES.md",
 910             "configure",
 911             "x.py",
 912             "config.toml.example",
 913             "Cargo.toml",
 914             "Cargo.lock",
 915         ];
 916         let src_dirs = ["src", "compiler", "library"];
 917
 918         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 919
 920         // Copy the files normally
 921         for item in &src_files {
 922             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 923         }
 924
 925         // Create the version file
 926         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 927         if let Some(info) = builder.rust_info().info() {
 928             channel::write_commit_hash_file(&plain_dst_src, &info.sha);
 929             channel::write_commit_info_file(&plain_dst_src, info);
 930         }
 931
 932         // If we're building from git sources, we need to vendor a complete distribution.
 933         if builder.rust_info().is_managed_git_subrepository() {
 934             // Ensure we have the submodules checked out.
 935             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 936
 937             // Vendor all Cargo dependencies
 938             let mut cmd = Command::new(&builder.initial_cargo);
 939             cmd.arg("vendor")
 940                 .arg("--sync")
 941                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 942                 .arg("--sync")
 943                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 944                 .arg("--sync")
 945                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 946                 .current_dir(&plain_dst_src);
 947
 948             let config = if !builder.config.dry_run() {
 949                 t!(String::from_utf8(t!(cmd.output()).stdout))
 950             } else {
 951                 String::new()
 952             };
 953
 954             let cargo_config_dir = plain_dst_src.join(".cargo");
 955             builder.create_dir(&cargo_config_dir);
 956             builder.create(&cargo_config_dir.join("config.toml"), &config);
 957         }
 958
 959         tarball.bare()
 960     }
 961 }
 962
 963 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 964 pub struct Cargo {
 965     pub compiler: Compiler,
 966     pub target: TargetSelection,
 967 }
 968
 969 impl Step for Cargo {
 970     type Output = Option<GeneratedTarball>;
 971     const DEFAULT: bool = true;
 972     const ONLY_HOSTS: bool = true;
 973
 974     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 975         let default = should_build_extended_tool(&run.builder, "cargo");
 976         run.alias("cargo").default_condition(default)
 977     }
 978
 979     fn make_run(run: RunConfig<'_>) {
 980         run.builder.ensure(Cargo {
 981             compiler: run.builder.compiler_for(
 982                 run.builder.top_stage,
 983                 run.builder.config.build,
 984                 run.target,
 985             ),
 986             target: run.target,
 987         });
 988     }
 989
 990     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 991         let compiler = self.compiler;
 992         let target = self.target;
 993
 994         let cargo = builder.ensure(tool::Cargo { compiler, target });
 995         let src = builder.src.join("src/tools/cargo");
 996         let etc = src.join("src/etc");
 997
 998         // Prepare the image directory
 999         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
1000         tarball.set_overlay(OverlayKind::Cargo);
1001
1002         tarball.add_file(&cargo, "bin", 0o755);
1003         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
1004         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1005         tarball.add_dir(etc.join("man"), "share/man/man1");
1006         tarball.add_legal_and_readme_to("share/doc/cargo");
1007
1008         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1009             let dirent = dirent.expect("read dir entry");
1010             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1011                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1012             }
1013         }
1014
1015         Some(tarball.generate())
1016     }
1017 }
1018
1019 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1020 pub struct Rls {
1021     pub compiler: Compiler,
1022     pub target: TargetSelection,
1023 }
1024
1025 impl Step for Rls {
1026     type Output = Option<GeneratedTarball>;
1027     const ONLY_HOSTS: bool = true;
1028     const DEFAULT: bool = true;
1029
1030     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1031         let default = should_build_extended_tool(&run.builder, "rls");
1032         run.alias("rls").default_condition(default)
1033     }
1034
1035     fn make_run(run: RunConfig<'_>) {
1036         run.builder.ensure(Rls {
1037             compiler: run.builder.compiler_for(
1038                 run.builder.top_stage,
1039                 run.builder.config.build,
1040                 run.target,
1041             ),
1042             target: run.target,
1043         });
1044     }
1045
1046     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1047         let compiler = self.compiler;
1048         let target = self.target;
1049
1050         let rls = builder
1051             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1052             .expect("rls expected to build");
1053
1054         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1055         tarball.set_overlay(OverlayKind::RLS);
1056         tarball.is_preview(true);
1057         tarball.add_file(rls, "bin", 0o755);
1058         tarball.add_legal_and_readme_to("share/doc/rls");
1059         Some(tarball.generate())
1060     }
1061 }
1062
1063 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1064 pub struct RustAnalyzer {
1065     pub compiler: Compiler,
1066     pub target: TargetSelection,
1067 }
1068
1069 impl Step for RustAnalyzer {
1070     type Output = Option<GeneratedTarball>;
1071     const DEFAULT: bool = true;
1072     const ONLY_HOSTS: bool = true;
1073
1074     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1075         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1076         run.alias("rust-analyzer").default_condition(default)
1077     }
1078
1079     fn make_run(run: RunConfig<'_>) {
1080         run.builder.ensure(RustAnalyzer {
1081             compiler: run.builder.compiler_for(
1082                 run.builder.top_stage,
1083                 run.builder.config.build,
1084                 run.target,
1085             ),
1086             target: run.target,
1087         });
1088     }
1089
1090     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1091         let compiler = self.compiler;
1092         let target = self.target;
1093
1094         if target.contains("riscv64") {
1095             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1096             // to build. See #74813 for details.
1097             return None;
1098         }
1099
1100         let rust_analyzer = builder
1101             .ensure(tool::RustAnalyzer { compiler, target })
1102             .expect("rust-analyzer always builds");
1103
1104         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1105         tarball.set_overlay(OverlayKind::RustAnalyzer);
1106         tarball.is_preview(true);
1107         tarball.add_file(rust_analyzer, "bin", 0o755);
1108         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1109         Some(tarball.generate())
1110     }
1111 }
1112
1113 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1114 pub struct Clippy {
1115     pub compiler: Compiler,
1116     pub target: TargetSelection,
1117 }
1118
1119 impl Step for Clippy {
1120     type Output = Option<GeneratedTarball>;
1121     const DEFAULT: bool = true;
1122     const ONLY_HOSTS: bool = true;
1123
1124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1125         let default = should_build_extended_tool(&run.builder, "clippy");
1126         run.alias("clippy").default_condition(default)
1127     }
1128
1129     fn make_run(run: RunConfig<'_>) {
1130         run.builder.ensure(Clippy {
1131             compiler: run.builder.compiler_for(
1132                 run.builder.top_stage,
1133                 run.builder.config.build,
1134                 run.target,
1135             ),
1136             target: run.target,
1137         });
1138     }
1139
1140     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1141         let compiler = self.compiler;
1142         let target = self.target;
1143
1144         // Prepare the image directory
1145         // We expect clippy to build, because we've exited this step above if tool
1146         // state for clippy isn't testing.
1147         let clippy = builder
1148             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1149             .expect("clippy expected to build - essential tool");
1150         let cargoclippy = builder
1151             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1152             .expect("clippy expected to build - essential tool");
1153
1154         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1155         tarball.set_overlay(OverlayKind::Clippy);
1156         tarball.is_preview(true);
1157         tarball.add_file(clippy, "bin", 0o755);
1158         tarball.add_file(cargoclippy, "bin", 0o755);
1159         tarball.add_legal_and_readme_to("share/doc/clippy");
1160         Some(tarball.generate())
1161     }
1162 }
1163
1164 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1165 pub struct Miri {
1166     pub compiler: Compiler,
1167     pub target: TargetSelection,
1168 }
1169
1170 impl Step for Miri {
1171     type Output = Option<GeneratedTarball>;
1172     const DEFAULT: bool = true;
1173     const ONLY_HOSTS: bool = true;
1174
1175     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1176         let default = should_build_extended_tool(&run.builder, "miri");
1177         run.alias("miri").default_condition(default)
1178     }
1179
1180     fn make_run(run: RunConfig<'_>) {
1181         run.builder.ensure(Miri {
1182             compiler: run.builder.compiler_for(
1183                 run.builder.top_stage,
1184                 run.builder.config.build,
1185                 run.target,
1186             ),
1187             target: run.target,
1188         });
1189     }
1190
1191     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1192         // This prevents miri from being built for "dist" or "install"
1193         // on the stable/beta channels. It is a nightly-only tool and should
1194         // not be included.
1195         if !builder.build.unstable_features() {
1196             return None;
1197         }
1198         let compiler = self.compiler;
1199         let target = self.target;
1200
1201         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1202         let cargomiri =
1203             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1204
1205         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1206         tarball.set_overlay(OverlayKind::Miri);
1207         tarball.is_preview(true);
1208         tarball.add_file(miri, "bin", 0o755);
1209         tarball.add_file(cargomiri, "bin", 0o755);
1210         tarball.add_legal_and_readme_to("share/doc/miri");
1211         Some(tarball.generate())
1212     }
1213 }
1214
1215 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1216 pub struct Rustfmt {
1217     pub compiler: Compiler,
1218     pub target: TargetSelection,
1219 }
1220
1221 impl Step for Rustfmt {
1222     type Output = Option<GeneratedTarball>;
1223     const DEFAULT: bool = true;
1224     const ONLY_HOSTS: bool = true;
1225
1226     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1227         let default = should_build_extended_tool(&run.builder, "rustfmt");
1228         run.alias("rustfmt").default_condition(default)
1229     }
1230
1231     fn make_run(run: RunConfig<'_>) {
1232         run.builder.ensure(Rustfmt {
1233             compiler: run.builder.compiler_for(
1234                 run.builder.top_stage,
1235                 run.builder.config.build,
1236                 run.target,
1237             ),
1238             target: run.target,
1239         });
1240     }
1241
1242     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1243         let compiler = self.compiler;
1244         let target = self.target;
1245
1246         let rustfmt = builder
1247             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1248             .expect("rustfmt expected to build - essential tool");
1249         let cargofmt = builder
1250             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1251             .expect("cargo fmt expected to build - essential tool");
1252         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1253         tarball.set_overlay(OverlayKind::Rustfmt);
1254         tarball.is_preview(true);
1255         tarball.add_file(rustfmt, "bin", 0o755);
1256         tarball.add_file(cargofmt, "bin", 0o755);
1257         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1258         Some(tarball.generate())
1259     }
1260 }
1261
1262 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1263 pub struct RustDemangler {
1264     pub compiler: Compiler,
1265     pub target: TargetSelection,
1266 }
1267
1268 impl Step for RustDemangler {
1269     type Output = Option<GeneratedTarball>;
1270     const DEFAULT: bool = true;
1271     const ONLY_HOSTS: bool = true;
1272
1273     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1274         // While other tools use `should_build_extended_tool` to decide whether to be run by
1275         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1276         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1277         // we run the step by default when only `extended = true`, and decide whether to actually
1278         // run it or not later.
1279         let default = run.builder.config.extended;
1280         run.alias("rust-demangler").default_condition(default)
1281     }
1282
1283     fn make_run(run: RunConfig<'_>) {
1284         run.builder.ensure(RustDemangler {
1285             compiler: run.builder.compiler_for(
1286                 run.builder.top_stage,
1287                 run.builder.config.build,
1288                 run.target,
1289             ),
1290             target: run.target,
1291         });
1292     }
1293
1294     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1295         let compiler = self.compiler;
1296         let target = self.target;
1297
1298         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1299         let condition = should_build_extended_tool(builder, "rust-demangler")
1300             || builder.config.profiler_enabled(target);
1301         if builder.config.extended && !condition {
1302             return None;
1303         }
1304
1305         let rust_demangler = builder
1306             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1307             .expect("rust-demangler expected to build - in-tree tool");
1308
1309         // Prepare the image directory
1310         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1311         tarball.set_overlay(OverlayKind::RustDemangler);
1312         tarball.is_preview(true);
1313         tarball.add_file(&rust_demangler, "bin", 0o755);
1314         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1315         Some(tarball.generate())
1316     }
1317 }
1318
1319 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1320 pub struct Extended {
1321     stage: u32,
1322     host: TargetSelection,
1323     target: TargetSelection,
1324 }
1325
1326 impl Step for Extended {
1327     type Output = ();
1328     const DEFAULT: bool = true;
1329     const ONLY_HOSTS: bool = true;
1330
1331     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1332         let builder = run.builder;
1333         run.alias("extended").default_condition(builder.config.extended)
1334     }
1335
1336     fn make_run(run: RunConfig<'_>) {
1337         run.builder.ensure(Extended {
1338             stage: run.builder.top_stage,
1339             host: run.builder.config.build,
1340             target: run.target,
1341         });
1342     }
1343
1344     /// Creates a combined installer for the specified target in the provided stage.
1345     fn run(self, builder: &Builder<'_>) {
1346         let target = self.target;
1347         let stage = self.stage;
1348         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1349
1350         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1351
1352         let mut tarballs = Vec::new();
1353         let mut built_tools = HashSet::new();
1354         macro_rules! add_component {
1355             ($name:expr => $step:expr) => {
1356                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1357                     tarballs.push(tarball);
1358                     built_tools.insert($name);
1359                 }
1360             };
1361         }
1362
1363         // When rust-std package split from rustc, we needed to ensure that during
1364         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1365         // the std files during uninstall. To do this ensure that rustc comes
1366         // before rust-std in the list below.
1367         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1368         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1369
1370         if target.ends_with("windows-gnu") {
1371             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1372         }
1373
1374         add_component!("rust-docs" => Docs { host: target });
1375         add_component!("rust-json-docs" => JsonDocs { host: target });
1376         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1377         add_component!("cargo" => Cargo { compiler, target });
1378         add_component!("rustfmt" => Rustfmt { compiler, target });
1379         add_component!("rls" => Rls { compiler, target });
1380         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1381         add_component!("llvm-components" => LlvmTools { target });
1382         add_component!("clippy" => Clippy { compiler, target });
1383         add_component!("miri" => Miri { compiler, target });
1384         add_component!("analysis" => Analysis { compiler, target });
1385
1386         let etc = builder.src.join("src/etc/installer");
1387
1388         // Avoid producing tarballs during a dry run.
1389         if builder.config.dry_run() {
1390             return;
1391         }
1392
1393         let tarball = Tarball::new(builder, "rust", &target.triple);
1394         let generated = tarball.combine(&tarballs);
1395
1396         let tmp = tmpdir(builder).join("combined-tarball");
1397         let work = generated.work_dir();
1398
1399         let mut license = String::new();
1400         license += &builder.read(&builder.src.join("COPYRIGHT"));
1401         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1402         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1403         license.push('\n');
1404         license.push('\n');
1405
1406         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1407         let mut rtf = rtf.to_string();
1408         rtf.push('\n');
1409         for line in license.lines() {
1410             rtf.push_str(line);
1411             rtf.push_str("\\line ");
1412         }
1413         rtf.push('}');
1414
1415         fn filter(contents: &str, marker: &str) -> String {
1416             let start = format!("tool-{}-start", marker);
1417             let end = format!("tool-{}-end", marker);
1418             let mut lines = Vec::new();
1419             let mut omitted = false;
1420             for line in contents.lines() {
1421                 if line.contains(&start) {
1422                     omitted = true;
1423                 } else if line.contains(&end) {
1424                     omitted = false;
1425                 } else if !omitted {
1426                     lines.push(line);
1427                 }
1428             }
1429
1430             lines.join("\n")
1431         }
1432
1433         let xform = |p: &Path| {
1434             let mut contents = t!(fs::read_to_string(p));
1435             for tool in &["rust-demangler"] {
1436                 if !built_tools.contains(tool) {
1437                     contents = filter(&contents, tool);
1438                 }
1439             }
1440             let ret = tmp.join(p.file_name().unwrap());
1441             t!(fs::write(&ret, &contents));
1442             ret
1443         };
1444
1445         if target.contains("apple-darwin") {
1446             builder.info("building pkg installer");
1447             let pkg = tmp.join("pkg");
1448             let _ = fs::remove_dir_all(&pkg);
1449
1450             let pkgbuild = |component: &str| {
1451                 let mut cmd = Command::new("pkgbuild");
1452                 cmd.arg("--identifier")
1453                     .arg(format!("org.rust-lang.{}", component))
1454                     .arg("--scripts")
1455                     .arg(pkg.join(component))
1456                     .arg("--nopayload")
1457                     .arg(pkg.join(component).with_extension("pkg"));
1458                 builder.run(&mut cmd);
1459             };
1460
1461             let prepare = |name: &str| {
1462                 builder.create_dir(&pkg.join(name));
1463                 builder.cp_r(
1464                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1465                     &pkg.join(name),
1466                 );
1467                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1468                 pkgbuild(name);
1469             };
1470             prepare("rustc");
1471             prepare("cargo");
1472             prepare("rust-std");
1473             prepare("rust-analysis");
1474             prepare("clippy");
1475             prepare("miri");
1476             prepare("rust-analyzer");
1477             for tool in &["rust-docs", "rust-demangler"] {
1478                 if built_tools.contains(tool) {
1479                     prepare(tool);
1480                 }
1481             }
1482             // create an 'uninstall' package
1483             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1484             pkgbuild("uninstall");
1485
1486             builder.create_dir(&pkg.join("res"));
1487             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1488             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1489             let mut cmd = Command::new("productbuild");
1490             cmd.arg("--distribution")
1491                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1492                 .arg("--resources")
1493                 .arg(pkg.join("res"))
1494                 .arg(distdir(builder).join(format!(
1495                     "{}-{}.pkg",
1496                     pkgname(builder, "rust"),
1497                     target.triple
1498                 )))
1499                 .arg("--package-path")
1500                 .arg(&pkg);
1501             let _time = timeit(builder);
1502             builder.run(&mut cmd);
1503         }
1504
1505         if target.contains("windows") {
1506             let exe = tmp.join("exe");
1507             let _ = fs::remove_dir_all(&exe);
1508
1509             let prepare = |name: &str| {
1510                 builder.create_dir(&exe.join(name));
1511                 let dir = if name == "rust-std" || name == "rust-analysis" {
1512                     format!("{}-{}", name, target.triple)
1513                 } else if name == "rust-analyzer" {
1514                     "rust-analyzer-preview".to_string()
1515                 } else if name == "clippy" {
1516                     "clippy-preview".to_string()
1517                 } else if name == "rust-demangler" {
1518                     "rust-demangler-preview".to_string()
1519                 } else if name == "miri" {
1520                     "miri-preview".to_string()
1521                 } else {
1522                     name.to_string()
1523                 };
1524                 builder.cp_r(
1525                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1526                     &exe.join(name),
1527                 );
1528                 builder.remove(&exe.join(name).join("manifest.in"));
1529             };
1530             prepare("rustc");
1531             prepare("cargo");
1532             prepare("rust-analysis");
1533             prepare("rust-docs");
1534             prepare("rust-std");
1535             prepare("clippy");
1536             prepare("miri");
1537             prepare("rust-analyzer");
1538             for tool in &["rust-demangler"] {
1539                 if built_tools.contains(tool) {
1540                     prepare(tool);
1541                 }
1542             }
1543             if target.ends_with("windows-gnu") {
1544                 prepare("rust-mingw");
1545             }
1546
1547             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1548
1549             // Generate msi installer
1550             let wix_path = env::var_os("WIX")
1551                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1552             let wix = PathBuf::from(wix_path);
1553             let heat = wix.join("bin/heat.exe");
1554             let candle = wix.join("bin/candle.exe");
1555             let light = wix.join("bin/light.exe");
1556
1557             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1558             builder.run(
1559                 Command::new(&heat)
1560                     .current_dir(&exe)
1561                     .arg("dir")
1562                     .arg("rustc")
1563                     .args(&heat_flags)
1564                     .arg("-cg")
1565                     .arg("RustcGroup")
1566                     .arg("-dr")
1567                     .arg("Rustc")
1568                     .arg("-var")
1569                     .arg("var.RustcDir")
1570                     .arg("-out")
1571                     .arg(exe.join("RustcGroup.wxs")),
1572             );
1573             builder.run(
1574                 Command::new(&heat)
1575                     .current_dir(&exe)
1576                     .arg("dir")
1577                     .arg("rust-docs")
1578                     .args(&heat_flags)
1579                     .arg("-cg")
1580                     .arg("DocsGroup")
1581                     .arg("-dr")
1582                     .arg("Docs")
1583                     .arg("-var")
1584                     .arg("var.DocsDir")
1585                     .arg("-out")
1586                     .arg(exe.join("DocsGroup.wxs"))
1587                     .arg("-t")
1588                     .arg(etc.join("msi/squash-components.xsl")),
1589             );
1590             builder.run(
1591                 Command::new(&heat)
1592                     .current_dir(&exe)
1593                     .arg("dir")
1594                     .arg("cargo")
1595                     .args(&heat_flags)
1596                     .arg("-cg")
1597                     .arg("CargoGroup")
1598                     .arg("-dr")
1599                     .arg("Cargo")
1600                     .arg("-var")
1601                     .arg("var.CargoDir")
1602                     .arg("-out")
1603                     .arg(exe.join("CargoGroup.wxs"))
1604                     .arg("-t")
1605                     .arg(etc.join("msi/remove-duplicates.xsl")),
1606             );
1607             builder.run(
1608                 Command::new(&heat)
1609                     .current_dir(&exe)
1610                     .arg("dir")
1611                     .arg("rust-std")
1612                     .args(&heat_flags)
1613                     .arg("-cg")
1614                     .arg("StdGroup")
1615                     .arg("-dr")
1616                     .arg("Std")
1617                     .arg("-var")
1618                     .arg("var.StdDir")
1619                     .arg("-out")
1620                     .arg(exe.join("StdGroup.wxs")),
1621             );
1622             builder.run(
1623                 Command::new(&heat)
1624                     .current_dir(&exe)
1625                     .arg("dir")
1626                     .arg("rust-analyzer")
1627                     .args(&heat_flags)
1628                     .arg("-cg")
1629                     .arg("RustAnalyzerGroup")
1630                     .arg("-dr")
1631                     .arg("RustAnalyzer")
1632                     .arg("-var")
1633                     .arg("var.RustAnalyzerDir")
1634                     .arg("-out")
1635                     .arg(exe.join("RustAnalyzerGroup.wxs"))
1636                     .arg("-t")
1637                     .arg(etc.join("msi/remove-duplicates.xsl")),
1638             );
1639             builder.run(
1640                 Command::new(&heat)
1641                     .current_dir(&exe)
1642                     .arg("dir")
1643                     .arg("clippy")
1644                     .args(&heat_flags)
1645                     .arg("-cg")
1646                     .arg("ClippyGroup")
1647                     .arg("-dr")
1648                     .arg("Clippy")
1649                     .arg("-var")
1650                     .arg("var.ClippyDir")
1651                     .arg("-out")
1652                     .arg(exe.join("ClippyGroup.wxs"))
1653                     .arg("-t")
1654                     .arg(etc.join("msi/remove-duplicates.xsl")),
1655             );
1656             if built_tools.contains("rust-demangler") {
1657                 builder.run(
1658                     Command::new(&heat)
1659                         .current_dir(&exe)
1660                         .arg("dir")
1661                         .arg("rust-demangler")
1662                         .args(&heat_flags)
1663                         .arg("-cg")
1664                         .arg("RustDemanglerGroup")
1665                         .arg("-dr")
1666                         .arg("RustDemangler")
1667                         .arg("-var")
1668                         .arg("var.RustDemanglerDir")
1669                         .arg("-out")
1670                         .arg(exe.join("RustDemanglerGroup.wxs"))
1671                         .arg("-t")
1672                         .arg(etc.join("msi/remove-duplicates.xsl")),
1673                 );
1674             }
1675             builder.run(
1676                 Command::new(&heat)
1677                     .current_dir(&exe)
1678                     .arg("dir")
1679                     .arg("miri")
1680                     .args(&heat_flags)
1681                     .arg("-cg")
1682                     .arg("MiriGroup")
1683                     .arg("-dr")
1684                     .arg("Miri")
1685                     .arg("-var")
1686                     .arg("var.MiriDir")
1687                     .arg("-out")
1688                     .arg(exe.join("MiriGroup.wxs"))
1689                     .arg("-t")
1690                     .arg(etc.join("msi/remove-duplicates.xsl")),
1691             );
1692             builder.run(
1693                 Command::new(&heat)
1694                     .current_dir(&exe)
1695                     .arg("dir")
1696                     .arg("rust-analysis")
1697                     .args(&heat_flags)
1698                     .arg("-cg")
1699                     .arg("AnalysisGroup")
1700                     .arg("-dr")
1701                     .arg("Analysis")
1702                     .arg("-var")
1703                     .arg("var.AnalysisDir")
1704                     .arg("-out")
1705                     .arg(exe.join("AnalysisGroup.wxs"))
1706                     .arg("-t")
1707                     .arg(etc.join("msi/remove-duplicates.xsl")),
1708             );
1709             if target.ends_with("windows-gnu") {
1710                 builder.run(
1711                     Command::new(&heat)
1712                         .current_dir(&exe)
1713                         .arg("dir")
1714                         .arg("rust-mingw")
1715                         .args(&heat_flags)
1716                         .arg("-cg")
1717                         .arg("GccGroup")
1718                         .arg("-dr")
1719                         .arg("Gcc")
1720                         .arg("-var")
1721                         .arg("var.GccDir")
1722                         .arg("-out")
1723                         .arg(exe.join("GccGroup.wxs")),
1724                 );
1725             }
1726
1727             let candle = |input: &Path| {
1728                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1729                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1730                 let mut cmd = Command::new(&candle);
1731                 cmd.current_dir(&exe)
1732                     .arg("-nologo")
1733                     .arg("-dRustcDir=rustc")
1734                     .arg("-dDocsDir=rust-docs")
1735                     .arg("-dCargoDir=cargo")
1736                     .arg("-dStdDir=rust-std")
1737                     .arg("-dAnalysisDir=rust-analysis")
1738                     .arg("-dClippyDir=clippy")
1739                     .arg("-dMiriDir=miri")
1740                     .arg("-arch")
1741                     .arg(&arch)
1742                     .arg("-out")
1743                     .arg(&output)
1744                     .arg(&input);
1745                 add_env(builder, &mut cmd, target);
1746
1747                 if built_tools.contains("rust-demangler") {
1748                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1749                 }
1750                 if built_tools.contains("rust-analyzer") {
1751                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1752                 }
1753                 if target.ends_with("windows-gnu") {
1754                     cmd.arg("-dGccDir=rust-mingw");
1755                 }
1756                 builder.run(&mut cmd);
1757             };
1758             candle(&xform(&etc.join("msi/rust.wxs")));
1759             candle(&etc.join("msi/ui.wxs"));
1760             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1761             candle("RustcGroup.wxs".as_ref());
1762             candle("DocsGroup.wxs".as_ref());
1763             candle("CargoGroup.wxs".as_ref());
1764             candle("StdGroup.wxs".as_ref());
1765             candle("ClippyGroup.wxs".as_ref());
1766             candle("MiriGroup.wxs".as_ref());
1767             if built_tools.contains("rust-demangler") {
1768                 candle("RustDemanglerGroup.wxs".as_ref());
1769             }
1770             if built_tools.contains("rust-analyzer") {
1771                 candle("RustAnalyzerGroup.wxs".as_ref());
1772             }
1773             candle("AnalysisGroup.wxs".as_ref());
1774
1775             if target.ends_with("windows-gnu") {
1776                 candle("GccGroup.wxs".as_ref());
1777             }
1778
1779             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1780             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1781             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1782
1783             builder.info(&format!("building `msi` installer with {:?}", light));
1784             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1785             let mut cmd = Command::new(&light);
1786             cmd.arg("-nologo")
1787                 .arg("-ext")
1788                 .arg("WixUIExtension")
1789                 .arg("-ext")
1790                 .arg("WixUtilExtension")
1791                 .arg("-out")
1792                 .arg(exe.join(&filename))
1793                 .arg("rust.wixobj")
1794                 .arg("ui.wixobj")
1795                 .arg("rustwelcomedlg.wixobj")
1796                 .arg("RustcGroup.wixobj")
1797                 .arg("DocsGroup.wixobj")
1798                 .arg("CargoGroup.wixobj")
1799                 .arg("StdGroup.wixobj")
1800                 .arg("AnalysisGroup.wixobj")
1801                 .arg("ClippyGroup.wixobj")
1802                 .arg("MiriGroup.wixobj")
1803                 .current_dir(&exe);
1804
1805             if built_tools.contains("rust-analyzer") {
1806                 cmd.arg("RustAnalyzerGroup.wixobj");
1807             }
1808             if built_tools.contains("rust-demangler") {
1809                 cmd.arg("RustDemanglerGroup.wixobj");
1810             }
1811
1812             if target.ends_with("windows-gnu") {
1813                 cmd.arg("GccGroup.wixobj");
1814             }
1815             // ICE57 wrongly complains about the shortcuts
1816             cmd.arg("-sice:ICE57");
1817
1818             let _time = timeit(builder);
1819             builder.run(&mut cmd);
1820
1821             if !builder.config.dry_run() {
1822                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1823             }
1824         }
1825     }
1826 }
1827
1828 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1829     let mut parts = builder.version.split('.');
1830     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1831         .env("CFG_RELEASE_NUM", &builder.version)
1832         .env("CFG_RELEASE", builder.rust_release())
1833         .env("CFG_VER_MAJOR", parts.next().unwrap())
1834         .env("CFG_VER_MINOR", parts.next().unwrap())
1835         .env("CFG_VER_PATCH", parts.next().unwrap())
1836         .env("CFG_VER_BUILD", "0") // just needed to build
1837         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1838         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1839         .env("CFG_BUILD", target.triple)
1840         .env("CFG_CHANNEL", &builder.config.channel);
1841
1842     if target.contains("windows-gnullvm") {
1843         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1844     } else if target.contains("windows-gnu") {
1845         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1846     } else {
1847         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1848     }
1849 }
1850
1851 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1852 ///
1853 /// Returns whether the files were actually copied.
1854 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1855     if let Some(config) = builder.config.target_config.get(&target) {
1856         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1857             // If the LLVM was externally provided, then we don't currently copy
1858             // artifacts into the sysroot. This is not necessarily the right
1859             // choice (in particular, it will require the LLVM dylib to be in
1860             // the linker's load path at runtime), but the common use case for
1861             // external LLVMs is distribution provided LLVMs, and in that case
1862             // they're usually in the standard search path (e.g., /usr/lib) and
1863             // copying them here is going to cause problems as we may end up
1864             // with the wrong files and isn't what distributions want.
1865             //
1866             // This behavior may be revisited in the future though.
1867             //
1868             // If the LLVM is coming from ourselves (just from CI) though, we
1869             // still want to install it, as it otherwise won't be available.
1870             return false;
1871         }
1872     }
1873
1874     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1875     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1876     // clear why this is the case, though. llvm-config will emit the versioned
1877     // paths and we don't want those in the sysroot (as we're expecting
1878     // unversioned paths).
1879     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1880         let src_libdir = builder.llvm_out(target).join("lib");
1881         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1882         if llvm_dylib_path.exists() {
1883             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1884         }
1885         !builder.config.dry_run()
1886     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1887         let mut cmd = Command::new(llvm_config);
1888         cmd.arg("--libfiles");
1889         builder.verbose(&format!("running {:?}", cmd));
1890         let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) };
1891         let build_llvm_out = &builder.llvm_out(builder.config.build);
1892         let target_llvm_out = &builder.llvm_out(target);
1893         for file in files.trim_end().split(' ') {
1894             // If we're not using a custom LLVM, make sure we package for the target.
1895             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1896                 target_llvm_out.join(relative_path)
1897             } else {
1898                 PathBuf::from(file)
1899             };
1900             builder.install(&file, dst_libdir, 0o644);
1901         }
1902         !builder.config.dry_run()
1903     } else {
1904         false
1905     }
1906 }
1907
1908 /// Maybe add libLLVM.so to the target lib-dir for linking.
1909 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1910     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1911     // We do not need to copy LLVM files into the sysroot if it is not
1912     // dynamically linked; it is already included into librustc_llvm
1913     // statically.
1914     if builder.llvm_link_shared() {
1915         maybe_install_llvm(builder, target, &dst_libdir);
1916     }
1917 }
1918
1919 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1920 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1921     let dst_libdir =
1922         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1923     // We do not need to copy LLVM files into the sysroot if it is not
1924     // dynamically linked; it is already included into librustc_llvm
1925     // statically.
1926     if builder.llvm_link_shared() {
1927         maybe_install_llvm(builder, target, &dst_libdir);
1928     }
1929 }
1930
1931 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1932 pub struct LlvmTools {
1933     pub target: TargetSelection,
1934 }
1935
1936 impl Step for LlvmTools {
1937     type Output = Option<GeneratedTarball>;
1938     const ONLY_HOSTS: bool = true;
1939     const DEFAULT: bool = true;
1940
1941     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1942         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1943         // FIXME: allow using the names of the tools themselves?
1944         run.alias("llvm-tools").default_condition(default)
1945     }
1946
1947     fn make_run(run: RunConfig<'_>) {
1948         run.builder.ensure(LlvmTools { target: run.target });
1949     }
1950
1951     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1952         let target = self.target;
1953
1954         /* run only if llvm-config isn't used */
1955         if let Some(config) = builder.config.target_config.get(&target) {
1956             if let Some(ref _s) = config.llvm_config {
1957                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1958                 return None;
1959             }
1960         }
1961
1962         builder.ensure(crate::native::Llvm { target });
1963
1964         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1965         tarball.set_overlay(OverlayKind::LLVM);
1966         tarball.is_preview(true);
1967
1968         // Prepare the image directory
1969         let src_bindir = builder.llvm_out(target).join("bin");
1970         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1971         for tool in LLVM_TOOLS {
1972             let exe = src_bindir.join(exe(tool, target));
1973             tarball.add_file(&exe, &dst_bindir, 0o755);
1974         }
1975
1976         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1977         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1978         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1979         // compiler libraries.
1980         maybe_install_llvm_target(builder, target, tarball.image_dir());
1981
1982         Some(tarball.generate())
1983     }
1984 }
1985
1986 // Tarball intended for internal consumption to ease rustc/std development.
1987 //
1988 // Should not be considered stable by end users.
1989 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1990 pub struct RustDev {
1991     pub target: TargetSelection,
1992 }
1993
1994 impl Step for RustDev {
1995     type Output = Option<GeneratedTarball>;
1996     const DEFAULT: bool = true;
1997     const ONLY_HOSTS: bool = true;
1998
1999     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2000         run.alias("rust-dev")
2001     }
2002
2003     fn make_run(run: RunConfig<'_>) {
2004         run.builder.ensure(RustDev { target: run.target });
2005     }
2006
2007     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2008         let target = self.target;
2009
2010         /* run only if llvm-config isn't used */
2011         if let Some(config) = builder.config.target_config.get(&target) {
2012             if let Some(ref _s) = config.llvm_config {
2013                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2014                 return None;
2015             }
2016         }
2017
2018         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2019         tarball.set_overlay(OverlayKind::LLVM);
2020
2021         builder.ensure(crate::native::Llvm { target });
2022
2023         let src_bindir = builder.llvm_out(target).join("bin");
2024         // If updating this list, you likely want to change
2025         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2026         // will not pick up the extra file until LLVM gets bumped.
2027         for bin in &[
2028             "llvm-config",
2029             "llvm-ar",
2030             "llvm-objdump",
2031             "llvm-profdata",
2032             "llvm-bcanalyzer",
2033             "llvm-cov",
2034             "llvm-dwp",
2035             "llvm-nm",
2036             "llvm-dwarfdump",
2037             "llvm-dis",
2038             "llvm-tblgen",
2039         ] {
2040             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2041         }
2042
2043         // We don't build LLD on some platforms, so only add it if it exists
2044         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2045         if lld_path.exists() {
2046             tarball.add_file(lld_path, "bin", 0o755);
2047         }
2048
2049         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2050
2051         // Copy the include directory as well; needed mostly to build
2052         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2053         // just broadly useful to be able to link against the bundled LLVM.
2054         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2055
2056         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2057         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2058         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2059         // compiler libraries.
2060         let dst_libdir = tarball.image_dir().join("lib");
2061         maybe_install_llvm(builder, target, &dst_libdir);
2062         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2063         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2064
2065         Some(tarball.generate())
2066     }
2067 }
2068
2069 // Tarball intended for internal consumption to ease rustc/std development.
2070 //
2071 // Should not be considered stable by end users.
2072 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2073 pub struct Bootstrap {
2074     pub target: TargetSelection,
2075 }
2076
2077 impl Step for Bootstrap {
2078     type Output = Option<GeneratedTarball>;
2079     const DEFAULT: bool = false;
2080     const ONLY_HOSTS: bool = true;
2081
2082     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2083         run.alias("bootstrap")
2084     }
2085
2086     fn make_run(run: RunConfig<'_>) {
2087         run.builder.ensure(Bootstrap { target: run.target });
2088     }
2089
2090     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2091         let target = self.target;
2092
2093         let tarball = Tarball::new(builder, "bootstrap", &target.triple);
2094
2095         let bootstrap_outdir = &builder.bootstrap_out;
2096         for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
2097             tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
2098         }
2099
2100         Some(tarball.generate())
2101     }
2102 }
2103
2104 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2105 /// release process to avoid cloning the monorepo and building stuff.
2106 ///
2107 /// Should not be considered stable by end users.
2108 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2109 pub struct BuildManifest {
2110     pub target: TargetSelection,
2111 }
2112
2113 impl Step for BuildManifest {
2114     type Output = GeneratedTarball;
2115     const DEFAULT: bool = false;
2116     const ONLY_HOSTS: bool = true;
2117
2118     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2119         run.alias("build-manifest")
2120     }
2121
2122     fn make_run(run: RunConfig<'_>) {
2123         run.builder.ensure(BuildManifest { target: run.target });
2124     }
2125
2126     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2127         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2128
2129         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2130         tarball.add_file(&build_manifest, "bin", 0o755);
2131         tarball.generate()
2132     }
2133 }
2134
2135 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2136 ///
2137 /// Currently this is the PGO profile data.
2138 ///
2139 /// Should not be considered stable by end users.
2140 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2141 pub struct ReproducibleArtifacts {
2142     pub target: TargetSelection,
2143 }
2144
2145 impl Step for ReproducibleArtifacts {
2146     type Output = Option<GeneratedTarball>;
2147     const DEFAULT: bool = true;
2148     const ONLY_HOSTS: bool = true;
2149
2150     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2151         run.alias("reproducible-artifacts")
2152     }
2153
2154     fn make_run(run: RunConfig<'_>) {
2155         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2156     }
2157
2158     fn run(self, builder: &Builder<'_>) -> Self::Output {
2159         let mut added_anything = false;
2160         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2161         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2162             tarball.add_file(path, ".", 0o644);
2163             added_anything = true;
2164         }
2165         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2166             tarball.add_file(path, ".", 0o644);
2167             added_anything = true;
2168         }
2169         if let Some(path) = builder.config.llvm_bolt_profile_use.as_ref() {
2170             tarball.add_file(path, ".", 0o644);
2171             added_anything = true;
2172         }
2173         if added_anything { Some(tarball.generate()) } else { None }
2174     }
2175 }