src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::collections::HashSet;
  12 use std::env;
  13 use std::fs;
  14 use std::path::{Path, PathBuf};
  15 use std::process::Command;
  16
  17 use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
  18 use crate::cache::{Interned, INTERNER};
  19 use crate::compile;
  20 use crate::config::TargetSelection;
  21 use crate::tarball::{GeneratedTarball, OverlayKind, Tarball};
  22 use crate::tool::{self, Tool};
  23 use crate::util::{exe, is_dylib, output, t, timeit};
  24 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
  25
  26 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  27     format!("{}-{}", component, builder.rust_package_vers())
  28 }
  29
  30 pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
  31     builder.out.join("dist")
  32 }
  33
  34 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  35     builder.out.join("tmp/dist")
  36 }
  37
  38 fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool {
  39     if !builder.config.extended {
  40         return false;
  41     }
  42     builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool))
  43 }
  44
  45 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  46 pub struct Docs {
  47     pub host: TargetSelection,
  48 }
  49
  50 impl Step for Docs {
  51     type Output = Option<GeneratedTarball>;
  52     const DEFAULT: bool = true;
  53
  54     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  55         let default = run.builder.config.docs;
  56         run.alias("rust-docs").default_condition(default)
  57     }
  58
  59     fn make_run(run: RunConfig<'_>) {
  60         run.builder.ensure(Docs { host: run.target });
  61     }
  62
  63     /// Builds the `rust-docs` installer component.
  64     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  65         let host = self.host;
  66         builder.default_doc(&[]);
  67
  68         let dest = "share/doc/rust/html";
  69
  70         let mut tarball = Tarball::new(builder, "rust-docs", &host.triple);
  71         tarball.set_product_name("Rust Documentation");
  72         tarball.add_bulk_dir(&builder.doc_out(host), dest);
  73         tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644);
  74         Some(tarball.generate())
  75     }
  76 }
  77
  78 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  79 pub struct JsonDocs {
  80     pub host: TargetSelection,
  81 }
  82
  83 impl Step for JsonDocs {
  84     type Output = Option<GeneratedTarball>;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  88         let default = run.builder.config.docs;
  89         run.alias("rust-json-docs").default_condition(default)
  90     }
  91
  92     fn make_run(run: RunConfig<'_>) {
  93         run.builder.ensure(JsonDocs { host: run.target });
  94     }
  95
  96     /// Builds the `rust-json-docs` installer component.
  97     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
  98         // This prevents JSON docs from being built for "dist" or "install"
  99         // on the stable/beta channels. The JSON format is not stable yet and
 100         // should not be included in stable/beta toolchains.
 101         if !builder.build.unstable_features() {
 102             return None;
 103         }
 104
 105         let host = self.host;
 106         builder.ensure(crate::doc::JsonStd { stage: builder.top_stage, target: host });
 107
 108         let dest = "share/doc/rust/json";
 109
 110         let mut tarball = Tarball::new(builder, "rust-json-docs", &host.triple);
 111         tarball.set_product_name("Rust Documentation In JSON Format");
 112         tarball.add_bulk_dir(&builder.json_doc_out(host), dest);
 113         Some(tarball.generate())
 114     }
 115 }
 116
 117 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 118 pub struct RustcDocs {
 119     pub host: TargetSelection,
 120 }
 121
 122 impl Step for RustcDocs {
 123     type Output = Option<GeneratedTarball>;
 124     const DEFAULT: bool = true;
 125
 126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 127         let builder = run.builder;
 128         run.alias("rustc-docs").default_condition(builder.config.compiler_docs)
 129     }
 130
 131     fn make_run(run: RunConfig<'_>) {
 132         run.builder.ensure(RustcDocs { host: run.target });
 133     }
 134
 135     /// Builds the `rustc-docs` installer component.
 136     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 137         let host = self.host;
 138         builder.default_doc(&[]);
 139
 140         let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple);
 141         tarball.set_product_name("Rustc Documentation");
 142         tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc");
 143         Some(tarball.generate())
 144     }
 145 }
 146
 147 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 148     let mut found = Vec::with_capacity(files.len());
 149
 150     for file in files {
 151         let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists());
 152
 153         if let Some(file_path) = file_path {
 154             found.push(file_path);
 155         } else {
 156             panic!("Could not find '{}' in {:?}", file, path);
 157         }
 158     }
 159
 160     found
 161 }
 162
 163 fn make_win_dist(
 164     rust_root: &Path,
 165     plat_root: &Path,
 166     target: TargetSelection,
 167     builder: &Builder<'_>,
 168 ) {
 169     //Ask gcc where it keeps its stuff
 170     let mut cmd = Command::new(builder.cc(target));
 171     cmd.arg("-print-search-dirs");
 172     let gcc_out = output(&mut cmd);
 173
 174     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 175     let mut lib_path = Vec::new();
 176
 177     for line in gcc_out.lines() {
 178         let idx = line.find(':').unwrap();
 179         let key = &line[..idx];
 180         let trim_chars: &[_] = &[' ', '='];
 181         let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars));
 182
 183         if key == "programs" {
 184             bin_path.extend(value);
 185         } else if key == "libraries" {
 186             lib_path.extend(value);
 187         }
 188     }
 189
 190     let compiler = if target == "i686-pc-windows-gnu" {
 191         "i686-w64-mingw32-gcc.exe"
 192     } else if target == "x86_64-pc-windows-gnu" {
 193         "x86_64-w64-mingw32-gcc.exe"
 194     } else {
 195         "gcc.exe"
 196     };
 197     let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 198     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 199     if target.starts_with("i686-") {
 200         rustc_dlls.push("libgcc_s_dw2-1.dll");
 201     } else {
 202         rustc_dlls.push("libgcc_s_seh-1.dll");
 203     }
 204
 205     let target_libs = [
 206         //MinGW libs
 207         "libgcc.a",
 208         "libgcc_eh.a",
 209         "libgcc_s.a",
 210         "libm.a",
 211         "libmingw32.a",
 212         "libmingwex.a",
 213         "libstdc++.a",
 214         "libiconv.a",
 215         "libmoldname.a",
 216         "libpthread.a",
 217         //Windows import libs
 218         "libadvapi32.a",
 219         "libbcrypt.a",
 220         "libcomctl32.a",
 221         "libcomdlg32.a",
 222         "libcredui.a",
 223         "libcrypt32.a",
 224         "libdbghelp.a",
 225         "libgdi32.a",
 226         "libimagehlp.a",
 227         "libiphlpapi.a",
 228         "libkernel32.a",
 229         "libmsimg32.a",
 230         "libmsvcrt.a",
 231         "libodbc32.a",
 232         "libole32.a",
 233         "liboleaut32.a",
 234         "libopengl32.a",
 235         "libpsapi.a",
 236         "librpcrt4.a",
 237         "libsecur32.a",
 238         "libsetupapi.a",
 239         "libshell32.a",
 240         "libsynchronization.a",
 241         "libuser32.a",
 242         "libuserenv.a",
 243         "libuuid.a",
 244         "libwinhttp.a",
 245         "libwinmm.a",
 246         "libwinspool.a",
 247         "libws2_32.a",
 248         "libwsock32.a",
 249     ];
 250
 251     //Find mingw artifacts we want to bundle
 252     let target_tools = find_files(&target_tools, &bin_path);
 253     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 254     let target_libs = find_files(&target_libs, &lib_path);
 255
 256     // Copy runtime dlls next to rustc.exe
 257     let dist_bin_dir = rust_root.join("bin/");
 258     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 259     for src in rustc_dlls {
 260         builder.copy_to_folder(&src, &dist_bin_dir);
 261     }
 262
 263     //Copy platform tools to platform-specific bin directory
 264     let target_bin_dir = plat_root
 265         .join("lib")
 266         .join("rustlib")
 267         .join(target.triple)
 268         .join("bin")
 269         .join("self-contained");
 270     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 271     for src in target_tools {
 272         builder.copy_to_folder(&src, &target_bin_dir);
 273     }
 274
 275     // Warn windows-gnu users that the bundled GCC cannot compile C files
 276     builder.create(
 277         &target_bin_dir.join("GCC-WARNING.txt"),
 278         "gcc.exe contained in this folder cannot be used for compiling C files - it is only \
 279          used as a linker. In order to be able to compile projects containing C code use \
 280          the GCC provided by MinGW or Cygwin.",
 281     );
 282
 283     //Copy platform libs to platform-specific lib directory
 284     let target_lib_dir = plat_root
 285         .join("lib")
 286         .join("rustlib")
 287         .join(target.triple)
 288         .join("lib")
 289         .join("self-contained");
 290     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 291     for src in target_libs {
 292         builder.copy_to_folder(&src, &target_lib_dir);
 293     }
 294 }
 295
 296 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 297 pub struct Mingw {
 298     pub host: TargetSelection,
 299 }
 300
 301 impl Step for Mingw {
 302     type Output = Option<GeneratedTarball>;
 303     const DEFAULT: bool = true;
 304
 305     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 306         run.alias("rust-mingw")
 307     }
 308
 309     fn make_run(run: RunConfig<'_>) {
 310         run.builder.ensure(Mingw { host: run.target });
 311     }
 312
 313     /// Builds the `rust-mingw` installer component.
 314     ///
 315     /// This contains all the bits and pieces to run the MinGW Windows targets
 316     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 317     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 318         let host = self.host;
 319         if !host.ends_with("pc-windows-gnu") {
 320             return None;
 321         }
 322
 323         let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple);
 324         tarball.set_product_name("Rust MinGW");
 325
 326         // The first argument is a "temporary directory" which is just
 327         // thrown away (this contains the runtime DLLs included in the rustc package
 328         // above) and the second argument is where to place all the MinGW components
 329         // (which is what we want).
 330         make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder);
 331
 332         Some(tarball.generate())
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Rustc {
 338     pub compiler: Compiler,
 339 }
 340
 341 impl Step for Rustc {
 342     type Output = GeneratedTarball;
 343     const DEFAULT: bool = true;
 344     const ONLY_HOSTS: bool = true;
 345
 346     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 347         run.alias("rustc")
 348     }
 349
 350     fn make_run(run: RunConfig<'_>) {
 351         run.builder
 352             .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
 353     }
 354
 355     /// Creates the `rustc` installer component.
 356     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 357         let compiler = self.compiler;
 358         let host = self.compiler.host;
 359
 360         let tarball = Tarball::new(builder, "rustc", &host.triple);
 361
 362         // Prepare the rustc "image", what will actually end up getting installed
 363         prepare_image(builder, compiler, tarball.image_dir());
 364
 365         // On MinGW we've got a few runtime DLL dependencies that we need to
 366         // include. The first argument to this script is where to put these DLLs
 367         // (the image we're creating), and the second argument is a junk directory
 368         // to ignore all other MinGW stuff the script creates.
 369         //
 370         // On 32-bit MinGW we're always including a DLL which needs some extra
 371         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 372         // anything requiring us to distribute a license, but it's likely the
 373         // install will *also* include the rust-mingw package, which also needs
 374         // licenses, so to be safe we just include it here in all MinGW packages.
 375         if host.ends_with("pc-windows-gnu") {
 376             make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder);
 377             tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc");
 378         }
 379
 380         return tarball.generate();
 381
 382         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 383             let host = compiler.host;
 384             let src = builder.sysroot(compiler);
 385
 386             // Copy rustc/rustdoc binaries
 387             t!(fs::create_dir_all(image.join("bin")));
 388             builder.cp_r(&src.join("bin"), &image.join("bin"));
 389
 390             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 391
 392             let ra_proc_macro_srv = builder
 393                 .ensure(tool::RustAnalyzerProcMacroSrv {
 394                     compiler: builder.compiler_for(
 395                         compiler.stage,
 396                         builder.config.build,
 397                         compiler.host,
 398                     ),
 399                     target: compiler.host,
 400                 })
 401                 .expect("rust-analyzer-proc-macro-server always builds");
 402             builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
 403
 404             let libdir_relative = builder.libdir_relative(compiler);
 405
 406             // Copy runtime DLLs needed by the compiler
 407             if libdir_relative.to_str() != Some("bin") {
 408                 let libdir = builder.rustc_libdir(compiler);
 409                 for entry in builder.read_dir(&libdir) {
 410                     let name = entry.file_name();
 411                     if let Some(s) = name.to_str() {
 412                         if is_dylib(s) {
 413                             // Don't use custom libdir here because ^lib/ will be resolved again
 414                             // with installer
 415                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 416                         }
 417                     }
 418                 }
 419             }
 420
 421             // Copy over the codegen backends
 422             let backends_src = builder.sysroot_codegen_backends(compiler);
 423             let backends_rel = backends_src
 424                 .strip_prefix(&src)
 425                 .unwrap()
 426                 .strip_prefix(builder.sysroot_libdir_relative(compiler))
 427                 .unwrap();
 428             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 429             let backends_dst = image.join("lib").join(&backends_rel);
 430
 431             t!(fs::create_dir_all(&backends_dst));
 432             builder.cp_r(&backends_src, &backends_dst);
 433
 434             // Copy libLLVM.so to the lib dir as well, if needed. While not
 435             // technically needed by rustc itself it's needed by lots of other
 436             // components like the llvm tools and LLD. LLD is included below and
 437             // tools/LLDB come later, so let's just throw it in the rustc
 438             // component for now.
 439             maybe_install_llvm_runtime(builder, host, image);
 440
 441             let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin");
 442             t!(fs::create_dir_all(&dst_dir));
 443
 444             // Copy over lld if it's there
 445             if builder.config.lld_enabled {
 446                 let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
 447                 let rust_lld = exe("rust-lld", compiler.host);
 448                 builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld));
 449                 // for `-Z gcc-ld=lld`
 450                 let gcc_lld_src_dir = src_dir.join("gcc-ld");
 451                 let gcc_lld_dst_dir = dst_dir.join("gcc-ld");
 452                 t!(fs::create_dir(&gcc_lld_dst_dir));
 453                 for name in crate::LLD_FILE_NAMES {
 454                     let exe_name = exe(name, compiler.host);
 455                     builder
 456                         .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name));
 457                 }
 458             }
 459
 460             // Man pages
 461             t!(fs::create_dir_all(image.join("share/man/man1")));
 462             let man_src = builder.src.join("src/doc/man");
 463             let man_dst = image.join("share/man/man1");
 464
 465             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 466             // to hardlink, and we don't want to edit the source templates
 467             for file_entry in builder.read_dir(&man_src) {
 468                 let page_src = file_entry.path();
 469                 let page_dst = man_dst.join(file_entry.file_name());
 470                 let src_text = t!(std::fs::read_to_string(&page_src));
 471                 let new_text = src_text.replace("<INSERT VERSION HERE>", &builder.version);
 472                 t!(std::fs::write(&page_dst, &new_text));
 473                 t!(fs::copy(&page_src, &page_dst));
 474             }
 475
 476             // Debugger scripts
 477             builder
 478                 .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host });
 479
 480             // Misc license info
 481             let cp = |file: &str| {
 482                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 483             };
 484             cp("COPYRIGHT");
 485             cp("LICENSE-APACHE");
 486             cp("LICENSE-MIT");
 487             cp("README.md");
 488         }
 489     }
 490 }
 491
 492 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 493 pub struct DebuggerScripts {
 494     pub sysroot: Interned<PathBuf>,
 495     pub host: TargetSelection,
 496 }
 497
 498 impl Step for DebuggerScripts {
 499     type Output = ();
 500
 501     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 502         run.never()
 503     }
 504
 505     /// Copies debugger scripts for `target` into the `sysroot` specified.
 506     fn run(self, builder: &Builder<'_>) {
 507         let host = self.host;
 508         let sysroot = self.sysroot;
 509         let dst = sysroot.join("lib/rustlib/etc");
 510         t!(fs::create_dir_all(&dst));
 511         let cp_debugger_script = |file: &str| {
 512             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 513         };
 514         if host.contains("windows-msvc") {
 515             // windbg debugger scripts
 516             builder.install(
 517                 &builder.src.join("src/etc/rust-windbg.cmd"),
 518                 &sysroot.join("bin"),
 519                 0o755,
 520             );
 521
 522             cp_debugger_script("natvis/intrinsic.natvis");
 523             cp_debugger_script("natvis/liballoc.natvis");
 524             cp_debugger_script("natvis/libcore.natvis");
 525             cp_debugger_script("natvis/libstd.natvis");
 526         } else {
 527             cp_debugger_script("rust_types.py");
 528
 529             // gdb debugger scripts
 530             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755);
 531             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755);
 532
 533             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 534             cp_debugger_script("gdb_lookup.py");
 535             cp_debugger_script("gdb_providers.py");
 536
 537             // lldb debugger scripts
 538             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755);
 539
 540             cp_debugger_script("lldb_lookup.py");
 541             cp_debugger_script("lldb_providers.py");
 542             cp_debugger_script("lldb_commands")
 543         }
 544     }
 545 }
 546
 547 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 548     // The only true set of target libraries came from the build triple, so
 549     // let's reduce redundant work by only producing archives from that host.
 550     if compiler.host != builder.config.build {
 551         builder.info("\tskipping, not a build host");
 552         true
 553     } else {
 554         false
 555     }
 556 }
 557
 558 /// Copy stamped files into an image's `target/lib` directory.
 559 fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) {
 560     let dst = image.join("lib/rustlib").join(target.triple).join("lib");
 561     let self_contained_dst = dst.join("self-contained");
 562     t!(fs::create_dir_all(&dst));
 563     t!(fs::create_dir_all(&self_contained_dst));
 564     for (path, dependency_type) in builder.read_stamp_file(stamp) {
 565         if dependency_type == DependencyType::TargetSelfContained {
 566             builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap()));
 567         } else if dependency_type == DependencyType::Target || builder.config.build == target {
 568             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 569         }
 570     }
 571 }
 572
 573 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 574 pub struct Std {
 575     pub compiler: Compiler,
 576     pub target: TargetSelection,
 577 }
 578
 579 impl Step for Std {
 580     type Output = Option<GeneratedTarball>;
 581     const DEFAULT: bool = true;
 582
 583     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 584         run.alias("rust-std")
 585     }
 586
 587     fn make_run(run: RunConfig<'_>) {
 588         run.builder.ensure(Std {
 589             compiler: run.builder.compiler_for(
 590                 run.builder.top_stage,
 591                 run.builder.config.build,
 592                 run.target,
 593             ),
 594             target: run.target,
 595         });
 596     }
 597
 598     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 599         let compiler = self.compiler;
 600         let target = self.target;
 601
 602         if skip_host_target_lib(builder, compiler) {
 603             return None;
 604         }
 605
 606         builder.ensure(compile::Std::new(compiler, target));
 607
 608         let mut tarball = Tarball::new(builder, "rust-std", &target.triple);
 609         tarball.include_target_in_component_name(true);
 610
 611         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 612         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 613         copy_target_libs(builder, target, &tarball.image_dir(), &stamp);
 614
 615         Some(tarball.generate())
 616     }
 617 }
 618
 619 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 620 pub struct RustcDev {
 621     pub compiler: Compiler,
 622     pub target: TargetSelection,
 623 }
 624
 625 impl Step for RustcDev {
 626     type Output = Option<GeneratedTarball>;
 627     const DEFAULT: bool = true;
 628     const ONLY_HOSTS: bool = true;
 629
 630     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 631         run.alias("rustc-dev")
 632     }
 633
 634     fn make_run(run: RunConfig<'_>) {
 635         run.builder.ensure(RustcDev {
 636             compiler: run.builder.compiler_for(
 637                 run.builder.top_stage,
 638                 run.builder.config.build,
 639                 run.target,
 640             ),
 641             target: run.target,
 642         });
 643     }
 644
 645     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 646         let compiler = self.compiler;
 647         let target = self.target;
 648         if skip_host_target_lib(builder, compiler) {
 649             return None;
 650         }
 651
 652         builder.ensure(compile::Rustc::new(compiler, target));
 653
 654         let tarball = Tarball::new(builder, "rustc-dev", &target.triple);
 655
 656         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 657         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 658         copy_target_libs(builder, target, tarball.image_dir(), &stamp);
 659
 660         let src_files = &["Cargo.lock"];
 661         // This is the reduced set of paths which will become the rustc-dev component
 662         // (essentially the compiler crates and all of their path dependencies).
 663         copy_src_dirs(
 664             builder,
 665             &builder.src,
 666             &["compiler"],
 667             &[],
 668             &tarball.image_dir().join("lib/rustlib/rustc-src/rust"),
 669         );
 670         for file in src_files {
 671             tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644);
 672         }
 673
 674         Some(tarball.generate())
 675     }
 676 }
 677
 678 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 679 pub struct Analysis {
 680     pub compiler: Compiler,
 681     pub target: TargetSelection,
 682 }
 683
 684 impl Step for Analysis {
 685     type Output = Option<GeneratedTarball>;
 686     const DEFAULT: bool = true;
 687
 688     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 689         let default = should_build_extended_tool(&run.builder, "analysis");
 690         run.alias("rust-analysis").default_condition(default)
 691     }
 692
 693     fn make_run(run: RunConfig<'_>) {
 694         run.builder.ensure(Analysis {
 695             // Find the actual compiler (handling the full bootstrap option) which
 696             // produced the save-analysis data because that data isn't copied
 697             // through the sysroot uplifting.
 698             compiler: run.builder.compiler_for(
 699                 run.builder.top_stage,
 700                 run.builder.config.build,
 701                 run.target,
 702             ),
 703             target: run.target,
 704         });
 705     }
 706
 707     /// Creates a tarball of save-analysis metadata, if available.
 708     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 709         let compiler = self.compiler;
 710         let target = self.target;
 711         if compiler.host != builder.config.build {
 712             return None;
 713         }
 714
 715         builder.ensure(compile::Std::new(compiler, target));
 716         let src = builder
 717             .stage_out(compiler, Mode::Std)
 718             .join(target.triple)
 719             .join(builder.cargo_dir())
 720             .join("deps")
 721             .join("save-analysis");
 722
 723         let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple);
 724         tarball.include_target_in_component_name(true);
 725         tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple));
 726         Some(tarball.generate())
 727     }
 728 }
 729
 730 /// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to
 731 /// `dst_dir`.
 732 fn copy_src_dirs(
 733     builder: &Builder<'_>,
 734     base: &Path,
 735     src_dirs: &[&str],
 736     exclude_dirs: &[&str],
 737     dst_dir: &Path,
 738 ) {
 739     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 740         let spath = match path.to_str() {
 741             Some(path) => path,
 742             None => return false,
 743         };
 744         if spath.ends_with('~') || spath.ends_with(".pyc") {
 745             return false;
 746         }
 747
 748         const LLVM_PROJECTS: &[&str] = &[
 749             "llvm-project/clang",
 750             "llvm-project\\clang",
 751             "llvm-project/libunwind",
 752             "llvm-project\\libunwind",
 753             "llvm-project/lld",
 754             "llvm-project\\lld",
 755             "llvm-project/lldb",
 756             "llvm-project\\lldb",
 757             "llvm-project/llvm",
 758             "llvm-project\\llvm",
 759             "llvm-project/compiler-rt",
 760             "llvm-project\\compiler-rt",
 761             "llvm-project/cmake",
 762             "llvm-project\\cmake",
 763         ];
 764         if spath.contains("llvm-project")
 765             && !spath.ends_with("llvm-project")
 766             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 767         {
 768             return false;
 769         }
 770
 771         const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"];
 772         if LLVM_TEST.iter().any(|path| spath.contains(path))
 773             && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s"))
 774         {
 775             return false;
 776         }
 777
 778         let full_path = Path::new(dir).join(path);
 779         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 780             return false;
 781         }
 782
 783         let excludes = [
 784             "CVS",
 785             "RCS",
 786             "SCCS",
 787             ".git",
 788             ".gitignore",
 789             ".gitmodules",
 790             ".gitattributes",
 791             ".cvsignore",
 792             ".svn",
 793             ".arch-ids",
 794             "{arch}",
 795             "=RELEASE-ID",
 796             "=meta-update",
 797             "=update",
 798             ".bzr",
 799             ".bzrignore",
 800             ".bzrtags",
 801             ".hg",
 802             ".hgignore",
 803             ".hgrags",
 804             "_darcs",
 805         ];
 806         !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s))
 807     }
 808
 809     // Copy the directories using our filter
 810     for item in src_dirs {
 811         let dst = &dst_dir.join(item);
 812         t!(fs::create_dir_all(dst));
 813         builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 814     }
 815 }
 816
 817 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 818 pub struct Src;
 819
 820 impl Step for Src {
 821     /// The output path of the src installer tarball
 822     type Output = GeneratedTarball;
 823     const DEFAULT: bool = true;
 824     const ONLY_HOSTS: bool = true;
 825
 826     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 827         run.alias("rust-src")
 828     }
 829
 830     fn make_run(run: RunConfig<'_>) {
 831         run.builder.ensure(Src);
 832     }
 833
 834     /// Creates the `rust-src` installer component
 835     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 836         let tarball = Tarball::new_targetless(builder, "rust-src");
 837
 838         // A lot of tools expect the rust-src component to be entirely in this directory, so if you
 839         // change that (e.g. by adding another directory `lib/rustlib/src/foo` or
 840         // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions
 841         // and fix them...
 842         //
 843         // NOTE: if you update the paths here, you also should update the "virtual" path
 844         // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs`
 845         let dst_src = tarball.image_dir().join("lib/rustlib/src/rust");
 846
 847         let src_files = ["Cargo.lock"];
 848         // This is the reduced set of paths which will become the rust-src component
 849         // (essentially libstd and all of its path dependencies).
 850         copy_src_dirs(
 851             builder,
 852             &builder.src,
 853             &["library", "src/llvm-project/libunwind"],
 854             &[
 855                 // not needed and contains symlinks which rustup currently
 856                 // chokes on when unpacking.
 857                 "library/backtrace/crates",
 858                 // these are 30MB combined and aren't necessary for building
 859                 // the standard library.
 860                 "library/stdarch/Cargo.toml",
 861                 "library/stdarch/crates/stdarch-verify",
 862                 "library/stdarch/crates/intrinsic-test",
 863             ],
 864             &dst_src,
 865         );
 866         for file in src_files.iter() {
 867             builder.copy(&builder.src.join(file), &dst_src.join(file));
 868         }
 869
 870         tarball.generate()
 871     }
 872 }
 873
 874 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 875 pub struct PlainSourceTarball;
 876
 877 impl Step for PlainSourceTarball {
 878     /// Produces the location of the tarball generated
 879     type Output = GeneratedTarball;
 880     const DEFAULT: bool = true;
 881     const ONLY_HOSTS: bool = true;
 882
 883     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 884         let builder = run.builder;
 885         run.alias("rustc-src").default_condition(builder.config.rust_dist_src)
 886     }
 887
 888     fn make_run(run: RunConfig<'_>) {
 889         run.builder.ensure(PlainSourceTarball);
 890     }
 891
 892     /// Creates the plain source tarball
 893     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
 894         // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which
 895         // means neither rustup nor rustup-toolchain-install-master know how to download it.
 896         // It also contains symbolic links, unlike other any other dist tarball.
 897         // It's used for distros building rustc from source in a pre-vendored environment.
 898         let mut tarball = Tarball::new(builder, "rustc", "src");
 899         tarball.permit_symlinks(true);
 900         let plain_dst_src = tarball.image_dir();
 901
 902         // This is the set of root paths which will become part of the source package
 903         let src_files = [
 904             "COPYRIGHT",
 905             "LICENSE-APACHE",
 906             "LICENSE-MIT",
 907             "CONTRIBUTING.md",
 908             "README.md",
 909             "RELEASES.md",
 910             "configure",
 911             "x.py",
 912             "config.toml.example",
 913             "Cargo.toml",
 914             "Cargo.lock",
 915         ];
 916         let src_dirs = ["src", "compiler", "library"];
 917
 918         copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src);
 919
 920         // Copy the files normally
 921         for item in &src_files {
 922             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 923         }
 924
 925         // Create the version file
 926         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 927         if let Some(sha) = builder.rust_sha() {
 928             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 929         }
 930
 931         // If we're building from git sources, we need to vendor a complete distribution.
 932         if builder.rust_info.is_git() {
 933             // Ensure we have the submodules checked out.
 934             builder.update_submodule(Path::new("src/tools/rust-analyzer"));
 935
 936             // Vendor all Cargo dependencies
 937             let mut cmd = Command::new(&builder.initial_cargo);
 938             cmd.arg("vendor")
 939                 .arg("--sync")
 940                 .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
 941                 .arg("--sync")
 942                 .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml"))
 943                 .arg("--sync")
 944                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
 945                 .current_dir(&plain_dst_src);
 946
 947             let config = if !builder.config.dry_run {
 948                 t!(String::from_utf8(t!(cmd.output()).stdout))
 949             } else {
 950                 String::new()
 951             };
 952
 953             let cargo_config_dir = plain_dst_src.join(".cargo");
 954             builder.create_dir(&cargo_config_dir);
 955             builder.create(&cargo_config_dir.join("config.toml"), &config);
 956         }
 957
 958         tarball.bare()
 959     }
 960 }
 961
 962 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 963 pub struct Cargo {
 964     pub compiler: Compiler,
 965     pub target: TargetSelection,
 966 }
 967
 968 impl Step for Cargo {
 969     type Output = Option<GeneratedTarball>;
 970     const DEFAULT: bool = true;
 971     const ONLY_HOSTS: bool = true;
 972
 973     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 974         let default = should_build_extended_tool(&run.builder, "cargo");
 975         run.alias("cargo").default_condition(default)
 976     }
 977
 978     fn make_run(run: RunConfig<'_>) {
 979         run.builder.ensure(Cargo {
 980             compiler: run.builder.compiler_for(
 981                 run.builder.top_stage,
 982                 run.builder.config.build,
 983                 run.target,
 984             ),
 985             target: run.target,
 986         });
 987     }
 988
 989     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
 990         let compiler = self.compiler;
 991         let target = self.target;
 992
 993         let cargo = builder.ensure(tool::Cargo { compiler, target });
 994         let src = builder.src.join("src/tools/cargo");
 995         let etc = src.join("src/etc");
 996
 997         // Prepare the image directory
 998         let mut tarball = Tarball::new(builder, "cargo", &target.triple);
 999         tarball.set_overlay(OverlayKind::Cargo);
1000
1001         tarball.add_file(&cargo, "bin", 0o755);
1002         tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
1003         tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
1004         tarball.add_dir(etc.join("man"), "share/man/man1");
1005         tarball.add_legal_and_readme_to("share/doc/cargo");
1006
1007         for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") {
1008             let dirent = dirent.expect("read dir entry");
1009             if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") {
1010                 tarball.add_file(&dirent.path(), "libexec", 0o755);
1011             }
1012         }
1013
1014         Some(tarball.generate())
1015     }
1016 }
1017
1018 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1019 pub struct Rls {
1020     pub compiler: Compiler,
1021     pub target: TargetSelection,
1022 }
1023
1024 impl Step for Rls {
1025     type Output = Option<GeneratedTarball>;
1026     const ONLY_HOSTS: bool = true;
1027     const DEFAULT: bool = true;
1028
1029     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1030         let default = should_build_extended_tool(&run.builder, "rls");
1031         run.alias("rls").default_condition(default)
1032     }
1033
1034     fn make_run(run: RunConfig<'_>) {
1035         run.builder.ensure(Rls {
1036             compiler: run.builder.compiler_for(
1037                 run.builder.top_stage,
1038                 run.builder.config.build,
1039                 run.target,
1040             ),
1041             target: run.target,
1042         });
1043     }
1044
1045     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1046         let compiler = self.compiler;
1047         let target = self.target;
1048
1049         let rls = builder
1050             .ensure(tool::Rls { compiler, target, extra_features: Vec::new() })
1051             .expect("rls expected to build");
1052
1053         let mut tarball = Tarball::new(builder, "rls", &target.triple);
1054         tarball.set_overlay(OverlayKind::RLS);
1055         tarball.is_preview(true);
1056         tarball.add_file(rls, "bin", 0o755);
1057         tarball.add_legal_and_readme_to("share/doc/rls");
1058         Some(tarball.generate())
1059     }
1060 }
1061
1062 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1063 pub struct RustAnalyzer {
1064     pub compiler: Compiler,
1065     pub target: TargetSelection,
1066 }
1067
1068 impl Step for RustAnalyzer {
1069     type Output = Option<GeneratedTarball>;
1070     const DEFAULT: bool = true;
1071     const ONLY_HOSTS: bool = true;
1072
1073     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1074         let default = should_build_extended_tool(&run.builder, "rust-analyzer");
1075         run.alias("rust-analyzer").default_condition(default)
1076     }
1077
1078     fn make_run(run: RunConfig<'_>) {
1079         run.builder.ensure(RustAnalyzer {
1080             compiler: run.builder.compiler_for(
1081                 run.builder.top_stage,
1082                 run.builder.config.build,
1083                 run.target,
1084             ),
1085             target: run.target,
1086         });
1087     }
1088
1089     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1090         let compiler = self.compiler;
1091         let target = self.target;
1092
1093         if target.contains("riscv64") {
1094             // riscv64 currently has an LLVM bug that makes rust-analyzer unable
1095             // to build. See #74813 for details.
1096             return None;
1097         }
1098
1099         let rust_analyzer = builder
1100             .ensure(tool::RustAnalyzer { compiler, target })
1101             .expect("rust-analyzer always builds");
1102
1103         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
1104         tarball.set_overlay(OverlayKind::RustAnalyzer);
1105         tarball.is_preview(true);
1106         tarball.add_file(rust_analyzer, "bin", 0o755);
1107         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
1108         Some(tarball.generate())
1109     }
1110 }
1111
1112 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1113 pub struct Clippy {
1114     pub compiler: Compiler,
1115     pub target: TargetSelection,
1116 }
1117
1118 impl Step for Clippy {
1119     type Output = Option<GeneratedTarball>;
1120     const DEFAULT: bool = true;
1121     const ONLY_HOSTS: bool = true;
1122
1123     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1124         let default = should_build_extended_tool(&run.builder, "clippy");
1125         run.alias("clippy").default_condition(default)
1126     }
1127
1128     fn make_run(run: RunConfig<'_>) {
1129         run.builder.ensure(Clippy {
1130             compiler: run.builder.compiler_for(
1131                 run.builder.top_stage,
1132                 run.builder.config.build,
1133                 run.target,
1134             ),
1135             target: run.target,
1136         });
1137     }
1138
1139     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1140         let compiler = self.compiler;
1141         let target = self.target;
1142
1143         // Prepare the image directory
1144         // We expect clippy to build, because we've exited this step above if tool
1145         // state for clippy isn't testing.
1146         let clippy = builder
1147             .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() })
1148             .expect("clippy expected to build - essential tool");
1149         let cargoclippy = builder
1150             .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() })
1151             .expect("clippy expected to build - essential tool");
1152
1153         let mut tarball = Tarball::new(builder, "clippy", &target.triple);
1154         tarball.set_overlay(OverlayKind::Clippy);
1155         tarball.is_preview(true);
1156         tarball.add_file(clippy, "bin", 0o755);
1157         tarball.add_file(cargoclippy, "bin", 0o755);
1158         tarball.add_legal_and_readme_to("share/doc/clippy");
1159         Some(tarball.generate())
1160     }
1161 }
1162
1163 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1164 pub struct Miri {
1165     pub compiler: Compiler,
1166     pub target: TargetSelection,
1167 }
1168
1169 impl Step for Miri {
1170     type Output = Option<GeneratedTarball>;
1171     const DEFAULT: bool = true;
1172     const ONLY_HOSTS: bool = true;
1173
1174     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1175         let default = should_build_extended_tool(&run.builder, "miri");
1176         run.alias("miri").default_condition(default)
1177     }
1178
1179     fn make_run(run: RunConfig<'_>) {
1180         run.builder.ensure(Miri {
1181             compiler: run.builder.compiler_for(
1182                 run.builder.top_stage,
1183                 run.builder.config.build,
1184                 run.target,
1185             ),
1186             target: run.target,
1187         });
1188     }
1189
1190     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1191         // This prevents miri from being built for "dist" or "install"
1192         // on the stable/beta channels. It is a nightly-only tool and should
1193         // not be included.
1194         if !builder.build.unstable_features() {
1195             return None;
1196         }
1197         let compiler = self.compiler;
1198         let target = self.target;
1199
1200         let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?;
1201         let cargomiri =
1202             builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?;
1203
1204         let mut tarball = Tarball::new(builder, "miri", &target.triple);
1205         tarball.set_overlay(OverlayKind::Miri);
1206         tarball.is_preview(true);
1207         tarball.add_file(miri, "bin", 0o755);
1208         tarball.add_file(cargomiri, "bin", 0o755);
1209         tarball.add_legal_and_readme_to("share/doc/miri");
1210         Some(tarball.generate())
1211     }
1212 }
1213
1214 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1215 pub struct Rustfmt {
1216     pub compiler: Compiler,
1217     pub target: TargetSelection,
1218 }
1219
1220 impl Step for Rustfmt {
1221     type Output = Option<GeneratedTarball>;
1222     const DEFAULT: bool = true;
1223     const ONLY_HOSTS: bool = true;
1224
1225     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1226         let default = should_build_extended_tool(&run.builder, "rustfmt");
1227         run.alias("rustfmt").default_condition(default)
1228     }
1229
1230     fn make_run(run: RunConfig<'_>) {
1231         run.builder.ensure(Rustfmt {
1232             compiler: run.builder.compiler_for(
1233                 run.builder.top_stage,
1234                 run.builder.config.build,
1235                 run.target,
1236             ),
1237             target: run.target,
1238         });
1239     }
1240
1241     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1242         let compiler = self.compiler;
1243         let target = self.target;
1244
1245         let rustfmt = builder
1246             .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() })
1247             .expect("rustfmt expected to build - essential tool");
1248         let cargofmt = builder
1249             .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() })
1250             .expect("cargo fmt expected to build - essential tool");
1251         let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
1252         tarball.set_overlay(OverlayKind::Rustfmt);
1253         tarball.is_preview(true);
1254         tarball.add_file(rustfmt, "bin", 0o755);
1255         tarball.add_file(cargofmt, "bin", 0o755);
1256         tarball.add_legal_and_readme_to("share/doc/rustfmt");
1257         Some(tarball.generate())
1258     }
1259 }
1260
1261 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1262 pub struct RustDemangler {
1263     pub compiler: Compiler,
1264     pub target: TargetSelection,
1265 }
1266
1267 impl Step for RustDemangler {
1268     type Output = Option<GeneratedTarball>;
1269     const DEFAULT: bool = true;
1270     const ONLY_HOSTS: bool = true;
1271
1272     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1273         // While other tools use `should_build_extended_tool` to decide whether to be run by
1274         // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like
1275         // the other ones or if `profiler = true`. Because we don't know the target at this stage
1276         // we run the step by default when only `extended = true`, and decide whether to actually
1277         // run it or not later.
1278         let default = run.builder.config.extended;
1279         run.alias("rust-demangler").default_condition(default)
1280     }
1281
1282     fn make_run(run: RunConfig<'_>) {
1283         run.builder.ensure(RustDemangler {
1284             compiler: run.builder.compiler_for(
1285                 run.builder.top_stage,
1286                 run.builder.config.build,
1287                 run.target,
1288             ),
1289             target: run.target,
1290         });
1291     }
1292
1293     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1294         let compiler = self.compiler;
1295         let target = self.target;
1296
1297         // Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
1298         let condition = should_build_extended_tool(builder, "rust-demangler")
1299             || builder.config.profiler_enabled(target);
1300         if builder.config.extended && !condition {
1301             return None;
1302         }
1303
1304         let rust_demangler = builder
1305             .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
1306             .expect("rust-demangler expected to build - in-tree tool");
1307
1308         // Prepare the image directory
1309         let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
1310         tarball.set_overlay(OverlayKind::RustDemangler);
1311         tarball.is_preview(true);
1312         tarball.add_file(&rust_demangler, "bin", 0o755);
1313         tarball.add_legal_and_readme_to("share/doc/rust-demangler");
1314         Some(tarball.generate())
1315     }
1316 }
1317
1318 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1319 pub struct Extended {
1320     stage: u32,
1321     host: TargetSelection,
1322     target: TargetSelection,
1323 }
1324
1325 impl Step for Extended {
1326     type Output = ();
1327     const DEFAULT: bool = true;
1328     const ONLY_HOSTS: bool = true;
1329
1330     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1331         let builder = run.builder;
1332         run.alias("extended").default_condition(builder.config.extended)
1333     }
1334
1335     fn make_run(run: RunConfig<'_>) {
1336         run.builder.ensure(Extended {
1337             stage: run.builder.top_stage,
1338             host: run.builder.config.build,
1339             target: run.target,
1340         });
1341     }
1342
1343     /// Creates a combined installer for the specified target in the provided stage.
1344     fn run(self, builder: &Builder<'_>) {
1345         let target = self.target;
1346         let stage = self.stage;
1347         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1348
1349         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1350
1351         let mut tarballs = Vec::new();
1352         let mut built_tools = HashSet::new();
1353         macro_rules! add_component {
1354             ($name:expr => $step:expr) => {
1355                 if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) {
1356                     tarballs.push(tarball);
1357                     built_tools.insert($name);
1358                 }
1359             };
1360         }
1361
1362         // When rust-std package split from rustc, we needed to ensure that during
1363         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1364         // the std files during uninstall. To do this ensure that rustc comes
1365         // before rust-std in the list below.
1366         tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) }));
1367         tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std"));
1368
1369         if target.ends_with("windows-gnu") {
1370             tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw"));
1371         }
1372
1373         add_component!("rust-docs" => Docs { host: target });
1374         add_component!("rust-demangler"=> RustDemangler { compiler, target });
1375         add_component!("cargo" => Cargo { compiler, target });
1376         add_component!("rustfmt" => Rustfmt { compiler, target });
1377         add_component!("rls" => Rls { compiler, target });
1378         add_component!("rust-analyzer" => RustAnalyzer { compiler, target });
1379         add_component!("llvm-components" => LlvmTools { target });
1380         add_component!("clippy" => Clippy { compiler, target });
1381         add_component!("miri" => Miri { compiler, target });
1382         add_component!("analysis" => Analysis { compiler, target });
1383
1384         let etc = builder.src.join("src/etc/installer");
1385
1386         // Avoid producing tarballs during a dry run.
1387         if builder.config.dry_run {
1388             return;
1389         }
1390
1391         let tarball = Tarball::new(builder, "rust", &target.triple);
1392         let generated = tarball.combine(&tarballs);
1393
1394         let tmp = tmpdir(builder).join("combined-tarball");
1395         let work = generated.work_dir();
1396
1397         let mut license = String::new();
1398         license += &builder.read(&builder.src.join("COPYRIGHT"));
1399         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1400         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1401         license.push('\n');
1402         license.push('\n');
1403
1404         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1405         let mut rtf = rtf.to_string();
1406         rtf.push('\n');
1407         for line in license.lines() {
1408             rtf.push_str(line);
1409             rtf.push_str("\\line ");
1410         }
1411         rtf.push('}');
1412
1413         fn filter(contents: &str, marker: &str) -> String {
1414             let start = format!("tool-{}-start", marker);
1415             let end = format!("tool-{}-end", marker);
1416             let mut lines = Vec::new();
1417             let mut omitted = false;
1418             for line in contents.lines() {
1419                 if line.contains(&start) {
1420                     omitted = true;
1421                 } else if line.contains(&end) {
1422                     omitted = false;
1423                 } else if !omitted {
1424                     lines.push(line);
1425                 }
1426             }
1427
1428             lines.join("\n")
1429         }
1430
1431         let xform = |p: &Path| {
1432             let mut contents = t!(fs::read_to_string(p));
1433             for tool in &["rust-demangler", "rust-analyzer", "rustfmt"] {
1434                 if !built_tools.contains(tool) {
1435                     contents = filter(&contents, tool);
1436                 }
1437             }
1438             let ret = tmp.join(p.file_name().unwrap());
1439             t!(fs::write(&ret, &contents));
1440             ret
1441         };
1442
1443         if target.contains("apple-darwin") {
1444             builder.info("building pkg installer");
1445             let pkg = tmp.join("pkg");
1446             let _ = fs::remove_dir_all(&pkg);
1447
1448             let pkgbuild = |component: &str| {
1449                 let mut cmd = Command::new("pkgbuild");
1450                 cmd.arg("--identifier")
1451                     .arg(format!("org.rust-lang.{}", component))
1452                     .arg("--scripts")
1453                     .arg(pkg.join(component))
1454                     .arg("--nopayload")
1455                     .arg(pkg.join(component).with_extension("pkg"));
1456                 builder.run(&mut cmd);
1457             };
1458
1459             let prepare = |name: &str| {
1460                 builder.create_dir(&pkg.join(name));
1461                 builder.cp_r(
1462                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)),
1463                     &pkg.join(name),
1464                 );
1465                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1466                 pkgbuild(name);
1467             };
1468             prepare("rustc");
1469             prepare("cargo");
1470             prepare("rust-std");
1471             prepare("rust-analysis");
1472             prepare("clippy");
1473             prepare("miri");
1474             for tool in &["rust-docs", "rust-demangler", "rust-analyzer"] {
1475                 if built_tools.contains(tool) {
1476                     prepare(tool);
1477                 }
1478             }
1479             // create an 'uninstall' package
1480             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1481             pkgbuild("uninstall");
1482
1483             builder.create_dir(&pkg.join("res"));
1484             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1485             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1486             let mut cmd = Command::new("productbuild");
1487             cmd.arg("--distribution")
1488                 .arg(xform(&etc.join("pkg/Distribution.xml")))
1489                 .arg("--resources")
1490                 .arg(pkg.join("res"))
1491                 .arg(distdir(builder).join(format!(
1492                     "{}-{}.pkg",
1493                     pkgname(builder, "rust"),
1494                     target.triple
1495                 )))
1496                 .arg("--package-path")
1497                 .arg(&pkg);
1498             let _time = timeit(builder);
1499             builder.run(&mut cmd);
1500         }
1501
1502         if target.contains("windows") {
1503             let exe = tmp.join("exe");
1504             let _ = fs::remove_dir_all(&exe);
1505
1506             let prepare = |name: &str| {
1507                 builder.create_dir(&exe.join(name));
1508                 let dir = if name == "rust-std" || name == "rust-analysis" {
1509                     format!("{}-{}", name, target.triple)
1510                 } else if name == "rust-analyzer" {
1511                     "rust-analyzer-preview".to_string()
1512                 } else if name == "clippy" {
1513                     "clippy-preview".to_string()
1514                 } else if name == "rust-demangler" {
1515                     "rust-demangler-preview".to_string()
1516                 } else if name == "miri" {
1517                     "miri-preview".to_string()
1518                 } else {
1519                     name.to_string()
1520                 };
1521                 builder.cp_r(
1522                     &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir),
1523                     &exe.join(name),
1524                 );
1525                 builder.remove(&exe.join(name).join("manifest.in"));
1526             };
1527             prepare("rustc");
1528             prepare("cargo");
1529             prepare("rust-analysis");
1530             prepare("rust-docs");
1531             prepare("rust-std");
1532             prepare("clippy");
1533             prepare("miri");
1534             for tool in &["rust-demangler", "rust-analyzer"] {
1535                 if built_tools.contains(tool) {
1536                     prepare(tool);
1537                 }
1538             }
1539             if target.ends_with("windows-gnu") {
1540                 prepare("rust-mingw");
1541             }
1542
1543             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1544
1545             // Generate msi installer
1546             let wix_path = env::var_os("WIX")
1547                 .expect("`WIX` environment variable must be set for generating MSI installer(s).");
1548             let wix = PathBuf::from(wix_path);
1549             let heat = wix.join("bin/heat.exe");
1550             let candle = wix.join("bin/candle.exe");
1551             let light = wix.join("bin/light.exe");
1552
1553             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1554             builder.run(
1555                 Command::new(&heat)
1556                     .current_dir(&exe)
1557                     .arg("dir")
1558                     .arg("rustc")
1559                     .args(&heat_flags)
1560                     .arg("-cg")
1561                     .arg("RustcGroup")
1562                     .arg("-dr")
1563                     .arg("Rustc")
1564                     .arg("-var")
1565                     .arg("var.RustcDir")
1566                     .arg("-out")
1567                     .arg(exe.join("RustcGroup.wxs")),
1568             );
1569             builder.run(
1570                 Command::new(&heat)
1571                     .current_dir(&exe)
1572                     .arg("dir")
1573                     .arg("rust-docs")
1574                     .args(&heat_flags)
1575                     .arg("-cg")
1576                     .arg("DocsGroup")
1577                     .arg("-dr")
1578                     .arg("Docs")
1579                     .arg("-var")
1580                     .arg("var.DocsDir")
1581                     .arg("-out")
1582                     .arg(exe.join("DocsGroup.wxs"))
1583                     .arg("-t")
1584                     .arg(etc.join("msi/squash-components.xsl")),
1585             );
1586             builder.run(
1587                 Command::new(&heat)
1588                     .current_dir(&exe)
1589                     .arg("dir")
1590                     .arg("cargo")
1591                     .args(&heat_flags)
1592                     .arg("-cg")
1593                     .arg("CargoGroup")
1594                     .arg("-dr")
1595                     .arg("Cargo")
1596                     .arg("-var")
1597                     .arg("var.CargoDir")
1598                     .arg("-out")
1599                     .arg(exe.join("CargoGroup.wxs"))
1600                     .arg("-t")
1601                     .arg(etc.join("msi/remove-duplicates.xsl")),
1602             );
1603             builder.run(
1604                 Command::new(&heat)
1605                     .current_dir(&exe)
1606                     .arg("dir")
1607                     .arg("rust-std")
1608                     .args(&heat_flags)
1609                     .arg("-cg")
1610                     .arg("StdGroup")
1611                     .arg("-dr")
1612                     .arg("Std")
1613                     .arg("-var")
1614                     .arg("var.StdDir")
1615                     .arg("-out")
1616                     .arg(exe.join("StdGroup.wxs")),
1617             );
1618             if built_tools.contains("rust-analyzer") {
1619                 builder.run(
1620                     Command::new(&heat)
1621                         .current_dir(&exe)
1622                         .arg("dir")
1623                         .arg("rust-analyzer")
1624                         .args(&heat_flags)
1625                         .arg("-cg")
1626                         .arg("RustAnalyzerGroup")
1627                         .arg("-dr")
1628                         .arg("RustAnalyzer")
1629                         .arg("-var")
1630                         .arg("var.RustAnalyzerDir")
1631                         .arg("-out")
1632                         .arg(exe.join("RustAnalyzerGroup.wxs"))
1633                         .arg("-t")
1634                         .arg(etc.join("msi/remove-duplicates.xsl")),
1635                 );
1636             }
1637             builder.run(
1638                 Command::new(&heat)
1639                     .current_dir(&exe)
1640                     .arg("dir")
1641                     .arg("clippy")
1642                     .args(&heat_flags)
1643                     .arg("-cg")
1644                     .arg("ClippyGroup")
1645                     .arg("-dr")
1646                     .arg("Clippy")
1647                     .arg("-var")
1648                     .arg("var.ClippyDir")
1649                     .arg("-out")
1650                     .arg(exe.join("ClippyGroup.wxs"))
1651                     .arg("-t")
1652                     .arg(etc.join("msi/remove-duplicates.xsl")),
1653             );
1654             if built_tools.contains("rust-demangler") {
1655                 builder.run(
1656                     Command::new(&heat)
1657                         .current_dir(&exe)
1658                         .arg("dir")
1659                         .arg("rust-demangler")
1660                         .args(&heat_flags)
1661                         .arg("-cg")
1662                         .arg("RustDemanglerGroup")
1663                         .arg("-dr")
1664                         .arg("RustDemangler")
1665                         .arg("-var")
1666                         .arg("var.RustDemanglerDir")
1667                         .arg("-out")
1668                         .arg(exe.join("RustDemanglerGroup.wxs"))
1669                         .arg("-t")
1670                         .arg(etc.join("msi/remove-duplicates.xsl")),
1671                 );
1672             }
1673             builder.run(
1674                 Command::new(&heat)
1675                     .current_dir(&exe)
1676                     .arg("dir")
1677                     .arg("miri")
1678                     .args(&heat_flags)
1679                     .arg("-cg")
1680                     .arg("MiriGroup")
1681                     .arg("-dr")
1682                     .arg("Miri")
1683                     .arg("-var")
1684                     .arg("var.MiriDir")
1685                     .arg("-out")
1686                     .arg(exe.join("MiriGroup.wxs"))
1687                     .arg("-t")
1688                     .arg(etc.join("msi/remove-duplicates.xsl")),
1689             );
1690             builder.run(
1691                 Command::new(&heat)
1692                     .current_dir(&exe)
1693                     .arg("dir")
1694                     .arg("rust-analysis")
1695                     .args(&heat_flags)
1696                     .arg("-cg")
1697                     .arg("AnalysisGroup")
1698                     .arg("-dr")
1699                     .arg("Analysis")
1700                     .arg("-var")
1701                     .arg("var.AnalysisDir")
1702                     .arg("-out")
1703                     .arg(exe.join("AnalysisGroup.wxs"))
1704                     .arg("-t")
1705                     .arg(etc.join("msi/remove-duplicates.xsl")),
1706             );
1707             if target.ends_with("windows-gnu") {
1708                 builder.run(
1709                     Command::new(&heat)
1710                         .current_dir(&exe)
1711                         .arg("dir")
1712                         .arg("rust-mingw")
1713                         .args(&heat_flags)
1714                         .arg("-cg")
1715                         .arg("GccGroup")
1716                         .arg("-dr")
1717                         .arg("Gcc")
1718                         .arg("-var")
1719                         .arg("var.GccDir")
1720                         .arg("-out")
1721                         .arg(exe.join("GccGroup.wxs")),
1722                 );
1723             }
1724
1725             let candle = |input: &Path| {
1726                 let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj");
1727                 let arch = if target.contains("x86_64") { "x64" } else { "x86" };
1728                 let mut cmd = Command::new(&candle);
1729                 cmd.current_dir(&exe)
1730                     .arg("-nologo")
1731                     .arg("-dRustcDir=rustc")
1732                     .arg("-dDocsDir=rust-docs")
1733                     .arg("-dCargoDir=cargo")
1734                     .arg("-dStdDir=rust-std")
1735                     .arg("-dAnalysisDir=rust-analysis")
1736                     .arg("-dClippyDir=clippy")
1737                     .arg("-dMiriDir=miri")
1738                     .arg("-arch")
1739                     .arg(&arch)
1740                     .arg("-out")
1741                     .arg(&output)
1742                     .arg(&input);
1743                 add_env(builder, &mut cmd, target);
1744
1745                 if built_tools.contains("rust-demangler") {
1746                     cmd.arg("-dRustDemanglerDir=rust-demangler");
1747                 }
1748                 if built_tools.contains("rust-analyzer") {
1749                     cmd.arg("-dRustAnalyzerDir=rust-analyzer");
1750                 }
1751                 if target.ends_with("windows-gnu") {
1752                     cmd.arg("-dGccDir=rust-mingw");
1753                 }
1754                 builder.run(&mut cmd);
1755             };
1756             candle(&xform(&etc.join("msi/rust.wxs")));
1757             candle(&etc.join("msi/ui.wxs"));
1758             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1759             candle("RustcGroup.wxs".as_ref());
1760             candle("DocsGroup.wxs".as_ref());
1761             candle("CargoGroup.wxs".as_ref());
1762             candle("StdGroup.wxs".as_ref());
1763             candle("ClippyGroup.wxs".as_ref());
1764             candle("MiriGroup.wxs".as_ref());
1765             if built_tools.contains("rust-demangler") {
1766                 candle("RustDemanglerGroup.wxs".as_ref());
1767             }
1768             if built_tools.contains("rust-analyzer") {
1769                 candle("RustAnalyzerGroup.wxs".as_ref());
1770             }
1771             candle("AnalysisGroup.wxs".as_ref());
1772
1773             if target.ends_with("windows-gnu") {
1774                 candle("GccGroup.wxs".as_ref());
1775             }
1776
1777             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1778             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1779             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1780
1781             builder.info(&format!("building `msi` installer with {:?}", light));
1782             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple);
1783             let mut cmd = Command::new(&light);
1784             cmd.arg("-nologo")
1785                 .arg("-ext")
1786                 .arg("WixUIExtension")
1787                 .arg("-ext")
1788                 .arg("WixUtilExtension")
1789                 .arg("-out")
1790                 .arg(exe.join(&filename))
1791                 .arg("rust.wixobj")
1792                 .arg("ui.wixobj")
1793                 .arg("rustwelcomedlg.wixobj")
1794                 .arg("RustcGroup.wixobj")
1795                 .arg("DocsGroup.wixobj")
1796                 .arg("CargoGroup.wixobj")
1797                 .arg("StdGroup.wixobj")
1798                 .arg("AnalysisGroup.wixobj")
1799                 .arg("ClippyGroup.wixobj")
1800                 .arg("MiriGroup.wixobj")
1801                 .current_dir(&exe);
1802
1803             if built_tools.contains("rust-analyzer") {
1804                 cmd.arg("RustAnalyzerGroup.wixobj");
1805             }
1806             if built_tools.contains("rust-demangler") {
1807                 cmd.arg("RustDemanglerGroup.wixobj");
1808             }
1809
1810             if target.ends_with("windows-gnu") {
1811                 cmd.arg("GccGroup.wixobj");
1812             }
1813             // ICE57 wrongly complains about the shortcuts
1814             cmd.arg("-sice:ICE57");
1815
1816             let _time = timeit(builder);
1817             builder.run(&mut cmd);
1818
1819             if !builder.config.dry_run {
1820                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1821             }
1822         }
1823     }
1824 }
1825
1826 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
1827     let mut parts = builder.version.split('.');
1828     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1829         .env("CFG_RELEASE_NUM", &builder.version)
1830         .env("CFG_RELEASE", builder.rust_release())
1831         .env("CFG_VER_MAJOR", parts.next().unwrap())
1832         .env("CFG_VER_MINOR", parts.next().unwrap())
1833         .env("CFG_VER_PATCH", parts.next().unwrap())
1834         .env("CFG_VER_BUILD", "0") // just needed to build
1835         .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1836         .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1837         .env("CFG_BUILD", target.triple)
1838         .env("CFG_CHANNEL", &builder.config.channel);
1839
1840     if target.contains("windows-gnullvm") {
1841         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM");
1842     } else if target.contains("windows-gnu") {
1843         cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU");
1844     } else {
1845         cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC");
1846     }
1847 }
1848
1849 /// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
1850 ///
1851 /// Returns whether the files were actually copied.
1852 fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
1853     if let Some(config) = builder.config.target_config.get(&target) {
1854         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
1855             // If the LLVM was externally provided, then we don't currently copy
1856             // artifacts into the sysroot. This is not necessarily the right
1857             // choice (in particular, it will require the LLVM dylib to be in
1858             // the linker's load path at runtime), but the common use case for
1859             // external LLVMs is distribution provided LLVMs, and in that case
1860             // they're usually in the standard search path (e.g., /usr/lib) and
1861             // copying them here is going to cause problems as we may end up
1862             // with the wrong files and isn't what distributions want.
1863             //
1864             // This behavior may be revisited in the future though.
1865             //
1866             // If the LLVM is coming from ourselves (just from CI) though, we
1867             // still want to install it, as it otherwise won't be available.
1868             return false;
1869         }
1870     }
1871
1872     // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib
1873     // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely
1874     // clear why this is the case, though. llvm-config will emit the versioned
1875     // paths and we don't want those in the sysroot (as we're expecting
1876     // unversioned paths).
1877     if target.contains("apple-darwin") && builder.llvm_link_shared() {
1878         let src_libdir = builder.llvm_out(target).join("lib");
1879         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1880         if llvm_dylib_path.exists() {
1881             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
1882         }
1883         !builder.config.dry_run
1884     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
1885         let mut cmd = Command::new(llvm_config);
1886         cmd.arg("--libfiles");
1887         builder.verbose(&format!("running {:?}", cmd));
1888         let files = output(&mut cmd);
1889         let build_llvm_out = &builder.llvm_out(builder.config.build);
1890         let target_llvm_out = &builder.llvm_out(target);
1891         for file in files.trim_end().split(' ') {
1892             // If we're not using a custom LLVM, make sure we package for the target.
1893             let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) {
1894                 target_llvm_out.join(relative_path)
1895             } else {
1896                 PathBuf::from(file)
1897             };
1898             builder.install(&file, dst_libdir, 0o644);
1899         }
1900         !builder.config.dry_run
1901     } else {
1902         false
1903     }
1904 }
1905
1906 /// Maybe add libLLVM.so to the target lib-dir for linking.
1907 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1908     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
1909     // We do not need to copy LLVM files into the sysroot if it is not
1910     // dynamically linked; it is already included into librustc_llvm
1911     // statically.
1912     if builder.llvm_link_shared() {
1913         maybe_install_llvm(builder, target, &dst_libdir);
1914     }
1915 }
1916
1917 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
1918 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
1919     let dst_libdir =
1920         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
1921     // We do not need to copy LLVM files into the sysroot if it is not
1922     // dynamically linked; it is already included into librustc_llvm
1923     // statically.
1924     if builder.llvm_link_shared() {
1925         maybe_install_llvm(builder, target, &dst_libdir);
1926     }
1927 }
1928
1929 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1930 pub struct LlvmTools {
1931     pub target: TargetSelection,
1932 }
1933
1934 impl Step for LlvmTools {
1935     type Output = Option<GeneratedTarball>;
1936     const ONLY_HOSTS: bool = true;
1937     const DEFAULT: bool = true;
1938
1939     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1940         let default = should_build_extended_tool(&run.builder, "llvm-tools");
1941         // FIXME: allow using the names of the tools themselves?
1942         run.alias("llvm-tools").default_condition(default)
1943     }
1944
1945     fn make_run(run: RunConfig<'_>) {
1946         run.builder.ensure(LlvmTools { target: run.target });
1947     }
1948
1949     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
1950         let target = self.target;
1951
1952         /* run only if llvm-config isn't used */
1953         if let Some(config) = builder.config.target_config.get(&target) {
1954             if let Some(ref _s) = config.llvm_config {
1955                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM", target));
1956                 return None;
1957             }
1958         }
1959
1960         builder.ensure(crate::native::Llvm { target });
1961
1962         let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple);
1963         tarball.set_overlay(OverlayKind::LLVM);
1964         tarball.is_preview(true);
1965
1966         // Prepare the image directory
1967         let src_bindir = builder.llvm_out(target).join("bin");
1968         let dst_bindir = format!("lib/rustlib/{}/bin", target.triple);
1969         for tool in LLVM_TOOLS {
1970             let exe = src_bindir.join(exe(tool, target));
1971             tarball.add_file(&exe, &dst_bindir, 0o755);
1972         }
1973
1974         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
1975         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
1976         // of `rustc-dev` to support the inherited `-lLLVM` when using the
1977         // compiler libraries.
1978         maybe_install_llvm_target(builder, target, tarball.image_dir());
1979
1980         Some(tarball.generate())
1981     }
1982 }
1983
1984 // Tarball intended for internal consumption to ease rustc/std development.
1985 //
1986 // Should not be considered stable by end users.
1987 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1988 pub struct RustDev {
1989     pub target: TargetSelection,
1990 }
1991
1992 impl Step for RustDev {
1993     type Output = Option<GeneratedTarball>;
1994     const DEFAULT: bool = true;
1995     const ONLY_HOSTS: bool = true;
1996
1997     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1998         run.alias("rust-dev")
1999     }
2000
2001     fn make_run(run: RunConfig<'_>) {
2002         run.builder.ensure(RustDev { target: run.target });
2003     }
2004
2005     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
2006         let target = self.target;
2007
2008         /* run only if llvm-config isn't used */
2009         if let Some(config) = builder.config.target_config.get(&target) {
2010             if let Some(ref _s) = config.llvm_config {
2011                 builder.info(&format!("Skipping RustDev ({}): external LLVM", target));
2012                 return None;
2013             }
2014         }
2015
2016         let mut tarball = Tarball::new(builder, "rust-dev", &target.triple);
2017         tarball.set_overlay(OverlayKind::LLVM);
2018
2019         builder.ensure(crate::native::Llvm { target });
2020
2021         let src_bindir = builder.llvm_out(target).join("bin");
2022         // If updating this list, you likely want to change
2023         // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
2024         // will not pick up the extra file until LLVM gets bumped.
2025         for bin in &[
2026             "llvm-config",
2027             "llvm-ar",
2028             "llvm-objdump",
2029             "llvm-profdata",
2030             "llvm-bcanalyzer",
2031             "llvm-cov",
2032             "llvm-dwp",
2033             "llvm-nm",
2034             "llvm-dwarfdump",
2035         ] {
2036             tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
2037         }
2038
2039         // We don't build LLD on some platforms, so only add it if it exists
2040         let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target));
2041         if lld_path.exists() {
2042             tarball.add_file(lld_path, "bin", 0o755);
2043         }
2044
2045         tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755);
2046
2047         // Copy the include directory as well; needed mostly to build
2048         // librustc_llvm properly (e.g., llvm-config.h is in here). But also
2049         // just broadly useful to be able to link against the bundled LLVM.
2050         tarball.add_dir(&builder.llvm_out(target).join("include"), "include");
2051
2052         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
2053         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
2054         // of `rustc-dev` to support the inherited `-lLLVM` when using the
2055         // compiler libraries.
2056         let dst_libdir = tarball.image_dir().join("lib");
2057         maybe_install_llvm(builder, target, &dst_libdir);
2058         let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
2059         t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
2060
2061         Some(tarball.generate())
2062     }
2063 }
2064
2065 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
2066 /// release process to avoid cloning the monorepo and building stuff.
2067 ///
2068 /// Should not be considered stable by end users.
2069 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2070 pub struct BuildManifest {
2071     pub target: TargetSelection,
2072 }
2073
2074 impl Step for BuildManifest {
2075     type Output = GeneratedTarball;
2076     const DEFAULT: bool = false;
2077     const ONLY_HOSTS: bool = true;
2078
2079     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2080         run.alias("build-manifest")
2081     }
2082
2083     fn make_run(run: RunConfig<'_>) {
2084         run.builder.ensure(BuildManifest { target: run.target });
2085     }
2086
2087     fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
2088         let build_manifest = builder.tool_exe(Tool::BuildManifest);
2089
2090         let tarball = Tarball::new(builder, "build-manifest", &self.target.triple);
2091         tarball.add_file(&build_manifest, "bin", 0o755);
2092         tarball.generate()
2093     }
2094 }
2095
2096 /// Tarball containing artifacts necessary to reproduce the build of rustc.
2097 ///
2098 /// Currently this is the PGO profile data.
2099 ///
2100 /// Should not be considered stable by end users.
2101 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2102 pub struct ReproducibleArtifacts {
2103     pub target: TargetSelection,
2104 }
2105
2106 impl Step for ReproducibleArtifacts {
2107     type Output = Option<GeneratedTarball>;
2108     const DEFAULT: bool = true;
2109     const ONLY_HOSTS: bool = true;
2110
2111     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2112         run.alias("reproducible-artifacts")
2113     }
2114
2115     fn make_run(run: RunConfig<'_>) {
2116         run.builder.ensure(ReproducibleArtifacts { target: run.target });
2117     }
2118
2119     fn run(self, builder: &Builder<'_>) -> Self::Output {
2120         let mut added_anything = false;
2121         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
2122         if let Some(path) = builder.config.rust_profile_use.as_ref() {
2123             tarball.add_file(path, ".", 0o644);
2124             added_anything = true;
2125         }
2126         if let Some(path) = builder.config.llvm_profile_use.as_ref() {
2127             tarball.add_file(path, ".", 0o644);
2128             added_anything = true;
2129         }
2130         if added_anything { Some(tarball.generate()) } else { None }
2131     }
2132 }