src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe, timeit};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         if !builder.config.docs {
  95             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  96         }
  97
  98         builder.default_doc(None);
  99
 100         builder.info(&format!("Dist docs ({})", host));
 101         let _time = timeit(builder);
 102
 103         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 104         let _ = fs::remove_dir_all(&image);
 105
 106         let dst = image.join("share/doc/rust/html");
 107         t!(fs::create_dir_all(&dst));
 108         let src = builder.doc_out(host);
 109         builder.cp_r(&src, &dst);
 110
 111         let mut cmd = rust_installer(builder);
 112         cmd.arg("generate")
 113            .arg("--product-name=Rust-Documentation")
 114            .arg("--rel-manifest-dir=rustlib")
 115            .arg("--success-message=Rust-documentation-is-installed.")
 116            .arg("--image-dir").arg(&image)
 117            .arg("--work-dir").arg(&tmpdir(builder))
 118            .arg("--output-dir").arg(&distdir(builder))
 119            .arg(format!("--package-name={}-{}", name, host))
 120            .arg("--component-name=rust-docs")
 121            .arg("--legacy-manifest-dirs=rustlib,cargo")
 122            .arg("--bulk-dirs=share/doc/rust/html");
 123         builder.run(&mut cmd);
 124         builder.remove_dir(&image);
 125
 126         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 127     }
 128 }
 129
 130 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 131 pub struct RustcDocs {
 132     pub host: Interned<String>,
 133 }
 134
 135 impl Step for RustcDocs {
 136     type Output = PathBuf;
 137     const DEFAULT: bool = true;
 138
 139     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 140         run.path("src/librustc")
 141     }
 142
 143     fn make_run(run: RunConfig<'_>) {
 144         run.builder.ensure(RustcDocs {
 145             host: run.target,
 146         });
 147     }
 148
 149     /// Builds the `rustc-docs` installer component.
 150     fn run(self, builder: &Builder<'_>) -> PathBuf {
 151         let host = self.host;
 152
 153         let name = pkgname(builder, "rustc-docs");
 154
 155         if !builder.config.compiler_docs {
 156             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 157         }
 158
 159         builder.default_doc(None);
 160
 161         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 162         let _ = fs::remove_dir_all(&image);
 163
 164         let dst = image.join("share/doc/rust/html");
 165         t!(fs::create_dir_all(&dst));
 166         let src = builder.compiler_doc_out(host);
 167         builder.cp_r(&src, &dst);
 168
 169         let mut cmd = rust_installer(builder);
 170         cmd.arg("generate")
 171            .arg("--product-name=Rustc-Documentation")
 172            .arg("--rel-manifest-dir=rustlib")
 173            .arg("--success-message=Rustc-documentation-is-installed.")
 174            .arg("--image-dir").arg(&image)
 175            .arg("--work-dir").arg(&tmpdir(builder))
 176            .arg("--output-dir").arg(&distdir(builder))
 177            .arg(format!("--package-name={}-{}", name, host))
 178            .arg("--component-name=rustc-docs")
 179            .arg("--legacy-manifest-dirs=rustlib,cargo")
 180            .arg("--bulk-dirs=share/doc/rust/html");
 181
 182         builder.info(&format!("Dist compiler docs ({})", host));
 183         let _time = timeit(builder);
 184         builder.run(&mut cmd);
 185         builder.remove_dir(&image);
 186
 187         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 188     }
 189 }
 190
 191 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 192     let mut found = Vec::with_capacity(files.len());
 193
 194     for file in files {
 195         let file_path =
 196             path.iter()
 197                 .map(|dir| dir.join(file))
 198                 .find(|p| p.exists());
 199
 200         if let Some(file_path) = file_path {
 201             found.push(file_path);
 202         } else {
 203             panic!("Could not find '{}' in {:?}", file, path);
 204         }
 205     }
 206
 207     found
 208 }
 209
 210 fn make_win_dist(
 211     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 212 ) {
 213     //Ask gcc where it keeps its stuff
 214     let mut cmd = Command::new(builder.cc(target_triple));
 215     cmd.arg("-print-search-dirs");
 216     let gcc_out = output(&mut cmd);
 217
 218     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 219     let mut lib_path = Vec::new();
 220
 221     for line in gcc_out.lines() {
 222         let idx = line.find(':').unwrap();
 223         let key = &line[..idx];
 224         let trim_chars: &[_] = &[' ', '='];
 225         let value =
 226             line[(idx + 1)..]
 227                 .trim_start_matches(trim_chars)
 228                 .split(';')
 229                 .map(PathBuf::from);
 230
 231         if key == "programs" {
 232             bin_path.extend(value);
 233         } else if key == "libraries" {
 234             lib_path.extend(value);
 235         }
 236     }
 237
 238     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 239     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 240     if target_triple.starts_with("i686-") {
 241         rustc_dlls.push("libgcc_s_dw2-1.dll");
 242     } else {
 243         rustc_dlls.push("libgcc_s_seh-1.dll");
 244     }
 245
 246     let target_libs = [ //MinGW libs
 247         "libgcc.a",
 248         "libgcc_eh.a",
 249         "libgcc_s.a",
 250         "libm.a",
 251         "libmingw32.a",
 252         "libmingwex.a",
 253         "libstdc++.a",
 254         "libiconv.a",
 255         "libmoldname.a",
 256         "libpthread.a",
 257         //Windows import libs
 258         "libadvapi32.a",
 259         "libbcrypt.a",
 260         "libcomctl32.a",
 261         "libcomdlg32.a",
 262         "libcredui.a",
 263         "libcrypt32.a",
 264         "libdbghelp.a",
 265         "libgdi32.a",
 266         "libimagehlp.a",
 267         "libiphlpapi.a",
 268         "libkernel32.a",
 269         "libmsimg32.a",
 270         "libmsvcrt.a",
 271         "libodbc32.a",
 272         "libole32.a",
 273         "liboleaut32.a",
 274         "libopengl32.a",
 275         "libpsapi.a",
 276         "librpcrt4.a",
 277         "libsecur32.a",
 278         "libsetupapi.a",
 279         "libshell32.a",
 280         "libsynchronization.a",
 281         "libuser32.a",
 282         "libuserenv.a",
 283         "libuuid.a",
 284         "libwinhttp.a",
 285         "libwinmm.a",
 286         "libwinspool.a",
 287         "libws2_32.a",
 288         "libwsock32.a",
 289     ];
 290
 291     //Find mingw artifacts we want to bundle
 292     let target_tools = find_files(&target_tools, &bin_path);
 293     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 294     let target_libs = find_files(&target_libs, &lib_path);
 295
 296     // Copy runtime dlls next to rustc.exe
 297     let dist_bin_dir = rust_root.join("bin/");
 298     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 299     for src in rustc_dlls {
 300         builder.copy_to_folder(&src, &dist_bin_dir);
 301     }
 302
 303     //Copy platform tools to platform-specific bin directory
 304     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 305     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 306     for src in target_tools {
 307         builder.copy_to_folder(&src, &target_bin_dir);
 308     }
 309
 310     // Warn windows-gnu users that the bundled GCC cannot compile C files
 311     builder.create(
 312         &target_bin_dir.join("GCC-WARNING.txt"),
 313         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 314          used as a linker. In order to be able to compile projects containing C code use\
 315          the GCC provided by MinGW or Cygwin."
 316     );
 317
 318     //Copy platform libs to platform-specific lib directory
 319     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 320     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 321     for src in target_libs {
 322         builder.copy_to_folder(&src, &target_lib_dir);
 323     }
 324 }
 325
 326 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 327 pub struct Mingw {
 328     pub host: Interned<String>,
 329 }
 330
 331 impl Step for Mingw {
 332     type Output = Option<PathBuf>;
 333     const DEFAULT: bool = true;
 334
 335     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 336         run.never()
 337     }
 338
 339     fn make_run(run: RunConfig<'_>) {
 340         run.builder.ensure(Mingw { host: run.target });
 341     }
 342
 343     /// Builds the `rust-mingw` installer component.
 344     ///
 345     /// This contains all the bits and pieces to run the MinGW Windows targets
 346     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 347     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 348         let host = self.host;
 349
 350         if !host.contains("pc-windows-gnu") {
 351             return None;
 352         }
 353
 354         builder.info(&format!("Dist mingw ({})", host));
 355         let _time = timeit(builder);
 356         let name = pkgname(builder, "rust-mingw");
 357         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 358         let _ = fs::remove_dir_all(&image);
 359         t!(fs::create_dir_all(&image));
 360
 361         // The first argument is a "temporary directory" which is just
 362         // thrown away (this contains the runtime DLLs included in the rustc package
 363         // above) and the second argument is where to place all the MinGW components
 364         // (which is what we want).
 365         make_win_dist(&tmpdir(builder), &image, host, &builder);
 366
 367         let mut cmd = rust_installer(builder);
 368         cmd.arg("generate")
 369            .arg("--product-name=Rust-MinGW")
 370            .arg("--rel-manifest-dir=rustlib")
 371            .arg("--success-message=Rust-MinGW-is-installed.")
 372            .arg("--image-dir").arg(&image)
 373            .arg("--work-dir").arg(&tmpdir(builder))
 374            .arg("--output-dir").arg(&distdir(builder))
 375            .arg(format!("--package-name={}-{}", name, host))
 376            .arg("--component-name=rust-mingw")
 377            .arg("--legacy-manifest-dirs=rustlib,cargo");
 378         builder.run(&mut cmd);
 379         t!(fs::remove_dir_all(&image));
 380         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 381     }
 382 }
 383
 384 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 385 pub struct Rustc {
 386     pub compiler: Compiler,
 387 }
 388
 389 impl Step for Rustc {
 390     type Output = PathBuf;
 391     const DEFAULT: bool = true;
 392     const ONLY_HOSTS: bool = true;
 393
 394     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 395         run.path("src/librustc")
 396     }
 397
 398     fn make_run(run: RunConfig<'_>) {
 399         run.builder.ensure(Rustc {
 400             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 401         });
 402     }
 403
 404     /// Creates the `rustc` installer component.
 405     fn run(self, builder: &Builder<'_>) -> PathBuf {
 406         let compiler = self.compiler;
 407         let host = self.compiler.host;
 408
 409         let name = pkgname(builder, "rustc");
 410         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 411         let _ = fs::remove_dir_all(&image);
 412         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 413         let _ = fs::remove_dir_all(&overlay);
 414
 415         // Prepare the rustc "image", what will actually end up getting installed
 416         prepare_image(builder, compiler, &image);
 417
 418         // Prepare the overlay which is part of the tarball but won't actually be
 419         // installed
 420         let cp = |file: &str| {
 421             builder.install(&builder.src.join(file), &overlay, 0o644);
 422         };
 423         cp("COPYRIGHT");
 424         cp("LICENSE-APACHE");
 425         cp("LICENSE-MIT");
 426         cp("README.md");
 427         // tiny morsel of metadata is used by rust-packaging
 428         let version = builder.rust_version();
 429         builder.create(&overlay.join("version"), &version);
 430         if let Some(sha) = builder.rust_sha() {
 431             builder.create(&overlay.join("git-commit-hash"), &sha);
 432         }
 433
 434         // On MinGW we've got a few runtime DLL dependencies that we need to
 435         // include. The first argument to this script is where to put these DLLs
 436         // (the image we're creating), and the second argument is a junk directory
 437         // to ignore all other MinGW stuff the script creates.
 438         //
 439         // On 32-bit MinGW we're always including a DLL which needs some extra
 440         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 441         // anything requiring us to distribute a license, but it's likely the
 442         // install will *also* include the rust-mingw package, which also needs
 443         // licenses, so to be safe we just include it here in all MinGW packages.
 444         if host.contains("pc-windows-gnu") {
 445             make_win_dist(&image, &tmpdir(builder), host, builder);
 446
 447             let dst = image.join("share/doc");
 448             t!(fs::create_dir_all(&dst));
 449             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 450         }
 451
 452         // Finally, wrap everything up in a nice tarball!
 453         let mut cmd = rust_installer(builder);
 454         cmd.arg("generate")
 455            .arg("--product-name=Rust")
 456            .arg("--rel-manifest-dir=rustlib")
 457            .arg("--success-message=Rust-is-ready-to-roll.")
 458            .arg("--image-dir").arg(&image)
 459            .arg("--work-dir").arg(&tmpdir(builder))
 460            .arg("--output-dir").arg(&distdir(builder))
 461            .arg("--non-installed-overlay").arg(&overlay)
 462            .arg(format!("--package-name={}-{}", name, host))
 463            .arg("--component-name=rustc")
 464            .arg("--legacy-manifest-dirs=rustlib,cargo");
 465
 466         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 467         let _time = timeit(builder);
 468         builder.run(&mut cmd);
 469         builder.remove_dir(&image);
 470         builder.remove_dir(&overlay);
 471
 472         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 473
 474         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 475             let host = compiler.host;
 476             let src = builder.sysroot(compiler);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 483
 484             let libdir_relative = builder.libdir_relative(compiler);
 485
 486             // Copy runtime DLLs needed by the compiler
 487             if libdir_relative.to_str() != Some("bin") {
 488                 let libdir = builder.rustc_libdir(compiler);
 489                 for entry in builder.read_dir(&libdir) {
 490                     let name = entry.file_name();
 491                     if let Some(s) = name.to_str() {
 492                         if is_dylib(s) {
 493                             // Don't use custom libdir here because ^lib/ will be resolved again
 494                             // with installer
 495                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 496                         }
 497                     }
 498                 }
 499             }
 500
 501             // Copy over the codegen backends
 502             let backends_src = builder.sysroot_codegen_backends(compiler);
 503             let backends_rel = backends_src.strip_prefix(&src).unwrap()
 504                 .strip_prefix(builder.sysroot_libdir_relative(compiler)).unwrap();
 505             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 506             let backends_dst = image.join("lib").join(&backends_rel);
 507
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539
 540             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 541             let time = env::var("SOURCE_DATE_EPOCH")
 542                 .map(|timestamp| {
 543                     let epoch = timestamp.parse().map_err(|err| {
 544                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 545                     }).unwrap();
 546
 547                     time::at(Timespec::new(epoch, 0))
 548                 })
 549                 .unwrap_or_else(|_| time::now());
 550
 551             let month_year = t!(time::strftime("%B %Y", &time));
 552             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 553             // to hardlink, and we don't want to edit the source templates
 554             for file_entry in builder.read_dir(&man_src) {
 555                 let page_src = file_entry.path();
 556                 let page_dst = man_dst.join(file_entry.file_name());
 557                 t!(fs::copy(&page_src, &page_dst));
 558                 // template in month/year and version number
 559                 builder.replace_in_file(&page_dst,
 560                                 &[("<INSERT DATE HERE>", &month_year),
 561                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 562             }
 563
 564             // Debugger scripts
 565             builder.ensure(DebuggerScripts {
 566                 sysroot: INTERNER.intern_path(image.to_owned()),
 567                 host,
 568             });
 569
 570             // Misc license info
 571             let cp = |file: &str| {
 572                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 573             };
 574             cp("COPYRIGHT");
 575             cp("LICENSE-APACHE");
 576             cp("LICENSE-MIT");
 577             cp("README.md");
 578         }
 579     }
 580 }
 581
 582 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 583 pub struct DebuggerScripts {
 584     pub sysroot: Interned<PathBuf>,
 585     pub host: Interned<String>,
 586 }
 587
 588 impl Step for DebuggerScripts {
 589     type Output = ();
 590
 591     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 592         run.path("src/lldb_batchmode.py")
 593     }
 594
 595     fn make_run(run: RunConfig<'_>) {
 596         run.builder.ensure(DebuggerScripts {
 597             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 598             host: run.target,
 599         });
 600     }
 601
 602     /// Copies debugger scripts for `target` into the `sysroot` specified.
 603     fn run(self, builder: &Builder<'_>) {
 604         let host = self.host;
 605         let sysroot = self.sysroot;
 606         let dst = sysroot.join("lib/rustlib/etc");
 607         t!(fs::create_dir_all(&dst));
 608         let cp_debugger_script = |file: &str| {
 609             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 610         };
 611         if host.contains("windows-msvc") {
 612             // windbg debugger scripts
 613             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 614                 0o755);
 615
 616             cp_debugger_script("natvis/intrinsic.natvis");
 617             cp_debugger_script("natvis/liballoc.natvis");
 618             cp_debugger_script("natvis/libcore.natvis");
 619             cp_debugger_script("natvis/libstd.natvis");
 620         } else {
 621             cp_debugger_script("debugger_pretty_printers_common.py");
 622
 623             // gdb debugger scripts
 624             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 625                     0o755);
 626             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 627                     0o755);
 628
 629             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 630             cp_debugger_script("gdb_rust_pretty_printing.py");
 631
 632             // lldb debugger scripts
 633             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 634                     0o755);
 635
 636             cp_debugger_script("lldb_rust_formatters.py");
 637         }
 638     }
 639 }
 640
 641 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 642     // The only true set of target libraries came from the build triple, so
 643     // let's reduce redundant work by only producing archives from that host.
 644     if compiler.host != builder.config.build {
 645         builder.info("\tskipping, not a build host");
 646         true
 647     } else {
 648         false
 649     }
 650 }
 651
 652 /// Copy stamped files into an image's `target/lib` directory.
 653 fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
 654     let dst = image.join("lib/rustlib").join(target).join("lib");
 655     t!(fs::create_dir_all(&dst));
 656     for (path, host) in builder.read_stamp_file(stamp) {
 657         if !host || builder.config.build == target {
 658             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 659         }
 660     }
 661 }
 662
 663 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 664 pub struct Std {
 665     pub compiler: Compiler,
 666     pub target: Interned<String>,
 667 }
 668
 669 impl Step for Std {
 670     type Output = PathBuf;
 671     const DEFAULT: bool = true;
 672
 673     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 674         run.path("src/libstd")
 675     }
 676
 677     fn make_run(run: RunConfig<'_>) {
 678         run.builder.ensure(Std {
 679             compiler: run.builder.compiler_for(
 680                 run.builder.top_stage,
 681                 run.builder.config.build,
 682                 run.target,
 683             ),
 684             target: run.target,
 685         });
 686     }
 687
 688     fn run(self, builder: &Builder<'_>) -> PathBuf {
 689         let compiler = self.compiler;
 690         let target = self.target;
 691
 692         let name = pkgname(builder, "rust-std");
 693         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 694         if skip_host_target_lib(builder, compiler) {
 695             return archive;
 696         }
 697
 698         builder.ensure(compile::Std { compiler, target });
 699
 700         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 701         let _ = fs::remove_dir_all(&image);
 702
 703         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 704         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 705         copy_target_libs(builder, &target, &image, &stamp);
 706
 707         let mut cmd = rust_installer(builder);
 708         cmd.arg("generate")
 709            .arg("--product-name=Rust")
 710            .arg("--rel-manifest-dir=rustlib")
 711            .arg("--success-message=std-is-standing-at-the-ready.")
 712            .arg("--image-dir").arg(&image)
 713            .arg("--work-dir").arg(&tmpdir(builder))
 714            .arg("--output-dir").arg(&distdir(builder))
 715            .arg(format!("--package-name={}-{}", name, target))
 716            .arg(format!("--component-name=rust-std-{}", target))
 717            .arg("--legacy-manifest-dirs=rustlib,cargo");
 718
 719         builder.info(&format!("Dist std stage{} ({} -> {})",
 720             compiler.stage, &compiler.host, target));
 721         let _time = timeit(builder);
 722         builder.run(&mut cmd);
 723         builder.remove_dir(&image);
 724         archive
 725     }
 726 }
 727
 728 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 729 pub struct RustcDev {
 730     pub compiler: Compiler,
 731     pub target: Interned<String>,
 732 }
 733
 734 impl Step for RustcDev {
 735     type Output = PathBuf;
 736     const DEFAULT: bool = true;
 737     const ONLY_HOSTS: bool = true;
 738
 739     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 740         run.path("rustc-dev")
 741     }
 742
 743     fn make_run(run: RunConfig<'_>) {
 744         run.builder.ensure(RustcDev {
 745             compiler: run.builder.compiler_for(
 746                 run.builder.top_stage,
 747                 run.builder.config.build,
 748                 run.target,
 749             ),
 750             target: run.target,
 751         });
 752     }
 753
 754     fn run(self, builder: &Builder<'_>) -> PathBuf {
 755         let compiler = self.compiler;
 756         let target = self.target;
 757
 758         let name = pkgname(builder, "rustc-dev");
 759         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 760         if skip_host_target_lib(builder, compiler) {
 761             return archive;
 762         }
 763
 764         builder.ensure(compile::Rustc { compiler, target });
 765
 766         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 767         let _ = fs::remove_dir_all(&image);
 768
 769         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 770         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 771         copy_target_libs(builder, &target, &image, &stamp);
 772
 773         let mut cmd = rust_installer(builder);
 774         cmd.arg("generate")
 775            .arg("--product-name=Rust")
 776            .arg("--rel-manifest-dir=rustlib")
 777            .arg("--success-message=Rust-is-ready-to-develop.")
 778            .arg("--image-dir").arg(&image)
 779            .arg("--work-dir").arg(&tmpdir(builder))
 780            .arg("--output-dir").arg(&distdir(builder))
 781            .arg(format!("--package-name={}-{}", name, target))
 782            .arg(format!("--component-name=rustc-dev-{}", target))
 783            .arg("--legacy-manifest-dirs=rustlib,cargo");
 784
 785         builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
 786             compiler.stage, &compiler.host, target));
 787         let _time = timeit(builder);
 788         builder.run(&mut cmd);
 789         builder.remove_dir(&image);
 790         archive
 791     }
 792 }
 793
 794 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 795 pub struct Analysis {
 796     pub compiler: Compiler,
 797     pub target: Interned<String>,
 798 }
 799
 800 impl Step for Analysis {
 801     type Output = PathBuf;
 802     const DEFAULT: bool = true;
 803
 804     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 805         let builder = run.builder;
 806         run.path("analysis").default_condition(builder.config.extended)
 807     }
 808
 809     fn make_run(run: RunConfig<'_>) {
 810         run.builder.ensure(Analysis {
 811             // Find the actual compiler (handling the full bootstrap option) which
 812             // produced the save-analysis data because that data isn't copied
 813             // through the sysroot uplifting.
 814             compiler: run.builder.compiler_for(
 815                 run.builder.top_stage,
 816                 run.builder.config.build,
 817                 run.target,
 818             ),
 819             target: run.target,
 820         });
 821     }
 822
 823     /// Creates a tarball of save-analysis metadata, if available.
 824     fn run(self, builder: &Builder<'_>) -> PathBuf {
 825         let compiler = self.compiler;
 826         let target = self.target;
 827         assert!(builder.config.extended);
 828         let name = pkgname(builder, "rust-analysis");
 829
 830         if &compiler.host != builder.config.build {
 831             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 832         }
 833
 834         builder.ensure(compile::Std { compiler, target });
 835
 836         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 837
 838         let src = builder.stage_out(compiler, Mode::Std)
 839             .join(target).join(builder.cargo_dir()).join("deps");
 840
 841         let image_src = src.join("save-analysis");
 842         let dst = image.join("lib/rustlib").join(target).join("analysis");
 843         t!(fs::create_dir_all(&dst));
 844         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 845         builder.cp_r(&image_src, &dst);
 846
 847         let mut cmd = rust_installer(builder);
 848         cmd.arg("generate")
 849            .arg("--product-name=Rust")
 850            .arg("--rel-manifest-dir=rustlib")
 851            .arg("--success-message=save-analysis-saved.")
 852            .arg("--image-dir").arg(&image)
 853            .arg("--work-dir").arg(&tmpdir(builder))
 854            .arg("--output-dir").arg(&distdir(builder))
 855            .arg(format!("--package-name={}-{}", name, target))
 856            .arg(format!("--component-name=rust-analysis-{}", target))
 857            .arg("--legacy-manifest-dirs=rustlib,cargo");
 858
 859         builder.info("Dist analysis");
 860         let _time = timeit(builder);
 861         builder.run(&mut cmd);
 862         builder.remove_dir(&image);
 863         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 864     }
 865 }
 866
 867 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 868     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 869         let spath = match path.to_str() {
 870             Some(path) => path,
 871             None => return false,
 872         };
 873         if spath.ends_with("~") || spath.ends_with(".pyc") {
 874             return false
 875         }
 876
 877         const LLVM_PROJECTS: &[&str] = &[
 878             "llvm-project/clang", "llvm-project\\clang",
 879             "llvm-project/libunwind", "llvm-project\\libunwind",
 880             "llvm-project/lld", "llvm-project\\lld",
 881             "llvm-project/lldb", "llvm-project\\lldb",
 882             "llvm-project/llvm", "llvm-project\\llvm",
 883             "llvm-project/compiler-rt", "llvm-project\\compiler-rt",
 884         ];
 885         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 886             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 887         {
 888             return false;
 889         }
 890
 891         const LLVM_TEST: &[&str] = &[
 892             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 893         ];
 894         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 895             (spath.ends_with(".ll") ||
 896              spath.ends_with(".td") ||
 897              spath.ends_with(".s")) {
 898             return false
 899         }
 900
 901         let full_path = Path::new(dir).join(path);
 902         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 903             return false;
 904         }
 905
 906         let excludes = [
 907             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 908             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 909             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 910             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 911         ];
 912         !path.iter()
 913              .map(|s| s.to_str().unwrap())
 914              .any(|s| excludes.contains(&s))
 915     }
 916
 917     // Copy the directories using our filter
 918     for item in src_dirs {
 919         let dst = &dst_dir.join(item);
 920         t!(fs::create_dir_all(dst));
 921         builder.cp_filtered(
 922             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 923     }
 924 }
 925
 926 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 927 pub struct Src;
 928
 929 impl Step for Src {
 930     /// The output path of the src installer tarball
 931     type Output = PathBuf;
 932     const DEFAULT: bool = true;
 933     const ONLY_HOSTS: bool = true;
 934
 935     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 936         run.path("src")
 937     }
 938
 939     fn make_run(run: RunConfig<'_>) {
 940         run.builder.ensure(Src);
 941     }
 942
 943     /// Creates the `rust-src` installer component
 944     fn run(self, builder: &Builder<'_>) -> PathBuf {
 945         let name = pkgname(builder, "rust-src");
 946         let image = tmpdir(builder).join(format!("{}-image", name));
 947         let _ = fs::remove_dir_all(&image);
 948
 949         let dst = image.join("lib/rustlib/src");
 950         let dst_src = dst.join("rust");
 951         t!(fs::create_dir_all(&dst_src));
 952
 953         let src_files = [
 954             "Cargo.lock",
 955         ];
 956         // This is the reduced set of paths which will become the rust-src component
 957         // (essentially libstd and all of its path dependencies)
 958         let std_src_dirs = [
 959             "src/build_helper",
 960             "src/liballoc",
 961             "src/libcore",
 962             "src/libpanic_abort",
 963             "src/libpanic_unwind",
 964             "src/librustc_asan",
 965             "src/librustc_lsan",
 966             "src/librustc_msan",
 967             "src/librustc_tsan",
 968             "src/libstd",
 969             "src/libunwind",
 970             "src/libtest",
 971             "src/libterm",
 972             "src/libprofiler_builtins",
 973             "src/stdarch",
 974             "src/libproc_macro",
 975             "src/tools/rustc-std-workspace-core",
 976             "src/tools/rustc-std-workspace-alloc",
 977             "src/tools/rustc-std-workspace-std",
 978             "src/librustc",
 979             "src/libsyntax",
 980         ];
 981
 982         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 983         for file in src_files.iter() {
 984             builder.copy(&builder.src.join(file), &dst_src.join(file));
 985         }
 986
 987         // Create source tarball in rust-installer format
 988         let mut cmd = rust_installer(builder);
 989         cmd.arg("generate")
 990            .arg("--product-name=Rust")
 991            .arg("--rel-manifest-dir=rustlib")
 992            .arg("--success-message=Awesome-Source.")
 993            .arg("--image-dir").arg(&image)
 994            .arg("--work-dir").arg(&tmpdir(builder))
 995            .arg("--output-dir").arg(&distdir(builder))
 996            .arg(format!("--package-name={}", name))
 997            .arg("--component-name=rust-src")
 998            .arg("--legacy-manifest-dirs=rustlib,cargo");
 999
1000         builder.info("Dist src");
1001         let _time = timeit(builder);
1002         builder.run(&mut cmd);
1003
1004         builder.remove_dir(&image);
1005         distdir(builder).join(&format!("{}.tar.gz", name))
1006     }
1007 }
1008
1009 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1010 pub struct PlainSourceTarball;
1011
1012 impl Step for PlainSourceTarball {
1013     /// Produces the location of the tarball generated
1014     type Output = PathBuf;
1015     const DEFAULT: bool = true;
1016     const ONLY_HOSTS: bool = true;
1017
1018     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1019         let builder = run.builder;
1020         run.path("src").default_condition(builder.config.rust_dist_src)
1021     }
1022
1023     fn make_run(run: RunConfig<'_>) {
1024         run.builder.ensure(PlainSourceTarball);
1025     }
1026
1027     /// Creates the plain source tarball
1028     fn run(self, builder: &Builder<'_>) -> PathBuf {
1029         // Make sure that the root folder of tarball has the correct name
1030         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
1031         let plain_dst_src = tmpdir(builder).join(&plain_name);
1032         let _ = fs::remove_dir_all(&plain_dst_src);
1033         t!(fs::create_dir_all(&plain_dst_src));
1034
1035         // This is the set of root paths which will become part of the source package
1036         let src_files = [
1037             "COPYRIGHT",
1038             "LICENSE-APACHE",
1039             "LICENSE-MIT",
1040             "CONTRIBUTING.md",
1041             "README.md",
1042             "RELEASES.md",
1043             "configure",
1044             "x.py",
1045             "config.toml.example",
1046             "Cargo.toml",
1047             "Cargo.lock",
1048         ];
1049         let src_dirs = [
1050             "src",
1051         ];
1052
1053         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
1054
1055         // Copy the files normally
1056         for item in &src_files {
1057             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
1058         }
1059
1060         // Create the version file
1061         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
1062         if let Some(sha) = builder.rust_sha() {
1063             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
1064         }
1065
1066         // If we're building from git sources, we need to vendor a complete distribution.
1067         if builder.rust_info.is_git() {
1068             // Vendor all Cargo dependencies
1069             let mut cmd = Command::new(&builder.initial_cargo);
1070             cmd.arg("vendor")
1071                .current_dir(&plain_dst_src);
1072             builder.run(&mut cmd);
1073         }
1074
1075         // Create plain source tarball
1076         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1077         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1078         tarball.set_extension(""); // strip .gz
1079         tarball.set_extension(""); // strip .tar
1080         if let Some(dir) = tarball.parent() {
1081             builder.create_dir(&dir);
1082         }
1083         builder.info("running installer");
1084         let mut cmd = rust_installer(builder);
1085         cmd.arg("tarball")
1086            .arg("--input").arg(&plain_name)
1087            .arg("--output").arg(&tarball)
1088            .arg("--work-dir=.")
1089            .current_dir(tmpdir(builder));
1090
1091         builder.info("Create plain source tarball");
1092         let _time = timeit(builder);
1093         builder.run(&mut cmd);
1094         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1095     }
1096 }
1097
1098 // We have to run a few shell scripts, which choke quite a bit on both `\`
1099 // characters and on `C:\` paths, so normalize both of them away.
1100 pub fn sanitize_sh(path: &Path) -> String {
1101     let path = path.to_str().unwrap().replace("\\", "/");
1102     return change_drive(&path).unwrap_or(path);
1103
1104     fn change_drive(s: &str) -> Option<String> {
1105         let mut ch = s.chars();
1106         let drive = ch.next().unwrap_or('C');
1107         if ch.next() != Some(':') {
1108             return None
1109         }
1110         if ch.next() != Some('/') {
1111             return None
1112         }
1113         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1114     }
1115 }
1116
1117 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1118 pub struct Cargo {
1119     pub compiler: Compiler,
1120     pub target: Interned<String>,
1121 }
1122
1123 impl Step for Cargo {
1124     type Output = PathBuf;
1125     const ONLY_HOSTS: bool = true;
1126
1127     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1128         run.path("cargo")
1129     }
1130
1131     fn make_run(run: RunConfig<'_>) {
1132         run.builder.ensure(Cargo {
1133             compiler: run.builder.compiler_for(
1134                 run.builder.top_stage,
1135                 run.builder.config.build,
1136                 run.target,
1137             ),
1138             target: run.target,
1139         });
1140     }
1141
1142     fn run(self, builder: &Builder<'_>) -> PathBuf {
1143         let compiler = self.compiler;
1144         let target = self.target;
1145
1146         let src = builder.src.join("src/tools/cargo");
1147         let etc = src.join("src/etc");
1148         let release_num = builder.release_num("cargo");
1149         let name = pkgname(builder, "cargo");
1150         let version = builder.cargo_info.version(builder, &release_num);
1151
1152         let tmp = tmpdir(builder);
1153         let image = tmp.join("cargo-image");
1154         drop(fs::remove_dir_all(&image));
1155         builder.create_dir(&image);
1156
1157         // Prepare the image directory
1158         builder.create_dir(&image.join("share/zsh/site-functions"));
1159         builder.create_dir(&image.join("etc/bash_completion.d"));
1160         let cargo = builder.ensure(tool::Cargo { compiler, target });
1161         builder.install(&cargo, &image.join("bin"), 0o755);
1162         for man in t!(etc.join("man").read_dir()) {
1163             let man = t!(man);
1164             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1165         }
1166         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1167         builder.copy(&etc.join("cargo.bashcomp.sh"),
1168              &image.join("etc/bash_completion.d/cargo"));
1169         let doc = image.join("share/doc/cargo");
1170         builder.install(&src.join("README.md"), &doc, 0o644);
1171         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1172         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1173         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1174
1175         // Prepare the overlay
1176         let overlay = tmp.join("cargo-overlay");
1177         drop(fs::remove_dir_all(&overlay));
1178         builder.create_dir(&overlay);
1179         builder.install(&src.join("README.md"), &overlay, 0o644);
1180         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1181         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1182         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1183         builder.create(&overlay.join("version"), &version);
1184
1185         // Generate the installer tarball
1186         let mut cmd = rust_installer(builder);
1187         cmd.arg("generate")
1188            .arg("--product-name=Rust")
1189            .arg("--rel-manifest-dir=rustlib")
1190            .arg("--success-message=Rust-is-ready-to-roll.")
1191            .arg("--image-dir").arg(&image)
1192            .arg("--work-dir").arg(&tmpdir(builder))
1193            .arg("--output-dir").arg(&distdir(builder))
1194            .arg("--non-installed-overlay").arg(&overlay)
1195            .arg(format!("--package-name={}-{}", name, target))
1196            .arg("--component-name=cargo")
1197            .arg("--legacy-manifest-dirs=rustlib,cargo");
1198
1199         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1200         let _time = timeit(builder);
1201         builder.run(&mut cmd);
1202         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1203     }
1204 }
1205
1206 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1207 pub struct Rls {
1208     pub compiler: Compiler,
1209     pub target: Interned<String>,
1210 }
1211
1212 impl Step for Rls {
1213     type Output = Option<PathBuf>;
1214     const ONLY_HOSTS: bool = true;
1215
1216     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1217         run.path("rls")
1218     }
1219
1220     fn make_run(run: RunConfig<'_>) {
1221         run.builder.ensure(Rls {
1222             compiler: run.builder.compiler_for(
1223                 run.builder.top_stage,
1224                 run.builder.config.build,
1225                 run.target,
1226             ),
1227             target: run.target,
1228         });
1229     }
1230
1231     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1232         let compiler = self.compiler;
1233         let target = self.target;
1234         assert!(builder.config.extended);
1235
1236         let src = builder.src.join("src/tools/rls");
1237         let release_num = builder.release_num("rls");
1238         let name = pkgname(builder, "rls");
1239         let version = builder.rls_info.version(builder, &release_num);
1240
1241         let tmp = tmpdir(builder);
1242         let image = tmp.join("rls-image");
1243         drop(fs::remove_dir_all(&image));
1244         t!(fs::create_dir_all(&image));
1245
1246         // Prepare the image directory
1247         // We expect RLS to build, because we've exited this step above if tool
1248         // state for RLS isn't testing.
1249         let rls = builder.ensure(tool::Rls {
1250             compiler,
1251             target,
1252             extra_features: Vec::new(),
1253         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1254
1255         builder.install(&rls, &image.join("bin"), 0o755);
1256         let doc = image.join("share/doc/rls");
1257         builder.install(&src.join("README.md"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1259         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1260
1261         // Prepare the overlay
1262         let overlay = tmp.join("rls-overlay");
1263         drop(fs::remove_dir_all(&overlay));
1264         t!(fs::create_dir_all(&overlay));
1265         builder.install(&src.join("README.md"), &overlay, 0o644);
1266         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1267         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1268         builder.create(&overlay.join("version"), &version);
1269
1270         // Generate the installer tarball
1271         let mut cmd = rust_installer(builder);
1272         cmd.arg("generate")
1273            .arg("--product-name=Rust")
1274            .arg("--rel-manifest-dir=rustlib")
1275            .arg("--success-message=RLS-ready-to-serve.")
1276            .arg("--image-dir").arg(&image)
1277            .arg("--work-dir").arg(&tmpdir(builder))
1278            .arg("--output-dir").arg(&distdir(builder))
1279            .arg("--non-installed-overlay").arg(&overlay)
1280            .arg(format!("--package-name={}-{}", name, target))
1281            .arg("--legacy-manifest-dirs=rustlib,cargo")
1282            .arg("--component-name=rls-preview");
1283
1284         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1285         let _time = timeit(builder);
1286         builder.run(&mut cmd);
1287         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1288     }
1289 }
1290
1291 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1292 pub struct Clippy {
1293     pub compiler: Compiler,
1294     pub target: Interned<String>,
1295 }
1296
1297 impl Step for Clippy {
1298     type Output = Option<PathBuf>;
1299     const ONLY_HOSTS: bool = true;
1300
1301     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1302         run.path("clippy")
1303     }
1304
1305     fn make_run(run: RunConfig<'_>) {
1306         run.builder.ensure(Clippy {
1307             compiler: run.builder.compiler_for(
1308                 run.builder.top_stage,
1309                 run.builder.config.build,
1310                 run.target,
1311             ),
1312             target: run.target,
1313         });
1314     }
1315
1316     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1317         let compiler = self.compiler;
1318         let target = self.target;
1319         assert!(builder.config.extended);
1320
1321         let src = builder.src.join("src/tools/clippy");
1322         let release_num = builder.release_num("clippy");
1323         let name = pkgname(builder, "clippy");
1324         let version = builder.clippy_info.version(builder, &release_num);
1325
1326         let tmp = tmpdir(builder);
1327         let image = tmp.join("clippy-image");
1328         drop(fs::remove_dir_all(&image));
1329         builder.create_dir(&image);
1330
1331         // Prepare the image directory
1332         // We expect clippy to build, because we've exited this step above if tool
1333         // state for clippy isn't testing.
1334         let clippy = builder.ensure(tool::Clippy {
1335             compiler,
1336             target,
1337             extra_features: Vec::new(),
1338         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1339         let cargoclippy = builder.ensure(tool::CargoClippy {
1340             compiler,
1341             target, extra_features: Vec::new()
1342         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1343
1344         builder.install(&clippy, &image.join("bin"), 0o755);
1345         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1346         let doc = image.join("share/doc/clippy");
1347         builder.install(&src.join("README.md"), &doc, 0o644);
1348         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1349         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1350
1351         // Prepare the overlay
1352         let overlay = tmp.join("clippy-overlay");
1353         drop(fs::remove_dir_all(&overlay));
1354         t!(fs::create_dir_all(&overlay));
1355         builder.install(&src.join("README.md"), &overlay, 0o644);
1356         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1357         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1358         builder.create(&overlay.join("version"), &version);
1359
1360         // Generate the installer tarball
1361         let mut cmd = rust_installer(builder);
1362         cmd.arg("generate")
1363            .arg("--product-name=Rust")
1364            .arg("--rel-manifest-dir=rustlib")
1365            .arg("--success-message=clippy-ready-to-serve.")
1366            .arg("--image-dir").arg(&image)
1367            .arg("--work-dir").arg(&tmpdir(builder))
1368            .arg("--output-dir").arg(&distdir(builder))
1369            .arg("--non-installed-overlay").arg(&overlay)
1370            .arg(format!("--package-name={}-{}", name, target))
1371            .arg("--legacy-manifest-dirs=rustlib,cargo")
1372            .arg("--component-name=clippy-preview");
1373
1374         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1375         let _time = timeit(builder);
1376         builder.run(&mut cmd);
1377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1378     }
1379 }
1380
1381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1382 pub struct Miri {
1383     pub compiler: Compiler,
1384     pub target: Interned<String>,
1385 }
1386
1387 impl Step for Miri {
1388     type Output = Option<PathBuf>;
1389     const ONLY_HOSTS: bool = true;
1390
1391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1392         run.path("miri")
1393     }
1394
1395     fn make_run(run: RunConfig<'_>) {
1396         run.builder.ensure(Miri {
1397             compiler: run.builder.compiler_for(
1398                 run.builder.top_stage,
1399                 run.builder.config.build,
1400                 run.target,
1401             ),
1402             target: run.target,
1403         });
1404     }
1405
1406     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1407         let compiler = self.compiler;
1408         let target = self.target;
1409         assert!(builder.config.extended);
1410
1411         let src = builder.src.join("src/tools/miri");
1412         let release_num = builder.release_num("miri");
1413         let name = pkgname(builder, "miri");
1414         let version = builder.miri_info.version(builder, &release_num);
1415
1416         let tmp = tmpdir(builder);
1417         let image = tmp.join("miri-image");
1418         drop(fs::remove_dir_all(&image));
1419         builder.create_dir(&image);
1420
1421         // Prepare the image directory
1422         // We expect miri to build, because we've exited this step above if tool
1423         // state for miri isn't testing.
1424         let miri = builder.ensure(tool::Miri {
1425             compiler,
1426             target,
1427             extra_features: Vec::new(),
1428         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1429         let cargomiri = builder.ensure(tool::CargoMiri {
1430             compiler,
1431             target,
1432             extra_features: Vec::new()
1433         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1434
1435         builder.install(&miri, &image.join("bin"), 0o755);
1436         builder.install(&cargomiri, &image.join("bin"), 0o755);
1437         let doc = image.join("share/doc/miri");
1438         builder.install(&src.join("README.md"), &doc, 0o644);
1439         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1440         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1441
1442         // Prepare the overlay
1443         let overlay = tmp.join("miri-overlay");
1444         drop(fs::remove_dir_all(&overlay));
1445         t!(fs::create_dir_all(&overlay));
1446         builder.install(&src.join("README.md"), &overlay, 0o644);
1447         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1448         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1449         builder.create(&overlay.join("version"), &version);
1450
1451         // Generate the installer tarball
1452         let mut cmd = rust_installer(builder);
1453         cmd.arg("generate")
1454            .arg("--product-name=Rust")
1455            .arg("--rel-manifest-dir=rustlib")
1456            .arg("--success-message=miri-ready-to-serve.")
1457            .arg("--image-dir").arg(&image)
1458            .arg("--work-dir").arg(&tmpdir(builder))
1459            .arg("--output-dir").arg(&distdir(builder))
1460            .arg("--non-installed-overlay").arg(&overlay)
1461            .arg(format!("--package-name={}-{}", name, target))
1462            .arg("--legacy-manifest-dirs=rustlib,cargo")
1463            .arg("--component-name=miri-preview");
1464
1465         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1466         let _time = timeit(builder);
1467         builder.run(&mut cmd);
1468         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1469     }
1470 }
1471
1472 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1473 pub struct Rustfmt {
1474     pub compiler: Compiler,
1475     pub target: Interned<String>,
1476 }
1477
1478 impl Step for Rustfmt {
1479     type Output = Option<PathBuf>;
1480     const ONLY_HOSTS: bool = true;
1481
1482     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1483         run.path("rustfmt")
1484     }
1485
1486     fn make_run(run: RunConfig<'_>) {
1487         run.builder.ensure(Rustfmt {
1488             compiler: run.builder.compiler_for(
1489                 run.builder.top_stage,
1490                 run.builder.config.build,
1491                 run.target,
1492             ),
1493             target: run.target,
1494         });
1495     }
1496
1497     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1498         let compiler = self.compiler;
1499         let target = self.target;
1500
1501         let src = builder.src.join("src/tools/rustfmt");
1502         let release_num = builder.release_num("rustfmt");
1503         let name = pkgname(builder, "rustfmt");
1504         let version = builder.rustfmt_info.version(builder, &release_num);
1505
1506         let tmp = tmpdir(builder);
1507         let image = tmp.join("rustfmt-image");
1508         drop(fs::remove_dir_all(&image));
1509         builder.create_dir(&image);
1510
1511         // Prepare the image directory
1512         let rustfmt = builder.ensure(tool::Rustfmt {
1513             compiler,
1514             target,
1515             extra_features: Vec::new(),
1516         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1517         let cargofmt = builder.ensure(tool::Cargofmt {
1518             compiler,
1519             target,
1520             extra_features: Vec::new(),
1521         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1522
1523         builder.install(&rustfmt, &image.join("bin"), 0o755);
1524         builder.install(&cargofmt, &image.join("bin"), 0o755);
1525         let doc = image.join("share/doc/rustfmt");
1526         builder.install(&src.join("README.md"), &doc, 0o644);
1527         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1528         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1529
1530         // Prepare the overlay
1531         let overlay = tmp.join("rustfmt-overlay");
1532         drop(fs::remove_dir_all(&overlay));
1533         builder.create_dir(&overlay);
1534         builder.install(&src.join("README.md"), &overlay, 0o644);
1535         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1536         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1537         builder.create(&overlay.join("version"), &version);
1538
1539         // Generate the installer tarball
1540         let mut cmd = rust_installer(builder);
1541         cmd.arg("generate")
1542            .arg("--product-name=Rust")
1543            .arg("--rel-manifest-dir=rustlib")
1544            .arg("--success-message=rustfmt-ready-to-fmt.")
1545            .arg("--image-dir").arg(&image)
1546            .arg("--work-dir").arg(&tmpdir(builder))
1547            .arg("--output-dir").arg(&distdir(builder))
1548            .arg("--non-installed-overlay").arg(&overlay)
1549            .arg(format!("--package-name={}-{}", name, target))
1550            .arg("--legacy-manifest-dirs=rustlib,cargo")
1551            .arg("--component-name=rustfmt-preview");
1552
1553         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1554         let _time = timeit(builder);
1555         builder.run(&mut cmd);
1556         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1557     }
1558 }
1559
1560 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1561 pub struct Extended {
1562     stage: u32,
1563     host: Interned<String>,
1564     target: Interned<String>,
1565 }
1566
1567 impl Step for Extended {
1568     type Output = ();
1569     const DEFAULT: bool = true;
1570     const ONLY_HOSTS: bool = true;
1571
1572     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1573         let builder = run.builder;
1574         run.path("extended").default_condition(builder.config.extended)
1575     }
1576
1577     fn make_run(run: RunConfig<'_>) {
1578         run.builder.ensure(Extended {
1579             stage: run.builder.top_stage,
1580             host: run.builder.config.build,
1581             target: run.target,
1582         });
1583     }
1584
1585     /// Creates a combined installer for the specified target in the provided stage.
1586     fn run(self, builder: &Builder<'_>) {
1587         let target = self.target;
1588         let stage = self.stage;
1589         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1590
1591         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1592
1593         let rustc_installer = builder.ensure(Rustc {
1594             compiler: builder.compiler(stage, target),
1595         });
1596         let cargo_installer = builder.ensure(Cargo { compiler, target });
1597         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1598         let rls_installer = builder.ensure(Rls { compiler, target });
1599         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1600         let clippy_installer = builder.ensure(Clippy { compiler, target });
1601         let miri_installer = builder.ensure(Miri { compiler, target });
1602         let lldb_installer = builder.ensure(Lldb { target });
1603         let mingw_installer = builder.ensure(Mingw { host: target });
1604         let analysis_installer = builder.ensure(Analysis { compiler, target });
1605
1606         let docs_installer = builder.ensure(Docs { host: target, });
1607         let std_installer = builder.ensure(Std {
1608             compiler: builder.compiler(stage, target),
1609             target,
1610         });
1611
1612         let tmp = tmpdir(builder);
1613         let overlay = tmp.join("extended-overlay");
1614         let etc = builder.src.join("src/etc/installer");
1615         let work = tmp.join("work");
1616
1617         let _ = fs::remove_dir_all(&overlay);
1618         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1619         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1620         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1621         let version = builder.rust_version();
1622         builder.create(&overlay.join("version"), &version);
1623         if let Some(sha) = builder.rust_sha() {
1624             builder.create(&overlay.join("git-commit-hash"), &sha);
1625         }
1626         builder.install(&etc.join("README.md"), &overlay, 0o644);
1627
1628         // When rust-std package split from rustc, we needed to ensure that during
1629         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1630         // the std files during uninstall. To do this ensure that rustc comes
1631         // before rust-std in the list below.
1632         let mut tarballs = Vec::new();
1633         tarballs.push(rustc_installer);
1634         tarballs.push(cargo_installer);
1635         tarballs.extend(rls_installer.clone());
1636         tarballs.extend(clippy_installer.clone());
1637         tarballs.extend(miri_installer.clone());
1638         tarballs.extend(rustfmt_installer.clone());
1639         tarballs.extend(llvm_tools_installer);
1640         tarballs.extend(lldb_installer);
1641         tarballs.push(analysis_installer);
1642         tarballs.push(std_installer);
1643         if builder.config.docs {
1644             tarballs.push(docs_installer);
1645         }
1646         if target.contains("pc-windows-gnu") {
1647             tarballs.push(mingw_installer.unwrap());
1648         }
1649         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1650         for tarball in &tarballs[1..] {
1651             input_tarballs.push(",");
1652             input_tarballs.push(tarball);
1653         }
1654
1655         builder.info("building combined installer");
1656         let mut cmd = rust_installer(builder);
1657         cmd.arg("combine")
1658             .arg("--product-name=Rust")
1659             .arg("--rel-manifest-dir=rustlib")
1660             .arg("--success-message=Rust-is-ready-to-roll.")
1661             .arg("--work-dir").arg(&work)
1662             .arg("--output-dir").arg(&distdir(builder))
1663             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1664             .arg("--legacy-manifest-dirs=rustlib,cargo")
1665             .arg("--input-tarballs").arg(input_tarballs)
1666             .arg("--non-installed-overlay").arg(&overlay);
1667         let time = timeit(&builder);
1668         builder.run(&mut cmd);
1669         drop(time);
1670
1671         let mut license = String::new();
1672         license += &builder.read(&builder.src.join("COPYRIGHT"));
1673         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1674         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1675         license.push_str("\n");
1676         license.push_str("\n");
1677
1678         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1679         let mut rtf = rtf.to_string();
1680         rtf.push_str("\n");
1681         for line in license.lines() {
1682             rtf.push_str(line);
1683             rtf.push_str("\\line ");
1684         }
1685         rtf.push_str("}");
1686
1687         fn filter(contents: &str, marker: &str) -> String {
1688             let start = format!("tool-{}-start", marker);
1689             let end = format!("tool-{}-end", marker);
1690             let mut lines = Vec::new();
1691             let mut omitted = false;
1692             for line in contents.lines() {
1693                 if line.contains(&start) {
1694                     omitted = true;
1695                 } else if line.contains(&end) {
1696                     omitted = false;
1697                 } else if !omitted {
1698                     lines.push(line);
1699                 }
1700             }
1701
1702             lines.join("\n")
1703         }
1704
1705         let xform = |p: &Path| {
1706             let mut contents = t!(fs::read_to_string(p));
1707             if rls_installer.is_none() {
1708                 contents = filter(&contents, "rls");
1709             }
1710             if clippy_installer.is_none() {
1711                 contents = filter(&contents, "clippy");
1712             }
1713             if miri_installer.is_none() {
1714                 contents = filter(&contents, "miri");
1715             }
1716             if rustfmt_installer.is_none() {
1717                 contents = filter(&contents, "rustfmt");
1718             }
1719             let ret = tmp.join(p.file_name().unwrap());
1720             t!(fs::write(&ret, &contents));
1721             ret
1722         };
1723
1724         if target.contains("apple-darwin") {
1725             builder.info("building pkg installer");
1726             let pkg = tmp.join("pkg");
1727             let _ = fs::remove_dir_all(&pkg);
1728
1729             let pkgbuild = |component: &str| {
1730                 let mut cmd = Command::new("pkgbuild");
1731                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1732                     .arg("--scripts").arg(pkg.join(component))
1733                     .arg("--nopayload")
1734                     .arg(pkg.join(component).with_extension("pkg"));
1735                 builder.run(&mut cmd);
1736             };
1737
1738             let prepare = |name: &str| {
1739                 builder.create_dir(&pkg.join(name));
1740                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1741                         &pkg.join(name));
1742                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1743                 pkgbuild(name);
1744             };
1745             prepare("rustc");
1746             prepare("cargo");
1747             prepare("rust-docs");
1748             prepare("rust-std");
1749             prepare("rust-analysis");
1750
1751             if rls_installer.is_some() {
1752                 prepare("rls");
1753             }
1754             if clippy_installer.is_some() {
1755                 prepare("clippy");
1756             }
1757             if miri_installer.is_some() {
1758                 prepare("miri");
1759             }
1760
1761             // create an 'uninstall' package
1762             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1763             pkgbuild("uninstall");
1764
1765             builder.create_dir(&pkg.join("res"));
1766             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1767             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1768             let mut cmd = Command::new("productbuild");
1769             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1770                 .arg("--resources").arg(pkg.join("res"))
1771                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1772                                                     pkgname(builder, "rust"),
1773                                                     target)))
1774                 .arg("--package-path").arg(&pkg);
1775             let _time = timeit(builder);
1776             builder.run(&mut cmd);
1777         }
1778
1779         if target.contains("windows") {
1780             let exe = tmp.join("exe");
1781             let _ = fs::remove_dir_all(&exe);
1782
1783             let prepare = |name: &str| {
1784                 builder.create_dir(&exe.join(name));
1785                 let dir = if name == "rust-std" || name == "rust-analysis" {
1786                     format!("{}-{}", name, target)
1787                 } else if name == "rls" {
1788                     "rls-preview".to_string()
1789                 } else if name == "clippy" {
1790                     "clippy-preview".to_string()
1791                 } else if name == "miri" {
1792                     "miri-preview".to_string()
1793                 } else {
1794                     name.to_string()
1795                 };
1796                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1797                             .join(dir),
1798                         &exe.join(name));
1799                 builder.remove(&exe.join(name).join("manifest.in"));
1800             };
1801             prepare("rustc");
1802             prepare("cargo");
1803             prepare("rust-analysis");
1804             prepare("rust-docs");
1805             prepare("rust-std");
1806             if rls_installer.is_some() {
1807                 prepare("rls");
1808             }
1809             if clippy_installer.is_some() {
1810                 prepare("clippy");
1811             }
1812             if miri_installer.is_some() {
1813                 prepare("miri");
1814             }
1815             if target.contains("windows-gnu") {
1816                 prepare("rust-mingw");
1817             }
1818
1819             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1820             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1821             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1822             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1823             builder.create(&exe.join("LICENSE.txt"), &license);
1824
1825             // Generate exe installer
1826             builder.info("building `exe` installer with `iscc`");
1827             let mut cmd = Command::new("iscc");
1828             cmd.arg("rust.iss")
1829                 .arg("/Q")
1830                 .current_dir(&exe);
1831             if target.contains("windows-gnu") {
1832                 cmd.arg("/dMINGW");
1833             }
1834             add_env(builder, &mut cmd, target);
1835             let time = timeit(builder);
1836             builder.run(&mut cmd);
1837             drop(time);
1838             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1839                     &distdir(builder),
1840                     0o755);
1841
1842             // Generate msi installer
1843             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1844             let heat = wix.join("bin/heat.exe");
1845             let candle = wix.join("bin/candle.exe");
1846             let light = wix.join("bin/light.exe");
1847
1848             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1849             builder.run(Command::new(&heat)
1850                             .current_dir(&exe)
1851                             .arg("dir")
1852                             .arg("rustc")
1853                             .args(&heat_flags)
1854                             .arg("-cg").arg("RustcGroup")
1855                             .arg("-dr").arg("Rustc")
1856                             .arg("-var").arg("var.RustcDir")
1857                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1858             builder.run(Command::new(&heat)
1859                             .current_dir(&exe)
1860                             .arg("dir")
1861                             .arg("rust-docs")
1862                             .args(&heat_flags)
1863                             .arg("-cg").arg("DocsGroup")
1864                             .arg("-dr").arg("Docs")
1865                             .arg("-var").arg("var.DocsDir")
1866                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1867                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1868             builder.run(Command::new(&heat)
1869                             .current_dir(&exe)
1870                             .arg("dir")
1871                             .arg("cargo")
1872                             .args(&heat_flags)
1873                             .arg("-cg").arg("CargoGroup")
1874                             .arg("-dr").arg("Cargo")
1875                             .arg("-var").arg("var.CargoDir")
1876                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1877                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1878             builder.run(Command::new(&heat)
1879                             .current_dir(&exe)
1880                             .arg("dir")
1881                             .arg("rust-std")
1882                             .args(&heat_flags)
1883                             .arg("-cg").arg("StdGroup")
1884                             .arg("-dr").arg("Std")
1885                             .arg("-var").arg("var.StdDir")
1886                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1887             if rls_installer.is_some() {
1888                 builder.run(Command::new(&heat)
1889                                 .current_dir(&exe)
1890                                 .arg("dir")
1891                                 .arg("rls")
1892                                 .args(&heat_flags)
1893                                 .arg("-cg").arg("RlsGroup")
1894                                 .arg("-dr").arg("Rls")
1895                                 .arg("-var").arg("var.RlsDir")
1896                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1897                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1898             }
1899             if clippy_installer.is_some() {
1900                 builder.run(Command::new(&heat)
1901                                 .current_dir(&exe)
1902                                 .arg("dir")
1903                                 .arg("clippy")
1904                                 .args(&heat_flags)
1905                                 .arg("-cg").arg("ClippyGroup")
1906                                 .arg("-dr").arg("Clippy")
1907                                 .arg("-var").arg("var.ClippyDir")
1908                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1909                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1910             }
1911             if miri_installer.is_some() {
1912                 builder.run(Command::new(&heat)
1913                                 .current_dir(&exe)
1914                                 .arg("dir")
1915                                 .arg("miri")
1916                                 .args(&heat_flags)
1917                                 .arg("-cg").arg("MiriGroup")
1918                                 .arg("-dr").arg("Miri")
1919                                 .arg("-var").arg("var.MiriDir")
1920                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1921                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1922             }
1923             builder.run(Command::new(&heat)
1924                             .current_dir(&exe)
1925                             .arg("dir")
1926                             .arg("rust-analysis")
1927                             .args(&heat_flags)
1928                             .arg("-cg").arg("AnalysisGroup")
1929                             .arg("-dr").arg("Analysis")
1930                             .arg("-var").arg("var.AnalysisDir")
1931                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1932                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1933             if target.contains("windows-gnu") {
1934                 builder.run(Command::new(&heat)
1935                                 .current_dir(&exe)
1936                                 .arg("dir")
1937                                 .arg("rust-mingw")
1938                                 .args(&heat_flags)
1939                                 .arg("-cg").arg("GccGroup")
1940                                 .arg("-dr").arg("Gcc")
1941                                 .arg("-var").arg("var.GccDir")
1942                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1943             }
1944
1945             let candle = |input: &Path| {
1946                 let output = exe.join(input.file_stem().unwrap())
1947                                 .with_extension("wixobj");
1948                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1949                 let mut cmd = Command::new(&candle);
1950                 cmd.current_dir(&exe)
1951                     .arg("-nologo")
1952                     .arg("-dRustcDir=rustc")
1953                     .arg("-dDocsDir=rust-docs")
1954                     .arg("-dCargoDir=cargo")
1955                     .arg("-dStdDir=rust-std")
1956                     .arg("-dAnalysisDir=rust-analysis")
1957                     .arg("-arch").arg(&arch)
1958                     .arg("-out").arg(&output)
1959                     .arg(&input);
1960                 add_env(builder, &mut cmd, target);
1961
1962                 if rls_installer.is_some() {
1963                     cmd.arg("-dRlsDir=rls");
1964                 }
1965                 if clippy_installer.is_some() {
1966                     cmd.arg("-dClippyDir=clippy");
1967                 }
1968                 if miri_installer.is_some() {
1969                     cmd.arg("-dMiriDir=miri");
1970                 }
1971                 if target.contains("windows-gnu") {
1972                     cmd.arg("-dGccDir=rust-mingw");
1973                 }
1974                 builder.run(&mut cmd);
1975             };
1976             candle(&xform(&etc.join("msi/rust.wxs")));
1977             candle(&etc.join("msi/ui.wxs"));
1978             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1979             candle("RustcGroup.wxs".as_ref());
1980             candle("DocsGroup.wxs".as_ref());
1981             candle("CargoGroup.wxs".as_ref());
1982             candle("StdGroup.wxs".as_ref());
1983             if rls_installer.is_some() {
1984                 candle("RlsGroup.wxs".as_ref());
1985             }
1986             if clippy_installer.is_some() {
1987                 candle("ClippyGroup.wxs".as_ref());
1988             }
1989             if miri_installer.is_some() {
1990                 candle("MiriGroup.wxs".as_ref());
1991             }
1992             candle("AnalysisGroup.wxs".as_ref());
1993
1994             if target.contains("windows-gnu") {
1995                 candle("GccGroup.wxs".as_ref());
1996             }
1997
1998             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1999             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
2000             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
2001
2002             builder.info(&format!("building `msi` installer with {:?}", light));
2003             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
2004             let mut cmd = Command::new(&light);
2005             cmd.arg("-nologo")
2006                 .arg("-ext").arg("WixUIExtension")
2007                 .arg("-ext").arg("WixUtilExtension")
2008                 .arg("-out").arg(exe.join(&filename))
2009                 .arg("rust.wixobj")
2010                 .arg("ui.wixobj")
2011                 .arg("rustwelcomedlg.wixobj")
2012                 .arg("RustcGroup.wixobj")
2013                 .arg("DocsGroup.wixobj")
2014                 .arg("CargoGroup.wixobj")
2015                 .arg("StdGroup.wixobj")
2016                 .arg("AnalysisGroup.wixobj")
2017                 .current_dir(&exe);
2018
2019             if rls_installer.is_some() {
2020                 cmd.arg("RlsGroup.wixobj");
2021             }
2022             if clippy_installer.is_some() {
2023                 cmd.arg("ClippyGroup.wixobj");
2024             }
2025             if miri_installer.is_some() {
2026                 cmd.arg("MiriGroup.wixobj");
2027             }
2028
2029             if target.contains("windows-gnu") {
2030                 cmd.arg("GccGroup.wixobj");
2031             }
2032             // ICE57 wrongly complains about the shortcuts
2033             cmd.arg("-sice:ICE57");
2034
2035             let _time = timeit(builder);
2036             builder.run(&mut cmd);
2037
2038             if !builder.config.dry_run {
2039                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
2040             }
2041         }
2042     }
2043 }
2044
2045 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
2046     let mut parts = channel::CFG_RELEASE_NUM.split('.');
2047     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
2048        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
2049        .env("CFG_RELEASE", builder.rust_release())
2050        .env("CFG_VER_MAJOR", parts.next().unwrap())
2051        .env("CFG_VER_MINOR", parts.next().unwrap())
2052        .env("CFG_VER_PATCH", parts.next().unwrap())
2053        .env("CFG_VER_BUILD", "0") // just needed to build
2054        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
2055        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
2056        .env("CFG_BUILD", target)
2057        .env("CFG_CHANNEL", &builder.config.channel);
2058
2059     if target.contains("windows-gnu") {
2060        cmd.env("CFG_MINGW", "1")
2061           .env("CFG_ABI", "GNU");
2062     } else {
2063        cmd.env("CFG_MINGW", "0")
2064           .env("CFG_ABI", "MSVC");
2065     }
2066
2067     if target.contains("x86_64") {
2068        cmd.env("CFG_PLATFORM", "x64");
2069     } else {
2070        cmd.env("CFG_PLATFORM", "x86");
2071     }
2072 }
2073
2074 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
2075 pub struct HashSign;
2076
2077 impl Step for HashSign {
2078     type Output = ();
2079     const ONLY_HOSTS: bool = true;
2080
2081     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2082         run.path("hash-and-sign")
2083     }
2084
2085     fn make_run(run: RunConfig<'_>) {
2086         run.builder.ensure(HashSign);
2087     }
2088
2089     fn run(self, builder: &Builder<'_>) {
2090         // This gets called by `promote-release`
2091         // (https://github.com/rust-lang/rust-central-station/tree/master/promote-release).
2092         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2093         if builder.config.dry_run {
2094             return;
2095         }
2096         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2097             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2098         });
2099         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2100             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2101         });
2102         let pass = if env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err() {
2103             let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2104                 panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2105             });
2106             t!(fs::read_to_string(&file))
2107         } else {
2108             String::new()
2109         };
2110
2111         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2112
2113         cmd.arg(sign);
2114         cmd.arg(distdir(builder));
2115         cmd.arg(today.trim());
2116         cmd.arg(builder.rust_package_vers());
2117         cmd.arg(addr);
2118         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2119         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2120         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2121         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2122         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2123         cmd.arg(builder.llvm_tools_package_vers());
2124         cmd.arg(builder.lldb_package_vers());
2125
2126         builder.create_dir(&distdir(builder));
2127
2128         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2129         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2130         let status = t!(child.wait());
2131         assert!(status.success());
2132     }
2133 }
2134
2135 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2136 // LLVM tools are linked dynamically.
2137 // Note: This function does no yet support Windows but we also don't support
2138 //       linking LLVM tools dynamically on Windows yet.
2139 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2140                                 target: Interned<String>,
2141                                 sysroot: &Path) {
2142     let src_libdir = builder
2143         .llvm_out(target)
2144         .join("lib");
2145     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2146     t!(fs::create_dir_all(&dst_libdir));
2147
2148     if target.contains("apple-darwin") {
2149         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2150         if llvm_dylib_path.exists() {
2151             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2152         }
2153         return
2154     }
2155
2156     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2157     // Since tools link to the latter rather than the former, we have to
2158     // follow the symlink to find out what to distribute.
2159     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2160     if llvm_dylib_path.exists() {
2161         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2162             panic!("dist: Error calling canonicalize path `{}`: {}",
2163                    llvm_dylib_path.display(), e);
2164         });
2165
2166
2167         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2168     }
2169 }
2170
2171 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2172 pub struct LlvmTools {
2173     pub target: Interned<String>,
2174 }
2175
2176 impl Step for LlvmTools {
2177     type Output = Option<PathBuf>;
2178     const ONLY_HOSTS: bool = true;
2179
2180     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2181         run.path("llvm-tools")
2182     }
2183
2184     fn make_run(run: RunConfig<'_>) {
2185         run.builder.ensure(LlvmTools {
2186             target: run.target,
2187         });
2188     }
2189
2190     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2191         let target = self.target;
2192         assert!(builder.config.extended);
2193
2194         /* run only if llvm-config isn't used */
2195         if let Some(config) = builder.config.target_config.get(&target) {
2196             if let Some(ref _s) = config.llvm_config {
2197                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2198                     target));
2199                 return None;
2200             }
2201         }
2202
2203         builder.info(&format!("Dist LlvmTools ({})", target));
2204         let _time = timeit(builder);
2205         let src = builder.src.join("src/llvm-project/llvm");
2206         let name = pkgname(builder, "llvm-tools");
2207
2208         let tmp = tmpdir(builder);
2209         let image = tmp.join("llvm-tools-image");
2210         drop(fs::remove_dir_all(&image));
2211
2212         // Prepare the image directory
2213         let src_bindir = builder
2214             .llvm_out(target)
2215             .join("bin");
2216         let dst_bindir = image.join("lib/rustlib")
2217             .join(&*target)
2218             .join("bin");
2219         t!(fs::create_dir_all(&dst_bindir));
2220         for tool in LLVM_TOOLS {
2221             let exe = src_bindir.join(exe(tool, &target));
2222             builder.install(&exe, &dst_bindir, 0o755);
2223         }
2224
2225         // Prepare the overlay
2226         let overlay = tmp.join("llvm-tools-overlay");
2227         drop(fs::remove_dir_all(&overlay));
2228         builder.create_dir(&overlay);
2229         builder.install(&src.join("README.txt"), &overlay, 0o644);
2230         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2231         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2232
2233         // Generate the installer tarball
2234         let mut cmd = rust_installer(builder);
2235         cmd.arg("generate")
2236             .arg("--product-name=Rust")
2237             .arg("--rel-manifest-dir=rustlib")
2238             .arg("--success-message=llvm-tools-installed.")
2239             .arg("--image-dir").arg(&image)
2240             .arg("--work-dir").arg(&tmpdir(builder))
2241             .arg("--output-dir").arg(&distdir(builder))
2242             .arg("--non-installed-overlay").arg(&overlay)
2243             .arg(format!("--package-name={}-{}", name, target))
2244             .arg("--legacy-manifest-dirs=rustlib,cargo")
2245             .arg("--component-name=llvm-tools-preview");
2246
2247
2248         builder.run(&mut cmd);
2249         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2250     }
2251 }
2252
2253 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2254 pub struct Lldb {
2255     pub target: Interned<String>,
2256 }
2257
2258 impl Step for Lldb {
2259     type Output = Option<PathBuf>;
2260     const ONLY_HOSTS: bool = true;
2261     const DEFAULT: bool = true;
2262
2263     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2264         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2265     }
2266
2267     fn make_run(run: RunConfig<'_>) {
2268         run.builder.ensure(Lldb {
2269             target: run.target,
2270         });
2271     }
2272
2273     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2274         let target = self.target;
2275
2276         if builder.config.dry_run {
2277             return None;
2278         }
2279
2280         let bindir = builder
2281             .llvm_out(target)
2282             .join("bin");
2283         let lldb_exe = bindir.join(exe("lldb", &target));
2284         if !lldb_exe.exists() {
2285             return None;
2286         }
2287
2288         builder.info(&format!("Dist Lldb ({})", target));
2289         let src = builder.src.join("src/llvm-project/lldb");
2290         let name = pkgname(builder, "lldb");
2291
2292         let tmp = tmpdir(builder);
2293         let image = tmp.join("lldb-image");
2294         drop(fs::remove_dir_all(&image));
2295
2296         // Prepare the image directory
2297         let root = image.join("lib/rustlib").join(&*target);
2298         let dst = root.join("bin");
2299         t!(fs::create_dir_all(&dst));
2300         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2301             let exe = bindir.join(exe(program, &target));
2302             builder.install(&exe, &dst, 0o755);
2303         }
2304
2305         // The libraries.
2306         let libdir = builder.llvm_out(target).join("lib");
2307         let dst = root.join("lib");
2308         t!(fs::create_dir_all(&dst));
2309         for entry in t!(fs::read_dir(&libdir)) {
2310             let entry = entry.unwrap();
2311             if let Ok(name) = entry.file_name().into_string() {
2312                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2313                     if t!(entry.file_type()).is_symlink() {
2314                         builder.copy_to_folder(&entry.path(), &dst);
2315                     } else {
2316                        builder.install(&entry.path(), &dst, 0o755);
2317                     }
2318                 }
2319             }
2320         }
2321
2322         // The lldb scripts might be installed in lib/python$version
2323         // or in lib64/python$version.  If lib64 exists, use it;
2324         // otherwise lib.
2325         let libdir = builder.llvm_out(target).join("lib64");
2326         let (libdir, libdir_name) = if libdir.exists() {
2327             (libdir, "lib64")
2328         } else {
2329             (builder.llvm_out(target).join("lib"), "lib")
2330         };
2331         for entry in t!(fs::read_dir(&libdir)) {
2332             let entry = t!(entry);
2333             if let Ok(name) = entry.file_name().into_string() {
2334                 if name.starts_with("python") {
2335                     let dst = root.join(libdir_name)
2336                         .join(entry.file_name());
2337                     t!(fs::create_dir_all(&dst));
2338                     builder.cp_r(&entry.path(), &dst);
2339                     break;
2340                 }
2341             }
2342         }
2343
2344         // Prepare the overlay
2345         let overlay = tmp.join("lldb-overlay");
2346         drop(fs::remove_dir_all(&overlay));
2347         builder.create_dir(&overlay);
2348         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2349         builder.create(&overlay.join("version"), &builder.lldb_vers());
2350
2351         // Generate the installer tarball
2352         let mut cmd = rust_installer(builder);
2353         cmd.arg("generate")
2354             .arg("--product-name=Rust")
2355             .arg("--rel-manifest-dir=rustlib")
2356             .arg("--success-message=lldb-installed.")
2357             .arg("--image-dir").arg(&image)
2358             .arg("--work-dir").arg(&tmpdir(builder))
2359             .arg("--output-dir").arg(&distdir(builder))
2360             .arg("--non-installed-overlay").arg(&overlay)
2361             .arg(format!("--package-name={}-{}", name, target))
2362             .arg("--legacy-manifest-dirs=rustlib,cargo")
2363             .arg("--component-name=lldb-preview");
2364
2365
2366         builder.run(&mut cmd);
2367         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2368     }
2369 }