src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472
 473             // Copy rustc/rustdoc binaries
 474             t!(fs::create_dir_all(image.join("bin")));
 475             builder.cp_r(&src.join("bin"), &image.join("bin"));
 476
 477             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 478
 479             let libdir_relative = builder.libdir_relative(compiler);
 480
 481             // Copy runtime DLLs needed by the compiler
 482             if libdir_relative.to_str() != Some("bin") {
 483                 let libdir = builder.rustc_libdir(compiler);
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             // Don't use custom libdir here because ^lib/ will be resolved again
 489                             // with installer
 490                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap()
 499                 .strip_prefix(builder.sysroot_libdir_relative(compiler)).unwrap();
 500             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 501             let backends_dst = image.join("lib").join(&backends_rel);
 502
 503             t!(fs::create_dir_all(&backends_dst));
 504             builder.cp_r(&backends_src, &backends_dst);
 505
 506             // Copy libLLVM.so to the lib dir as well, if needed. While not
 507             // technically needed by rustc itself it's needed by lots of other
 508             // components like the llvm tools and LLD. LLD is included below and
 509             // tools/LLDB come later, so let's just throw it in the rustc
 510             // component for now.
 511             maybe_install_llvm_dylib(builder, host, image);
 512
 513             // Copy over lld if it's there
 514             if builder.config.lld_enabled {
 515                 let exe = exe("rust-lld", &compiler.host);
 516                 let src = builder.sysroot_libdir(compiler, host)
 517                     .parent()
 518                     .unwrap()
 519                     .join("bin")
 520                     .join(&exe);
 521                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 522                 let dst = image.join("lib/rustlib")
 523                     .join(&*host)
 524                     .join("bin")
 525                     .join(&exe);
 526                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 527                 builder.copy(&src, &dst);
 528             }
 529
 530             // Man pages
 531             t!(fs::create_dir_all(image.join("share/man/man1")));
 532             let man_src = builder.src.join("src/doc/man");
 533             let man_dst = image.join("share/man/man1");
 534
 535             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 536             let time = env::var("SOURCE_DATE_EPOCH")
 537                 .map(|timestamp| {
 538                     let epoch = timestamp.parse().map_err(|err| {
 539                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 540                     }).unwrap();
 541
 542                     time::at(Timespec::new(epoch, 0))
 543                 })
 544                 .unwrap_or_else(|_| time::now());
 545
 546             let month_year = t!(time::strftime("%B %Y", &time));
 547             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 548             // to hardlink, and we don't want to edit the source templates
 549             for file_entry in builder.read_dir(&man_src) {
 550                 let page_src = file_entry.path();
 551                 let page_dst = man_dst.join(file_entry.file_name());
 552                 t!(fs::copy(&page_src, &page_dst));
 553                 // template in month/year and version number
 554                 builder.replace_in_file(&page_dst,
 555                                 &[("<INSERT DATE HERE>", &month_year),
 556                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 557             }
 558
 559             // Debugger scripts
 560             builder.ensure(DebuggerScripts {
 561                 sysroot: INTERNER.intern_path(image.to_owned()),
 562                 host,
 563             });
 564
 565             // Misc license info
 566             let cp = |file: &str| {
 567                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 568             };
 569             cp("COPYRIGHT");
 570             cp("LICENSE-APACHE");
 571             cp("LICENSE-MIT");
 572             cp("README.md");
 573         }
 574     }
 575 }
 576
 577 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 578 pub struct DebuggerScripts {
 579     pub sysroot: Interned<PathBuf>,
 580     pub host: Interned<String>,
 581 }
 582
 583 impl Step for DebuggerScripts {
 584     type Output = ();
 585
 586     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 587         run.path("src/lldb_batchmode.py")
 588     }
 589
 590     fn make_run(run: RunConfig<'_>) {
 591         run.builder.ensure(DebuggerScripts {
 592             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 593             host: run.target,
 594         });
 595     }
 596
 597     /// Copies debugger scripts for `target` into the `sysroot` specified.
 598     fn run(self, builder: &Builder<'_>) {
 599         let host = self.host;
 600         let sysroot = self.sysroot;
 601         let dst = sysroot.join("lib/rustlib/etc");
 602         t!(fs::create_dir_all(&dst));
 603         let cp_debugger_script = |file: &str| {
 604             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 605         };
 606         if host.contains("windows-msvc") {
 607             // windbg debugger scripts
 608             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 609                 0o755);
 610
 611             cp_debugger_script("natvis/intrinsic.natvis");
 612             cp_debugger_script("natvis/liballoc.natvis");
 613             cp_debugger_script("natvis/libcore.natvis");
 614         } else {
 615             cp_debugger_script("debugger_pretty_printers_common.py");
 616
 617             // gdb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 619                     0o755);
 620             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 621                     0o755);
 622
 623             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 624             cp_debugger_script("gdb_rust_pretty_printing.py");
 625
 626             // lldb debugger scripts
 627             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 628                     0o755);
 629
 630             cp_debugger_script("lldb_rust_formatters.py");
 631         }
 632     }
 633 }
 634
 635 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 636 pub struct Std {
 637     pub compiler: Compiler,
 638     pub target: Interned<String>,
 639 }
 640
 641 impl Step for Std {
 642     type Output = PathBuf;
 643     const DEFAULT: bool = true;
 644
 645     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 646         run.path("src/libstd")
 647     }
 648
 649     fn make_run(run: RunConfig<'_>) {
 650         run.builder.ensure(Std {
 651             compiler: run.builder.compiler_for(
 652                 run.builder.top_stage,
 653                 run.builder.config.build,
 654                 run.target,
 655             ),
 656             target: run.target,
 657         });
 658     }
 659
 660     fn run(self, builder: &Builder<'_>) -> PathBuf {
 661         let compiler = self.compiler;
 662         let target = self.target;
 663
 664         let name = pkgname(builder, "rust-std");
 665         builder.info(&format!("Dist std stage{} ({} -> {})",
 666             compiler.stage, &compiler.host, target));
 667
 668         // The only true set of target libraries came from the build triple, so
 669         // let's reduce redundant work by only producing archives from that host.
 670         if compiler.host != builder.config.build {
 671             builder.info("\tskipping, not a build host");
 672             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 673         }
 674
 675         // We want to package up as many target libraries as possible
 676         // for the `rust-std` package, so if this is a host target we
 677         // depend on librustc and otherwise we just depend on libtest.
 678         if builder.hosts.iter().any(|t| t == target) {
 679             builder.ensure(compile::Rustc { compiler, target });
 680         } else {
 681             if builder.no_std(target) == Some(true) {
 682                 // the `test` doesn't compile for no-std targets
 683                 builder.ensure(compile::Std { compiler, target });
 684             } else {
 685                 builder.ensure(compile::Test { compiler, target });
 686             }
 687         }
 688
 689         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 690         let _ = fs::remove_dir_all(&image);
 691
 692         let dst = image.join("lib/rustlib").join(target);
 693         t!(fs::create_dir_all(&dst));
 694         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 695         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 696         builder.cp_filtered(&src, &dst, &|path| {
 697             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 698                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 699                     return false
 700                 }
 701                 if name == "bin" {
 702                     return false
 703                 }
 704                 if name.contains("LLVM") {
 705                     return false
 706                 }
 707             }
 708             true
 709         });
 710
 711         let mut cmd = rust_installer(builder);
 712         cmd.arg("generate")
 713            .arg("--product-name=Rust")
 714            .arg("--rel-manifest-dir=rustlib")
 715            .arg("--success-message=std-is-standing-at-the-ready.")
 716            .arg("--image-dir").arg(&image)
 717            .arg("--work-dir").arg(&tmpdir(builder))
 718            .arg("--output-dir").arg(&distdir(builder))
 719            .arg(format!("--package-name={}-{}", name, target))
 720            .arg(format!("--component-name=rust-std-{}", target))
 721            .arg("--legacy-manifest-dirs=rustlib,cargo");
 722         builder.run(&mut cmd);
 723         builder.remove_dir(&image);
 724         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 725     }
 726 }
 727
 728 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 729 pub struct Analysis {
 730     pub compiler: Compiler,
 731     pub target: Interned<String>,
 732 }
 733
 734 impl Step for Analysis {
 735     type Output = PathBuf;
 736     const DEFAULT: bool = true;
 737
 738     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 739         let builder = run.builder;
 740         run.path("analysis").default_condition(builder.config.extended)
 741     }
 742
 743     fn make_run(run: RunConfig<'_>) {
 744         run.builder.ensure(Analysis {
 745             // Find the actual compiler (handling the full bootstrap option) which
 746             // produced the save-analysis data because that data isn't copied
 747             // through the sysroot uplifting.
 748             compiler: run.builder.compiler_for(
 749                 run.builder.top_stage,
 750                 run.builder.config.build,
 751                 run.target,
 752             ),
 753             target: run.target,
 754         });
 755     }
 756
 757     /// Creates a tarball of save-analysis metadata, if available.
 758     fn run(self, builder: &Builder<'_>) -> PathBuf {
 759         let compiler = self.compiler;
 760         let target = self.target;
 761         assert!(builder.config.extended);
 762         builder.info("Dist analysis");
 763         let name = pkgname(builder, "rust-analysis");
 764
 765         if &compiler.host != builder.config.build {
 766             builder.info("\tskipping, not a build host");
 767             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 768         }
 769
 770         builder.ensure(Std { compiler, target });
 771
 772         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 773
 774         let src = builder.stage_out(compiler, Mode::Std)
 775             .join(target).join(builder.cargo_dir()).join("deps");
 776
 777         let image_src = src.join("save-analysis");
 778         let dst = image.join("lib/rustlib").join(target).join("analysis");
 779         t!(fs::create_dir_all(&dst));
 780         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 781         builder.cp_r(&image_src, &dst);
 782
 783         let mut cmd = rust_installer(builder);
 784         cmd.arg("generate")
 785            .arg("--product-name=Rust")
 786            .arg("--rel-manifest-dir=rustlib")
 787            .arg("--success-message=save-analysis-saved.")
 788            .arg("--image-dir").arg(&image)
 789            .arg("--work-dir").arg(&tmpdir(builder))
 790            .arg("--output-dir").arg(&distdir(builder))
 791            .arg(format!("--package-name={}-{}", name, target))
 792            .arg(format!("--component-name=rust-analysis-{}", target))
 793            .arg("--legacy-manifest-dirs=rustlib,cargo");
 794         builder.run(&mut cmd);
 795         builder.remove_dir(&image);
 796         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 797     }
 798 }
 799
 800 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 801     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 802         let spath = match path.to_str() {
 803             Some(path) => path,
 804             None => return false,
 805         };
 806         if spath.ends_with("~") || spath.ends_with(".pyc") {
 807             return false
 808         }
 809
 810         const LLVM_PROJECTS: &[&str] = &[
 811             "llvm-project/clang", "llvm-project\\clang",
 812             "llvm-project/libunwind", "llvm-project\\libunwind",
 813             "llvm-project/lld", "llvm-project\\lld",
 814             "llvm-project/lldb", "llvm-project\\lldb",
 815             "llvm-project/llvm", "llvm-project\\llvm",
 816         ];
 817         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 818             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 819         {
 820             return false;
 821         }
 822
 823         const LLVM_TEST: &[&str] = &[
 824             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 825             "llvm-emscripten/test", "llvm-emscripten\\test",
 826         ];
 827         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 828             (spath.ends_with(".ll") ||
 829              spath.ends_with(".td") ||
 830              spath.ends_with(".s")) {
 831             return false
 832         }
 833         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 834             return false
 835         }
 836
 837         let full_path = Path::new(dir).join(path);
 838         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 839             return false;
 840         }
 841
 842         let excludes = [
 843             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 844             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 845             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 846             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 847         ];
 848         !path.iter()
 849              .map(|s| s.to_str().unwrap())
 850              .any(|s| excludes.contains(&s))
 851     }
 852
 853     // Copy the directories using our filter
 854     for item in src_dirs {
 855         let dst = &dst_dir.join(item);
 856         t!(fs::create_dir_all(dst));
 857         builder.cp_filtered(
 858             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 859     }
 860 }
 861
 862 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 863 pub struct Src;
 864
 865 impl Step for Src {
 866     /// The output path of the src installer tarball
 867     type Output = PathBuf;
 868     const DEFAULT: bool = true;
 869     const ONLY_HOSTS: bool = true;
 870
 871     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 872         run.path("src")
 873     }
 874
 875     fn make_run(run: RunConfig<'_>) {
 876         run.builder.ensure(Src);
 877     }
 878
 879     /// Creates the `rust-src` installer component
 880     fn run(self, builder: &Builder<'_>) -> PathBuf {
 881         builder.info("Dist src");
 882
 883         let name = pkgname(builder, "rust-src");
 884         let image = tmpdir(builder).join(format!("{}-image", name));
 885         let _ = fs::remove_dir_all(&image);
 886
 887         let dst = image.join("lib/rustlib/src");
 888         let dst_src = dst.join("rust");
 889         t!(fs::create_dir_all(&dst_src));
 890
 891         let src_files = [
 892             "Cargo.lock",
 893         ];
 894         // This is the reduced set of paths which will become the rust-src component
 895         // (essentially libstd and all of its path dependencies)
 896         let std_src_dirs = [
 897             "src/build_helper",
 898             "src/liballoc",
 899             "src/libcore",
 900             "src/libpanic_abort",
 901             "src/libpanic_unwind",
 902             "src/librustc_asan",
 903             "src/librustc_lsan",
 904             "src/librustc_msan",
 905             "src/librustc_tsan",
 906             "src/libstd",
 907             "src/libunwind",
 908             "src/libtest",
 909             "src/libterm",
 910             "src/libprofiler_builtins",
 911             "src/stdarch",
 912             "src/libproc_macro",
 913             "src/tools/rustc-std-workspace-core",
 914             "src/tools/rustc-std-workspace-alloc",
 915             "src/librustc",
 916             "src/libsyntax",
 917         ];
 918
 919         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 920         for file in src_files.iter() {
 921             builder.copy(&builder.src.join(file), &dst_src.join(file));
 922         }
 923
 924         // Create source tarball in rust-installer format
 925         let mut cmd = rust_installer(builder);
 926         cmd.arg("generate")
 927            .arg("--product-name=Rust")
 928            .arg("--rel-manifest-dir=rustlib")
 929            .arg("--success-message=Awesome-Source.")
 930            .arg("--image-dir").arg(&image)
 931            .arg("--work-dir").arg(&tmpdir(builder))
 932            .arg("--output-dir").arg(&distdir(builder))
 933            .arg(format!("--package-name={}", name))
 934            .arg("--component-name=rust-src")
 935            .arg("--legacy-manifest-dirs=rustlib,cargo");
 936         builder.run(&mut cmd);
 937
 938         builder.remove_dir(&image);
 939         distdir(builder).join(&format!("{}.tar.gz", name))
 940     }
 941 }
 942
 943 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 944 pub struct PlainSourceTarball;
 945
 946 impl Step for PlainSourceTarball {
 947     /// Produces the location of the tarball generated
 948     type Output = PathBuf;
 949     const DEFAULT: bool = true;
 950     const ONLY_HOSTS: bool = true;
 951
 952     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 953         let builder = run.builder;
 954         run.path("src").default_condition(builder.config.rust_dist_src)
 955     }
 956
 957     fn make_run(run: RunConfig<'_>) {
 958         run.builder.ensure(PlainSourceTarball);
 959     }
 960
 961     /// Creates the plain source tarball
 962     fn run(self, builder: &Builder<'_>) -> PathBuf {
 963         builder.info("Create plain source tarball");
 964
 965         // Make sure that the root folder of tarball has the correct name
 966         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 967         let plain_dst_src = tmpdir(builder).join(&plain_name);
 968         let _ = fs::remove_dir_all(&plain_dst_src);
 969         t!(fs::create_dir_all(&plain_dst_src));
 970
 971         // This is the set of root paths which will become part of the source package
 972         let src_files = [
 973             "COPYRIGHT",
 974             "LICENSE-APACHE",
 975             "LICENSE-MIT",
 976             "CONTRIBUTING.md",
 977             "README.md",
 978             "RELEASES.md",
 979             "configure",
 980             "x.py",
 981             "config.toml.example",
 982             "Cargo.toml",
 983             "Cargo.lock",
 984         ];
 985         let src_dirs = [
 986             "src",
 987         ];
 988
 989         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 990
 991         // Copy the files normally
 992         for item in &src_files {
 993             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 994         }
 995
 996         // Create the version file
 997         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 998         if let Some(sha) = builder.rust_sha() {
 999             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
1000         }
1001
1002         // If we're building from git sources, we need to vendor a complete distribution.
1003         if builder.rust_info.is_git() {
1004             // Vendor all Cargo dependencies
1005             let mut cmd = Command::new(&builder.initial_cargo);
1006             cmd.arg("vendor")
1007                .current_dir(&plain_dst_src);
1008             builder.run(&mut cmd);
1009         }
1010
1011         // Create plain source tarball
1012         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1013         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1014         tarball.set_extension(""); // strip .gz
1015         tarball.set_extension(""); // strip .tar
1016         if let Some(dir) = tarball.parent() {
1017             builder.create_dir(&dir);
1018         }
1019         builder.info("running installer");
1020         let mut cmd = rust_installer(builder);
1021         cmd.arg("tarball")
1022            .arg("--input").arg(&plain_name)
1023            .arg("--output").arg(&tarball)
1024            .arg("--work-dir=.")
1025            .current_dir(tmpdir(builder));
1026         builder.run(&mut cmd);
1027         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1028     }
1029 }
1030
1031 // We have to run a few shell scripts, which choke quite a bit on both `\`
1032 // characters and on `C:\` paths, so normalize both of them away.
1033 pub fn sanitize_sh(path: &Path) -> String {
1034     let path = path.to_str().unwrap().replace("\\", "/");
1035     return change_drive(&path).unwrap_or(path);
1036
1037     fn change_drive(s: &str) -> Option<String> {
1038         let mut ch = s.chars();
1039         let drive = ch.next().unwrap_or('C');
1040         if ch.next() != Some(':') {
1041             return None
1042         }
1043         if ch.next() != Some('/') {
1044             return None
1045         }
1046         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1047     }
1048 }
1049
1050 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1051 pub struct Cargo {
1052     pub compiler: Compiler,
1053     pub target: Interned<String>,
1054 }
1055
1056 impl Step for Cargo {
1057     type Output = PathBuf;
1058     const ONLY_HOSTS: bool = true;
1059
1060     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1061         run.path("cargo")
1062     }
1063
1064     fn make_run(run: RunConfig<'_>) {
1065         run.builder.ensure(Cargo {
1066             compiler: run.builder.compiler_for(
1067                 run.builder.top_stage,
1068                 run.builder.config.build,
1069                 run.target,
1070             ),
1071             target: run.target,
1072         });
1073     }
1074
1075     fn run(self, builder: &Builder<'_>) -> PathBuf {
1076         let compiler = self.compiler;
1077         let target = self.target;
1078
1079         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1080         let src = builder.src.join("src/tools/cargo");
1081         let etc = src.join("src/etc");
1082         let release_num = builder.release_num("cargo");
1083         let name = pkgname(builder, "cargo");
1084         let version = builder.cargo_info.version(builder, &release_num);
1085
1086         let tmp = tmpdir(builder);
1087         let image = tmp.join("cargo-image");
1088         drop(fs::remove_dir_all(&image));
1089         builder.create_dir(&image);
1090
1091         // Prepare the image directory
1092         builder.create_dir(&image.join("share/zsh/site-functions"));
1093         builder.create_dir(&image.join("etc/bash_completion.d"));
1094         let cargo = builder.ensure(tool::Cargo { compiler, target });
1095         builder.install(&cargo, &image.join("bin"), 0o755);
1096         for man in t!(etc.join("man").read_dir()) {
1097             let man = t!(man);
1098             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1099         }
1100         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1101         builder.copy(&etc.join("cargo.bashcomp.sh"),
1102              &image.join("etc/bash_completion.d/cargo"));
1103         let doc = image.join("share/doc/cargo");
1104         builder.install(&src.join("README.md"), &doc, 0o644);
1105         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1106         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1107         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1108
1109         // Prepare the overlay
1110         let overlay = tmp.join("cargo-overlay");
1111         drop(fs::remove_dir_all(&overlay));
1112         builder.create_dir(&overlay);
1113         builder.install(&src.join("README.md"), &overlay, 0o644);
1114         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1115         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1116         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1117         builder.create(&overlay.join("version"), &version);
1118
1119         // Generate the installer tarball
1120         let mut cmd = rust_installer(builder);
1121         cmd.arg("generate")
1122            .arg("--product-name=Rust")
1123            .arg("--rel-manifest-dir=rustlib")
1124            .arg("--success-message=Rust-is-ready-to-roll.")
1125            .arg("--image-dir").arg(&image)
1126            .arg("--work-dir").arg(&tmpdir(builder))
1127            .arg("--output-dir").arg(&distdir(builder))
1128            .arg("--non-installed-overlay").arg(&overlay)
1129            .arg(format!("--package-name={}-{}", name, target))
1130            .arg("--component-name=cargo")
1131            .arg("--legacy-manifest-dirs=rustlib,cargo");
1132         builder.run(&mut cmd);
1133         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1134     }
1135 }
1136
1137 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1138 pub struct Rls {
1139     pub compiler: Compiler,
1140     pub target: Interned<String>,
1141 }
1142
1143 impl Step for Rls {
1144     type Output = Option<PathBuf>;
1145     const ONLY_HOSTS: bool = true;
1146
1147     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1148         run.path("rls")
1149     }
1150
1151     fn make_run(run: RunConfig<'_>) {
1152         run.builder.ensure(Rls {
1153             compiler: run.builder.compiler_for(
1154                 run.builder.top_stage,
1155                 run.builder.config.build,
1156                 run.target,
1157             ),
1158             target: run.target,
1159         });
1160     }
1161
1162     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1163         let compiler = self.compiler;
1164         let target = self.target;
1165         assert!(builder.config.extended);
1166
1167         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1168         let src = builder.src.join("src/tools/rls");
1169         let release_num = builder.release_num("rls");
1170         let name = pkgname(builder, "rls");
1171         let version = builder.rls_info.version(builder, &release_num);
1172
1173         let tmp = tmpdir(builder);
1174         let image = tmp.join("rls-image");
1175         drop(fs::remove_dir_all(&image));
1176         t!(fs::create_dir_all(&image));
1177
1178         // Prepare the image directory
1179         // We expect RLS to build, because we've exited this step above if tool
1180         // state for RLS isn't testing.
1181         let rls = builder.ensure(tool::Rls {
1182             compiler,
1183             target,
1184             extra_features: Vec::new(),
1185         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1186
1187         builder.install(&rls, &image.join("bin"), 0o755);
1188         let doc = image.join("share/doc/rls");
1189         builder.install(&src.join("README.md"), &doc, 0o644);
1190         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1191         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1192
1193         // Prepare the overlay
1194         let overlay = tmp.join("rls-overlay");
1195         drop(fs::remove_dir_all(&overlay));
1196         t!(fs::create_dir_all(&overlay));
1197         builder.install(&src.join("README.md"), &overlay, 0o644);
1198         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1199         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1200         builder.create(&overlay.join("version"), &version);
1201
1202         // Generate the installer tarball
1203         let mut cmd = rust_installer(builder);
1204         cmd.arg("generate")
1205            .arg("--product-name=Rust")
1206            .arg("--rel-manifest-dir=rustlib")
1207            .arg("--success-message=RLS-ready-to-serve.")
1208            .arg("--image-dir").arg(&image)
1209            .arg("--work-dir").arg(&tmpdir(builder))
1210            .arg("--output-dir").arg(&distdir(builder))
1211            .arg("--non-installed-overlay").arg(&overlay)
1212            .arg(format!("--package-name={}-{}", name, target))
1213            .arg("--legacy-manifest-dirs=rustlib,cargo")
1214            .arg("--component-name=rls-preview");
1215
1216         builder.run(&mut cmd);
1217         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1218     }
1219 }
1220
1221 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1222 pub struct Clippy {
1223     pub compiler: Compiler,
1224     pub target: Interned<String>,
1225 }
1226
1227 impl Step for Clippy {
1228     type Output = Option<PathBuf>;
1229     const ONLY_HOSTS: bool = true;
1230
1231     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1232         run.path("clippy")
1233     }
1234
1235     fn make_run(run: RunConfig<'_>) {
1236         run.builder.ensure(Clippy {
1237             compiler: run.builder.compiler_for(
1238                 run.builder.top_stage,
1239                 run.builder.config.build,
1240                 run.target,
1241             ),
1242             target: run.target,
1243         });
1244     }
1245
1246     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1247         let compiler = self.compiler;
1248         let target = self.target;
1249         assert!(builder.config.extended);
1250
1251         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1252         let src = builder.src.join("src/tools/clippy");
1253         let release_num = builder.release_num("clippy");
1254         let name = pkgname(builder, "clippy");
1255         let version = builder.clippy_info.version(builder, &release_num);
1256
1257         let tmp = tmpdir(builder);
1258         let image = tmp.join("clippy-image");
1259         drop(fs::remove_dir_all(&image));
1260         builder.create_dir(&image);
1261
1262         // Prepare the image directory
1263         // We expect clippy to build, because we've exited this step above if tool
1264         // state for clippy isn't testing.
1265         let clippy = builder.ensure(tool::Clippy {
1266             compiler,
1267             target,
1268             extra_features: Vec::new(),
1269         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1270         let cargoclippy = builder.ensure(tool::CargoClippy {
1271             compiler,
1272             target, extra_features: Vec::new()
1273         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1274
1275         builder.install(&clippy, &image.join("bin"), 0o755);
1276         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1277         let doc = image.join("share/doc/clippy");
1278         builder.install(&src.join("README.md"), &doc, 0o644);
1279         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1280         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1281
1282         // Prepare the overlay
1283         let overlay = tmp.join("clippy-overlay");
1284         drop(fs::remove_dir_all(&overlay));
1285         t!(fs::create_dir_all(&overlay));
1286         builder.install(&src.join("README.md"), &overlay, 0o644);
1287         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1288         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1289         builder.create(&overlay.join("version"), &version);
1290
1291         // Generate the installer tarball
1292         let mut cmd = rust_installer(builder);
1293         cmd.arg("generate")
1294            .arg("--product-name=Rust")
1295            .arg("--rel-manifest-dir=rustlib")
1296            .arg("--success-message=clippy-ready-to-serve.")
1297            .arg("--image-dir").arg(&image)
1298            .arg("--work-dir").arg(&tmpdir(builder))
1299            .arg("--output-dir").arg(&distdir(builder))
1300            .arg("--non-installed-overlay").arg(&overlay)
1301            .arg(format!("--package-name={}-{}", name, target))
1302            .arg("--legacy-manifest-dirs=rustlib,cargo")
1303            .arg("--component-name=clippy-preview");
1304
1305         builder.run(&mut cmd);
1306         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1307     }
1308 }
1309
1310 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1311 pub struct Miri {
1312     pub compiler: Compiler,
1313     pub target: Interned<String>,
1314 }
1315
1316 impl Step for Miri {
1317     type Output = Option<PathBuf>;
1318     const ONLY_HOSTS: bool = true;
1319
1320     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1321         run.path("miri")
1322     }
1323
1324     fn make_run(run: RunConfig<'_>) {
1325         run.builder.ensure(Miri {
1326             compiler: run.builder.compiler_for(
1327                 run.builder.top_stage,
1328                 run.builder.config.build,
1329                 run.target,
1330             ),
1331             target: run.target,
1332         });
1333     }
1334
1335     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1336         let compiler = self.compiler;
1337         let target = self.target;
1338         assert!(builder.config.extended);
1339
1340         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1341         let src = builder.src.join("src/tools/miri");
1342         let release_num = builder.release_num("miri");
1343         let name = pkgname(builder, "miri");
1344         let version = builder.miri_info.version(builder, &release_num);
1345
1346         let tmp = tmpdir(builder);
1347         let image = tmp.join("miri-image");
1348         drop(fs::remove_dir_all(&image));
1349         builder.create_dir(&image);
1350
1351         // Prepare the image directory
1352         // We expect miri to build, because we've exited this step above if tool
1353         // state for miri isn't testing.
1354         let miri = builder.ensure(tool::Miri {
1355             compiler,
1356             target,
1357             extra_features: Vec::new(),
1358         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1359         let cargomiri = builder.ensure(tool::CargoMiri {
1360             compiler,
1361             target,
1362             extra_features: Vec::new()
1363         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1364
1365         builder.install(&miri, &image.join("bin"), 0o755);
1366         builder.install(&cargomiri, &image.join("bin"), 0o755);
1367         let doc = image.join("share/doc/miri");
1368         builder.install(&src.join("README.md"), &doc, 0o644);
1369         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1370         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1371
1372         // Prepare the overlay
1373         let overlay = tmp.join("miri-overlay");
1374         drop(fs::remove_dir_all(&overlay));
1375         t!(fs::create_dir_all(&overlay));
1376         builder.install(&src.join("README.md"), &overlay, 0o644);
1377         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1378         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1379         builder.create(&overlay.join("version"), &version);
1380
1381         // Generate the installer tarball
1382         let mut cmd = rust_installer(builder);
1383         cmd.arg("generate")
1384            .arg("--product-name=Rust")
1385            .arg("--rel-manifest-dir=rustlib")
1386            .arg("--success-message=miri-ready-to-serve.")
1387            .arg("--image-dir").arg(&image)
1388            .arg("--work-dir").arg(&tmpdir(builder))
1389            .arg("--output-dir").arg(&distdir(builder))
1390            .arg("--non-installed-overlay").arg(&overlay)
1391            .arg(format!("--package-name={}-{}", name, target))
1392            .arg("--legacy-manifest-dirs=rustlib,cargo")
1393            .arg("--component-name=miri-preview");
1394
1395         builder.run(&mut cmd);
1396         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1397     }
1398 }
1399
1400 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1401 pub struct Rustfmt {
1402     pub compiler: Compiler,
1403     pub target: Interned<String>,
1404 }
1405
1406 impl Step for Rustfmt {
1407     type Output = Option<PathBuf>;
1408     const ONLY_HOSTS: bool = true;
1409
1410     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1411         run.path("rustfmt")
1412     }
1413
1414     fn make_run(run: RunConfig<'_>) {
1415         run.builder.ensure(Rustfmt {
1416             compiler: run.builder.compiler_for(
1417                 run.builder.top_stage,
1418                 run.builder.config.build,
1419                 run.target,
1420             ),
1421             target: run.target,
1422         });
1423     }
1424
1425     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1426         let compiler = self.compiler;
1427         let target = self.target;
1428
1429         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1430         let src = builder.src.join("src/tools/rustfmt");
1431         let release_num = builder.release_num("rustfmt");
1432         let name = pkgname(builder, "rustfmt");
1433         let version = builder.rustfmt_info.version(builder, &release_num);
1434
1435         let tmp = tmpdir(builder);
1436         let image = tmp.join("rustfmt-image");
1437         drop(fs::remove_dir_all(&image));
1438         builder.create_dir(&image);
1439
1440         // Prepare the image directory
1441         let rustfmt = builder.ensure(tool::Rustfmt {
1442             compiler,
1443             target,
1444             extra_features: Vec::new(),
1445         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1446         let cargofmt = builder.ensure(tool::Cargofmt {
1447             compiler,
1448             target,
1449             extra_features: Vec::new(),
1450         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1451
1452         builder.install(&rustfmt, &image.join("bin"), 0o755);
1453         builder.install(&cargofmt, &image.join("bin"), 0o755);
1454         let doc = image.join("share/doc/rustfmt");
1455         builder.install(&src.join("README.md"), &doc, 0o644);
1456         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1457         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1458
1459         // Prepare the overlay
1460         let overlay = tmp.join("rustfmt-overlay");
1461         drop(fs::remove_dir_all(&overlay));
1462         builder.create_dir(&overlay);
1463         builder.install(&src.join("README.md"), &overlay, 0o644);
1464         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1465         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1466         builder.create(&overlay.join("version"), &version);
1467
1468         // Generate the installer tarball
1469         let mut cmd = rust_installer(builder);
1470         cmd.arg("generate")
1471            .arg("--product-name=Rust")
1472            .arg("--rel-manifest-dir=rustlib")
1473            .arg("--success-message=rustfmt-ready-to-fmt.")
1474            .arg("--image-dir").arg(&image)
1475            .arg("--work-dir").arg(&tmpdir(builder))
1476            .arg("--output-dir").arg(&distdir(builder))
1477            .arg("--non-installed-overlay").arg(&overlay)
1478            .arg(format!("--package-name={}-{}", name, target))
1479            .arg("--legacy-manifest-dirs=rustlib,cargo")
1480            .arg("--component-name=rustfmt-preview");
1481
1482         builder.run(&mut cmd);
1483         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1484     }
1485 }
1486
1487 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1488 pub struct Extended {
1489     stage: u32,
1490     host: Interned<String>,
1491     target: Interned<String>,
1492 }
1493
1494 impl Step for Extended {
1495     type Output = ();
1496     const DEFAULT: bool = true;
1497     const ONLY_HOSTS: bool = true;
1498
1499     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1500         let builder = run.builder;
1501         run.path("extended").default_condition(builder.config.extended)
1502     }
1503
1504     fn make_run(run: RunConfig<'_>) {
1505         run.builder.ensure(Extended {
1506             stage: run.builder.top_stage,
1507             host: run.builder.config.build,
1508             target: run.target,
1509         });
1510     }
1511
1512     /// Creates a combined installer for the specified target in the provided stage.
1513     fn run(self, builder: &Builder<'_>) {
1514         let target = self.target;
1515         let stage = self.stage;
1516         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1517
1518         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1519
1520         let rustc_installer = builder.ensure(Rustc {
1521             compiler: builder.compiler(stage, target),
1522         });
1523         let cargo_installer = builder.ensure(Cargo { compiler, target });
1524         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1525         let rls_installer = builder.ensure(Rls { compiler, target });
1526         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1527         let clippy_installer = builder.ensure(Clippy { compiler, target });
1528         let miri_installer = builder.ensure(Miri { compiler, target });
1529         let lldb_installer = builder.ensure(Lldb { target });
1530         let mingw_installer = builder.ensure(Mingw { host: target });
1531         let analysis_installer = builder.ensure(Analysis { compiler, target });
1532
1533         let docs_installer = builder.ensure(Docs { host: target, });
1534         let std_installer = builder.ensure(Std {
1535             compiler: builder.compiler(stage, target),
1536             target,
1537         });
1538
1539         let tmp = tmpdir(builder);
1540         let overlay = tmp.join("extended-overlay");
1541         let etc = builder.src.join("src/etc/installer");
1542         let work = tmp.join("work");
1543
1544         let _ = fs::remove_dir_all(&overlay);
1545         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1546         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1547         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1548         let version = builder.rust_version();
1549         builder.create(&overlay.join("version"), &version);
1550         if let Some(sha) = builder.rust_sha() {
1551             builder.create(&overlay.join("git-commit-hash"), &sha);
1552         }
1553         builder.install(&etc.join("README.md"), &overlay, 0o644);
1554
1555         // When rust-std package split from rustc, we needed to ensure that during
1556         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1557         // the std files during uninstall. To do this ensure that rustc comes
1558         // before rust-std in the list below.
1559         let mut tarballs = Vec::new();
1560         tarballs.push(rustc_installer);
1561         tarballs.push(cargo_installer);
1562         tarballs.extend(rls_installer.clone());
1563         tarballs.extend(clippy_installer.clone());
1564         tarballs.extend(miri_installer.clone());
1565         tarballs.extend(rustfmt_installer.clone());
1566         tarballs.extend(llvm_tools_installer);
1567         tarballs.extend(lldb_installer);
1568         tarballs.push(analysis_installer);
1569         tarballs.push(std_installer);
1570         if builder.config.docs {
1571             tarballs.push(docs_installer);
1572         }
1573         if target.contains("pc-windows-gnu") {
1574             tarballs.push(mingw_installer.unwrap());
1575         }
1576         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1577         for tarball in &tarballs[1..] {
1578             input_tarballs.push(",");
1579             input_tarballs.push(tarball);
1580         }
1581
1582         let mut cmd = rust_installer(builder);
1583         cmd.arg("combine")
1584             .arg("--product-name=Rust")
1585             .arg("--rel-manifest-dir=rustlib")
1586             .arg("--success-message=Rust-is-ready-to-roll.")
1587             .arg("--work-dir").arg(&work)
1588             .arg("--output-dir").arg(&distdir(builder))
1589             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1590             .arg("--legacy-manifest-dirs=rustlib,cargo")
1591             .arg("--input-tarballs").arg(input_tarballs)
1592             .arg("--non-installed-overlay").arg(&overlay);
1593         builder.run(&mut cmd);
1594
1595         let mut license = String::new();
1596         license += &builder.read(&builder.src.join("COPYRIGHT"));
1597         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1598         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1599         license.push_str("\n");
1600         license.push_str("\n");
1601
1602         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1603         let mut rtf = rtf.to_string();
1604         rtf.push_str("\n");
1605         for line in license.lines() {
1606             rtf.push_str(line);
1607             rtf.push_str("\\line ");
1608         }
1609         rtf.push_str("}");
1610
1611         fn filter(contents: &str, marker: &str) -> String {
1612             let start = format!("tool-{}-start", marker);
1613             let end = format!("tool-{}-end", marker);
1614             let mut lines = Vec::new();
1615             let mut omitted = false;
1616             for line in contents.lines() {
1617                 if line.contains(&start) {
1618                     omitted = true;
1619                 } else if line.contains(&end) {
1620                     omitted = false;
1621                 } else if !omitted {
1622                     lines.push(line);
1623                 }
1624             }
1625
1626             lines.join("\n")
1627         }
1628
1629         let xform = |p: &Path| {
1630             let mut contents = t!(fs::read_to_string(p));
1631             if rls_installer.is_none() {
1632                 contents = filter(&contents, "rls");
1633             }
1634             if clippy_installer.is_none() {
1635                 contents = filter(&contents, "clippy");
1636             }
1637             if miri_installer.is_none() {
1638                 contents = filter(&contents, "miri");
1639             }
1640             if rustfmt_installer.is_none() {
1641                 contents = filter(&contents, "rustfmt");
1642             }
1643             let ret = tmp.join(p.file_name().unwrap());
1644             t!(fs::write(&ret, &contents));
1645             ret
1646         };
1647
1648         if target.contains("apple-darwin") {
1649             let pkg = tmp.join("pkg");
1650             let _ = fs::remove_dir_all(&pkg);
1651
1652             let pkgbuild = |component: &str| {
1653                 let mut cmd = Command::new("pkgbuild");
1654                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1655                     .arg("--scripts").arg(pkg.join(component))
1656                     .arg("--nopayload")
1657                     .arg(pkg.join(component).with_extension("pkg"));
1658                 builder.run(&mut cmd);
1659             };
1660
1661             let prepare = |name: &str| {
1662                 builder.create_dir(&pkg.join(name));
1663                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1664                         &pkg.join(name));
1665                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1666                 pkgbuild(name);
1667             };
1668             prepare("rustc");
1669             prepare("cargo");
1670             prepare("rust-docs");
1671             prepare("rust-std");
1672             prepare("rust-analysis");
1673
1674             if rls_installer.is_some() {
1675                 prepare("rls");
1676             }
1677             if clippy_installer.is_some() {
1678                 prepare("clippy");
1679             }
1680             if miri_installer.is_some() {
1681                 prepare("miri");
1682             }
1683
1684             // create an 'uninstall' package
1685             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1686             pkgbuild("uninstall");
1687
1688             builder.create_dir(&pkg.join("res"));
1689             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1690             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1691             let mut cmd = Command::new("productbuild");
1692             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1693                 .arg("--resources").arg(pkg.join("res"))
1694                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1695                                                     pkgname(builder, "rust"),
1696                                                     target)))
1697                 .arg("--package-path").arg(&pkg);
1698             builder.run(&mut cmd);
1699         }
1700
1701         if target.contains("windows") {
1702             let exe = tmp.join("exe");
1703             let _ = fs::remove_dir_all(&exe);
1704
1705             let prepare = |name: &str| {
1706                 builder.create_dir(&exe.join(name));
1707                 let dir = if name == "rust-std" || name == "rust-analysis" {
1708                     format!("{}-{}", name, target)
1709                 } else if name == "rls" {
1710                     "rls-preview".to_string()
1711                 } else if name == "clippy" {
1712                     "clippy-preview".to_string()
1713                 } else if name == "miri" {
1714                     "miri-preview".to_string()
1715                 } else {
1716                     name.to_string()
1717                 };
1718                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1719                             .join(dir),
1720                         &exe.join(name));
1721                 builder.remove(&exe.join(name).join("manifest.in"));
1722             };
1723             prepare("rustc");
1724             prepare("cargo");
1725             prepare("rust-analysis");
1726             prepare("rust-docs");
1727             prepare("rust-std");
1728             if rls_installer.is_some() {
1729                 prepare("rls");
1730             }
1731             if clippy_installer.is_some() {
1732                 prepare("clippy");
1733             }
1734             if miri_installer.is_some() {
1735                 prepare("miri");
1736             }
1737             if target.contains("windows-gnu") {
1738                 prepare("rust-mingw");
1739             }
1740
1741             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1742             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1743             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1744             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1745             builder.create(&exe.join("LICENSE.txt"), &license);
1746
1747             // Generate exe installer
1748             let mut cmd = Command::new("iscc");
1749             cmd.arg("rust.iss")
1750                 .current_dir(&exe);
1751             if target.contains("windows-gnu") {
1752                 cmd.arg("/dMINGW");
1753             }
1754             add_env(builder, &mut cmd, target);
1755             builder.run(&mut cmd);
1756             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1757                     &distdir(builder),
1758                     0o755);
1759
1760             // Generate msi installer
1761             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1762             let heat = wix.join("bin/heat.exe");
1763             let candle = wix.join("bin/candle.exe");
1764             let light = wix.join("bin/light.exe");
1765
1766             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1767             builder.run(Command::new(&heat)
1768                             .current_dir(&exe)
1769                             .arg("dir")
1770                             .arg("rustc")
1771                             .args(&heat_flags)
1772                             .arg("-cg").arg("RustcGroup")
1773                             .arg("-dr").arg("Rustc")
1774                             .arg("-var").arg("var.RustcDir")
1775                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1776             builder.run(Command::new(&heat)
1777                             .current_dir(&exe)
1778                             .arg("dir")
1779                             .arg("rust-docs")
1780                             .args(&heat_flags)
1781                             .arg("-cg").arg("DocsGroup")
1782                             .arg("-dr").arg("Docs")
1783                             .arg("-var").arg("var.DocsDir")
1784                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1785                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1786             builder.run(Command::new(&heat)
1787                             .current_dir(&exe)
1788                             .arg("dir")
1789                             .arg("cargo")
1790                             .args(&heat_flags)
1791                             .arg("-cg").arg("CargoGroup")
1792                             .arg("-dr").arg("Cargo")
1793                             .arg("-var").arg("var.CargoDir")
1794                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1795                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1796             builder.run(Command::new(&heat)
1797                             .current_dir(&exe)
1798                             .arg("dir")
1799                             .arg("rust-std")
1800                             .args(&heat_flags)
1801                             .arg("-cg").arg("StdGroup")
1802                             .arg("-dr").arg("Std")
1803                             .arg("-var").arg("var.StdDir")
1804                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1805             if rls_installer.is_some() {
1806                 builder.run(Command::new(&heat)
1807                                 .current_dir(&exe)
1808                                 .arg("dir")
1809                                 .arg("rls")
1810                                 .args(&heat_flags)
1811                                 .arg("-cg").arg("RlsGroup")
1812                                 .arg("-dr").arg("Rls")
1813                                 .arg("-var").arg("var.RlsDir")
1814                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1815                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1816             }
1817             if clippy_installer.is_some() {
1818                 builder.run(Command::new(&heat)
1819                                 .current_dir(&exe)
1820                                 .arg("dir")
1821                                 .arg("clippy")
1822                                 .args(&heat_flags)
1823                                 .arg("-cg").arg("ClippyGroup")
1824                                 .arg("-dr").arg("Clippy")
1825                                 .arg("-var").arg("var.ClippyDir")
1826                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1827                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1828             }
1829             if miri_installer.is_some() {
1830                 builder.run(Command::new(&heat)
1831                                 .current_dir(&exe)
1832                                 .arg("dir")
1833                                 .arg("miri")
1834                                 .args(&heat_flags)
1835                                 .arg("-cg").arg("MiriGroup")
1836                                 .arg("-dr").arg("Miri")
1837                                 .arg("-var").arg("var.MiriDir")
1838                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1839                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1840             }
1841             builder.run(Command::new(&heat)
1842                             .current_dir(&exe)
1843                             .arg("dir")
1844                             .arg("rust-analysis")
1845                             .args(&heat_flags)
1846                             .arg("-cg").arg("AnalysisGroup")
1847                             .arg("-dr").arg("Analysis")
1848                             .arg("-var").arg("var.AnalysisDir")
1849                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1850                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1851             if target.contains("windows-gnu") {
1852                 builder.run(Command::new(&heat)
1853                                 .current_dir(&exe)
1854                                 .arg("dir")
1855                                 .arg("rust-mingw")
1856                                 .args(&heat_flags)
1857                                 .arg("-cg").arg("GccGroup")
1858                                 .arg("-dr").arg("Gcc")
1859                                 .arg("-var").arg("var.GccDir")
1860                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1861             }
1862
1863             let candle = |input: &Path| {
1864                 let output = exe.join(input.file_stem().unwrap())
1865                                 .with_extension("wixobj");
1866                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1867                 let mut cmd = Command::new(&candle);
1868                 cmd.current_dir(&exe)
1869                     .arg("-nologo")
1870                     .arg("-dRustcDir=rustc")
1871                     .arg("-dDocsDir=rust-docs")
1872                     .arg("-dCargoDir=cargo")
1873                     .arg("-dStdDir=rust-std")
1874                     .arg("-dAnalysisDir=rust-analysis")
1875                     .arg("-arch").arg(&arch)
1876                     .arg("-out").arg(&output)
1877                     .arg(&input);
1878                 add_env(builder, &mut cmd, target);
1879
1880                 if rls_installer.is_some() {
1881                     cmd.arg("-dRlsDir=rls");
1882                 }
1883                 if clippy_installer.is_some() {
1884                     cmd.arg("-dClippyDir=clippy");
1885                 }
1886                 if miri_installer.is_some() {
1887                     cmd.arg("-dMiriDir=miri");
1888                 }
1889                 if target.contains("windows-gnu") {
1890                     cmd.arg("-dGccDir=rust-mingw");
1891                 }
1892                 builder.run(&mut cmd);
1893             };
1894             candle(&xform(&etc.join("msi/rust.wxs")));
1895             candle(&etc.join("msi/ui.wxs"));
1896             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1897             candle("RustcGroup.wxs".as_ref());
1898             candle("DocsGroup.wxs".as_ref());
1899             candle("CargoGroup.wxs".as_ref());
1900             candle("StdGroup.wxs".as_ref());
1901             if rls_installer.is_some() {
1902                 candle("RlsGroup.wxs".as_ref());
1903             }
1904             if clippy_installer.is_some() {
1905                 candle("ClippyGroup.wxs".as_ref());
1906             }
1907             if miri_installer.is_some() {
1908                 candle("MiriGroup.wxs".as_ref());
1909             }
1910             candle("AnalysisGroup.wxs".as_ref());
1911
1912             if target.contains("windows-gnu") {
1913                 candle("GccGroup.wxs".as_ref());
1914             }
1915
1916             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1917             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1918             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1919
1920             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1921             let mut cmd = Command::new(&light);
1922             cmd.arg("-nologo")
1923                 .arg("-ext").arg("WixUIExtension")
1924                 .arg("-ext").arg("WixUtilExtension")
1925                 .arg("-out").arg(exe.join(&filename))
1926                 .arg("rust.wixobj")
1927                 .arg("ui.wixobj")
1928                 .arg("rustwelcomedlg.wixobj")
1929                 .arg("RustcGroup.wixobj")
1930                 .arg("DocsGroup.wixobj")
1931                 .arg("CargoGroup.wixobj")
1932                 .arg("StdGroup.wixobj")
1933                 .arg("AnalysisGroup.wixobj")
1934                 .current_dir(&exe);
1935
1936             if rls_installer.is_some() {
1937                 cmd.arg("RlsGroup.wixobj");
1938             }
1939             if clippy_installer.is_some() {
1940                 cmd.arg("ClippyGroup.wixobj");
1941             }
1942             if miri_installer.is_some() {
1943                 cmd.arg("MiriGroup.wixobj");
1944             }
1945
1946             if target.contains("windows-gnu") {
1947                 cmd.arg("GccGroup.wixobj");
1948             }
1949             // ICE57 wrongly complains about the shortcuts
1950             cmd.arg("-sice:ICE57");
1951
1952             builder.run(&mut cmd);
1953
1954             if !builder.config.dry_run {
1955                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1956             }
1957         }
1958     }
1959 }
1960
1961 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1962     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1963     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1964        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1965        .env("CFG_RELEASE", builder.rust_release())
1966        .env("CFG_VER_MAJOR", parts.next().unwrap())
1967        .env("CFG_VER_MINOR", parts.next().unwrap())
1968        .env("CFG_VER_PATCH", parts.next().unwrap())
1969        .env("CFG_VER_BUILD", "0") // just needed to build
1970        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1971        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1972        .env("CFG_BUILD", target)
1973        .env("CFG_CHANNEL", &builder.config.channel);
1974
1975     if target.contains("windows-gnu") {
1976        cmd.env("CFG_MINGW", "1")
1977           .env("CFG_ABI", "GNU");
1978     } else {
1979        cmd.env("CFG_MINGW", "0")
1980           .env("CFG_ABI", "MSVC");
1981     }
1982
1983     if target.contains("x86_64") {
1984        cmd.env("CFG_PLATFORM", "x64");
1985     } else {
1986        cmd.env("CFG_PLATFORM", "x86");
1987     }
1988 }
1989
1990 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1991 pub struct HashSign;
1992
1993 impl Step for HashSign {
1994     type Output = ();
1995     const ONLY_HOSTS: bool = true;
1996
1997     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1998         run.path("hash-and-sign")
1999     }
2000
2001     fn make_run(run: RunConfig<'_>) {
2002         run.builder.ensure(HashSign);
2003     }
2004
2005     fn run(self, builder: &Builder<'_>) {
2006         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2007         if builder.config.dry_run {
2008             return;
2009         }
2010         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2011             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2012         });
2013         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2014             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2015         });
2016         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2017             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2018         });
2019         let pass = t!(fs::read_to_string(&file));
2020
2021         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2022
2023         cmd.arg(sign);
2024         cmd.arg(distdir(builder));
2025         cmd.arg(today.trim());
2026         cmd.arg(builder.rust_package_vers());
2027         cmd.arg(addr);
2028         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2029         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2030         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2031         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2032         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2033         cmd.arg(builder.llvm_tools_package_vers());
2034         cmd.arg(builder.lldb_package_vers());
2035
2036         builder.create_dir(&distdir(builder));
2037
2038         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2039         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2040         let status = t!(child.wait());
2041         assert!(status.success());
2042     }
2043 }
2044
2045 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2046 // LLVM tools are linked dynamically.
2047 // Note: This function does no yet support Windows but we also don't support
2048 //       linking LLVM tools dynamically on Windows yet.
2049 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2050                                 target: Interned<String>,
2051                                 sysroot: &Path) {
2052     let src_libdir = builder
2053         .llvm_out(target)
2054         .join("lib");
2055     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2056     t!(fs::create_dir_all(&dst_libdir));
2057
2058     if target.contains("apple-darwin") {
2059         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2060         if llvm_dylib_path.exists() {
2061             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2062         }
2063         return
2064     }
2065
2066     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2067     // Since tools link to the latter rather than the former, we have to
2068     // follow the symlink to find out what to distribute.
2069     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2070     if llvm_dylib_path.exists() {
2071         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2072             panic!("dist: Error calling canonicalize path `{}`: {}",
2073                    llvm_dylib_path.display(), e);
2074         });
2075
2076
2077         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2078     }
2079 }
2080
2081 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2082 pub struct LlvmTools {
2083     pub target: Interned<String>,
2084 }
2085
2086 impl Step for LlvmTools {
2087     type Output = Option<PathBuf>;
2088     const ONLY_HOSTS: bool = true;
2089
2090     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2091         run.path("llvm-tools")
2092     }
2093
2094     fn make_run(run: RunConfig<'_>) {
2095         run.builder.ensure(LlvmTools {
2096             target: run.target,
2097         });
2098     }
2099
2100     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2101         let target = self.target;
2102         assert!(builder.config.extended);
2103
2104         /* run only if llvm-config isn't used */
2105         if let Some(config) = builder.config.target_config.get(&target) {
2106             if let Some(ref _s) = config.llvm_config {
2107                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2108                     target));
2109                 return None;
2110             }
2111         }
2112
2113         builder.info(&format!("Dist LlvmTools ({})", target));
2114         let src = builder.src.join("src/llvm-project/llvm");
2115         let name = pkgname(builder, "llvm-tools");
2116
2117         let tmp = tmpdir(builder);
2118         let image = tmp.join("llvm-tools-image");
2119         drop(fs::remove_dir_all(&image));
2120
2121         // Prepare the image directory
2122         let src_bindir = builder
2123             .llvm_out(target)
2124             .join("bin");
2125         let dst_bindir = image.join("lib/rustlib")
2126             .join(&*target)
2127             .join("bin");
2128         t!(fs::create_dir_all(&dst_bindir));
2129         for tool in LLVM_TOOLS {
2130             let exe = src_bindir.join(exe(tool, &target));
2131             builder.install(&exe, &dst_bindir, 0o755);
2132         }
2133
2134         // Prepare the overlay
2135         let overlay = tmp.join("llvm-tools-overlay");
2136         drop(fs::remove_dir_all(&overlay));
2137         builder.create_dir(&overlay);
2138         builder.install(&src.join("README.txt"), &overlay, 0o644);
2139         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2140         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2141
2142         // Generate the installer tarball
2143         let mut cmd = rust_installer(builder);
2144         cmd.arg("generate")
2145             .arg("--product-name=Rust")
2146             .arg("--rel-manifest-dir=rustlib")
2147             .arg("--success-message=llvm-tools-installed.")
2148             .arg("--image-dir").arg(&image)
2149             .arg("--work-dir").arg(&tmpdir(builder))
2150             .arg("--output-dir").arg(&distdir(builder))
2151             .arg("--non-installed-overlay").arg(&overlay)
2152             .arg(format!("--package-name={}-{}", name, target))
2153             .arg("--legacy-manifest-dirs=rustlib,cargo")
2154             .arg("--component-name=llvm-tools-preview");
2155
2156
2157         builder.run(&mut cmd);
2158         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2159     }
2160 }
2161
2162 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2163 pub struct Lldb {
2164     pub target: Interned<String>,
2165 }
2166
2167 impl Step for Lldb {
2168     type Output = Option<PathBuf>;
2169     const ONLY_HOSTS: bool = true;
2170     const DEFAULT: bool = true;
2171
2172     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2173         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2174     }
2175
2176     fn make_run(run: RunConfig<'_>) {
2177         run.builder.ensure(Lldb {
2178             target: run.target,
2179         });
2180     }
2181
2182     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2183         let target = self.target;
2184
2185         if builder.config.dry_run {
2186             return None;
2187         }
2188
2189         let bindir = builder
2190             .llvm_out(target)
2191             .join("bin");
2192         let lldb_exe = bindir.join(exe("lldb", &target));
2193         if !lldb_exe.exists() {
2194             return None;
2195         }
2196
2197         builder.info(&format!("Dist Lldb ({})", target));
2198         let src = builder.src.join("src/llvm-project/lldb");
2199         let name = pkgname(builder, "lldb");
2200
2201         let tmp = tmpdir(builder);
2202         let image = tmp.join("lldb-image");
2203         drop(fs::remove_dir_all(&image));
2204
2205         // Prepare the image directory
2206         let root = image.join("lib/rustlib").join(&*target);
2207         let dst = root.join("bin");
2208         t!(fs::create_dir_all(&dst));
2209         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2210             let exe = bindir.join(exe(program, &target));
2211             builder.install(&exe, &dst, 0o755);
2212         }
2213
2214         // The libraries.
2215         let libdir = builder.llvm_out(target).join("lib");
2216         let dst = root.join("lib");
2217         t!(fs::create_dir_all(&dst));
2218         for entry in t!(fs::read_dir(&libdir)) {
2219             let entry = entry.unwrap();
2220             if let Ok(name) = entry.file_name().into_string() {
2221                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2222                     if t!(entry.file_type()).is_symlink() {
2223                         builder.copy_to_folder(&entry.path(), &dst);
2224                     } else {
2225                        builder.install(&entry.path(), &dst, 0o755);
2226                     }
2227                 }
2228             }
2229         }
2230
2231         // The lldb scripts might be installed in lib/python$version
2232         // or in lib64/python$version.  If lib64 exists, use it;
2233         // otherwise lib.
2234         let libdir = builder.llvm_out(target).join("lib64");
2235         let (libdir, libdir_name) = if libdir.exists() {
2236             (libdir, "lib64")
2237         } else {
2238             (builder.llvm_out(target).join("lib"), "lib")
2239         };
2240         for entry in t!(fs::read_dir(&libdir)) {
2241             let entry = t!(entry);
2242             if let Ok(name) = entry.file_name().into_string() {
2243                 if name.starts_with("python") {
2244                     let dst = root.join(libdir_name)
2245                         .join(entry.file_name());
2246                     t!(fs::create_dir_all(&dst));
2247                     builder.cp_r(&entry.path(), &dst);
2248                     break;
2249                 }
2250             }
2251         }
2252
2253         // Prepare the overlay
2254         let overlay = tmp.join("lldb-overlay");
2255         drop(fs::remove_dir_all(&overlay));
2256         builder.create_dir(&overlay);
2257         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2258         builder.create(&overlay.join("version"), &builder.lldb_vers());
2259
2260         // Generate the installer tarball
2261         let mut cmd = rust_installer(builder);
2262         cmd.arg("generate")
2263             .arg("--product-name=Rust")
2264             .arg("--rel-manifest-dir=rustlib")
2265             .arg("--success-message=lldb-installed.")
2266             .arg("--image-dir").arg(&image)
2267             .arg("--work-dir").arg(&tmpdir(builder))
2268             .arg("--output-dir").arg(&distdir(builder))
2269             .arg("--non-installed-overlay").arg(&overlay)
2270             .arg(format!("--package-name={}-{}", name, target))
2271             .arg("--legacy-manifest-dirs=rustlib,cargo")
2272             .arg("--component-name=lldb-preview");
2273
2274
2275         builder.run(&mut cmd);
2276         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2277     }
2278 }