src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472
 473             // Copy rustc/rustdoc binaries
 474             t!(fs::create_dir_all(image.join("bin")));
 475             builder.cp_r(&src.join("bin"), &image.join("bin"));
 476
 477             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 478
 479             let libdir_relative = builder.libdir_relative(compiler);
 480
 481             // Copy runtime DLLs needed by the compiler
 482             if libdir_relative.to_str() != Some("bin") {
 483                 let libdir = builder.rustc_libdir(compiler);
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             // Don't use custom libdir here because ^lib/ will be resolved again
 489                             // with installer
 490                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap()
 499                 .strip_prefix(builder.sysroot_libdir_relative(compiler)).unwrap();
 500             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 501             let backends_dst = image.join("lib").join(&backends_rel);
 502
 503             t!(fs::create_dir_all(&backends_dst));
 504             builder.cp_r(&backends_src, &backends_dst);
 505
 506             // Copy libLLVM.so to the lib dir as well, if needed. While not
 507             // technically needed by rustc itself it's needed by lots of other
 508             // components like the llvm tools and LLD. LLD is included below and
 509             // tools/LLDB come later, so let's just throw it in the rustc
 510             // component for now.
 511             maybe_install_llvm_dylib(builder, host, image);
 512
 513             // Copy over lld if it's there
 514             if builder.config.lld_enabled {
 515                 let exe = exe("rust-lld", &compiler.host);
 516                 let src = builder.sysroot_libdir(compiler, host)
 517                     .parent()
 518                     .unwrap()
 519                     .join("bin")
 520                     .join(&exe);
 521                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 522                 let dst = image.join("lib/rustlib")
 523                     .join(&*host)
 524                     .join("bin")
 525                     .join(&exe);
 526                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 527                 builder.copy(&src, &dst);
 528             }
 529
 530             // Man pages
 531             t!(fs::create_dir_all(image.join("share/man/man1")));
 532             let man_src = builder.src.join("src/doc/man");
 533             let man_dst = image.join("share/man/man1");
 534
 535             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 536             let time = env::var("SOURCE_DATE_EPOCH")
 537                 .map(|timestamp| {
 538                     let epoch = timestamp.parse().map_err(|err| {
 539                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 540                     }).unwrap();
 541
 542                     time::at(Timespec::new(epoch, 0))
 543                 })
 544                 .unwrap_or_else(|_| time::now());
 545
 546             let month_year = t!(time::strftime("%B %Y", &time));
 547             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 548             // to hardlink, and we don't want to edit the source templates
 549             for file_entry in builder.read_dir(&man_src) {
 550                 let page_src = file_entry.path();
 551                 let page_dst = man_dst.join(file_entry.file_name());
 552                 t!(fs::copy(&page_src, &page_dst));
 553                 // template in month/year and version number
 554                 builder.replace_in_file(&page_dst,
 555                                 &[("<INSERT DATE HERE>", &month_year),
 556                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 557             }
 558
 559             // Debugger scripts
 560             builder.ensure(DebuggerScripts {
 561                 sysroot: INTERNER.intern_path(image.to_owned()),
 562                 host,
 563             });
 564
 565             // Misc license info
 566             let cp = |file: &str| {
 567                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 568             };
 569             cp("COPYRIGHT");
 570             cp("LICENSE-APACHE");
 571             cp("LICENSE-MIT");
 572             cp("README.md");
 573         }
 574     }
 575 }
 576
 577 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 578 pub struct DebuggerScripts {
 579     pub sysroot: Interned<PathBuf>,
 580     pub host: Interned<String>,
 581 }
 582
 583 impl Step for DebuggerScripts {
 584     type Output = ();
 585
 586     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 587         run.path("src/lldb_batchmode.py")
 588     }
 589
 590     fn make_run(run: RunConfig<'_>) {
 591         run.builder.ensure(DebuggerScripts {
 592             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 593             host: run.target,
 594         });
 595     }
 596
 597     /// Copies debugger scripts for `target` into the `sysroot` specified.
 598     fn run(self, builder: &Builder<'_>) {
 599         let host = self.host;
 600         let sysroot = self.sysroot;
 601         let dst = sysroot.join("lib/rustlib/etc");
 602         t!(fs::create_dir_all(&dst));
 603         let cp_debugger_script = |file: &str| {
 604             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 605         };
 606         if host.contains("windows-msvc") {
 607             // windbg debugger scripts
 608             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 609                 0o755);
 610
 611             cp_debugger_script("natvis/intrinsic.natvis");
 612             cp_debugger_script("natvis/liballoc.natvis");
 613             cp_debugger_script("natvis/libcore.natvis");
 614         } else {
 615             cp_debugger_script("debugger_pretty_printers_common.py");
 616
 617             // gdb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 619                     0o755);
 620             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 621                     0o755);
 622
 623             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 624             cp_debugger_script("gdb_rust_pretty_printing.py");
 625
 626             // lldb debugger scripts
 627             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 628                     0o755);
 629
 630             cp_debugger_script("lldb_rust_formatters.py");
 631         }
 632     }
 633 }
 634
 635 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 636 pub struct Std {
 637     pub compiler: Compiler,
 638     pub target: Interned<String>,
 639 }
 640
 641 impl Step for Std {
 642     type Output = PathBuf;
 643     const DEFAULT: bool = true;
 644
 645     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 646         run.path("src/libstd")
 647     }
 648
 649     fn make_run(run: RunConfig<'_>) {
 650         run.builder.ensure(Std {
 651             compiler: run.builder.compiler_for(
 652                 run.builder.top_stage,
 653                 run.builder.config.build,
 654                 run.target,
 655             ),
 656             target: run.target,
 657         });
 658     }
 659
 660     fn run(self, builder: &Builder<'_>) -> PathBuf {
 661         let compiler = self.compiler;
 662         let target = self.target;
 663
 664         let name = pkgname(builder, "rust-std");
 665         builder.info(&format!("Dist std stage{} ({} -> {})",
 666             compiler.stage, &compiler.host, target));
 667
 668         // The only true set of target libraries came from the build triple, so
 669         // let's reduce redundant work by only producing archives from that host.
 670         if compiler.host != builder.config.build {
 671             builder.info("\tskipping, not a build host");
 672             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 673         }
 674
 675         // We want to package up as many target libraries as possible
 676         // for the `rust-std` package, so if this is a host target we
 677         // depend on librustc and otherwise we just depend on libtest.
 678         if builder.hosts.iter().any(|t| t == target) {
 679             builder.ensure(compile::Rustc { compiler, target });
 680         } else {
 681             builder.ensure(compile::Std { compiler, target });
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             // Find the actual compiler (handling the full bootstrap option) which
 741             // produced the save-analysis data because that data isn't copied
 742             // through the sysroot uplifting.
 743             compiler: run.builder.compiler_for(
 744                 run.builder.top_stage,
 745                 run.builder.config.build,
 746                 run.target,
 747             ),
 748             target: run.target,
 749         });
 750     }
 751
 752     /// Creates a tarball of save-analysis metadata, if available.
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756         assert!(builder.config.extended);
 757         builder.info("Dist analysis");
 758         let name = pkgname(builder, "rust-analysis");
 759
 760         if &compiler.host != builder.config.build {
 761             builder.info("\tskipping, not a build host");
 762             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 763         }
 764
 765         builder.ensure(Std { compiler, target });
 766
 767         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 768
 769         let src = builder.stage_out(compiler, Mode::Std)
 770             .join(target).join(builder.cargo_dir()).join("deps");
 771
 772         let image_src = src.join("save-analysis");
 773         let dst = image.join("lib/rustlib").join(target).join("analysis");
 774         t!(fs::create_dir_all(&dst));
 775         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 776         builder.cp_r(&image_src, &dst);
 777
 778         let mut cmd = rust_installer(builder);
 779         cmd.arg("generate")
 780            .arg("--product-name=Rust")
 781            .arg("--rel-manifest-dir=rustlib")
 782            .arg("--success-message=save-analysis-saved.")
 783            .arg("--image-dir").arg(&image)
 784            .arg("--work-dir").arg(&tmpdir(builder))
 785            .arg("--output-dir").arg(&distdir(builder))
 786            .arg(format!("--package-name={}-{}", name, target))
 787            .arg(format!("--component-name=rust-analysis-{}", target))
 788            .arg("--legacy-manifest-dirs=rustlib,cargo");
 789         builder.run(&mut cmd);
 790         builder.remove_dir(&image);
 791         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 792     }
 793 }
 794
 795 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 796     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 797         let spath = match path.to_str() {
 798             Some(path) => path,
 799             None => return false,
 800         };
 801         if spath.ends_with("~") || spath.ends_with(".pyc") {
 802             return false
 803         }
 804
 805         const LLVM_PROJECTS: &[&str] = &[
 806             "llvm-project/clang", "llvm-project\\clang",
 807             "llvm-project/libunwind", "llvm-project\\libunwind",
 808             "llvm-project/lld", "llvm-project\\lld",
 809             "llvm-project/lldb", "llvm-project\\lldb",
 810             "llvm-project/llvm", "llvm-project\\llvm",
 811             "llvm-project/compiler-rt", "llvm-project\\compiler-rt",
 812         ];
 813         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 814             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 815         {
 816             return false;
 817         }
 818
 819         const LLVM_TEST: &[&str] = &[
 820             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 821             "llvm-emscripten/test", "llvm-emscripten\\test",
 822         ];
 823         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 824             (spath.ends_with(".ll") ||
 825              spath.ends_with(".td") ||
 826              spath.ends_with(".s")) {
 827             return false
 828         }
 829         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 830             return false
 831         }
 832
 833         let full_path = Path::new(dir).join(path);
 834         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 835             return false;
 836         }
 837
 838         let excludes = [
 839             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 840             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 841             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 842             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 843         ];
 844         !path.iter()
 845              .map(|s| s.to_str().unwrap())
 846              .any(|s| excludes.contains(&s))
 847     }
 848
 849     // Copy the directories using our filter
 850     for item in src_dirs {
 851         let dst = &dst_dir.join(item);
 852         t!(fs::create_dir_all(dst));
 853         builder.cp_filtered(
 854             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 855     }
 856 }
 857
 858 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 859 pub struct Src;
 860
 861 impl Step for Src {
 862     /// The output path of the src installer tarball
 863     type Output = PathBuf;
 864     const DEFAULT: bool = true;
 865     const ONLY_HOSTS: bool = true;
 866
 867     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 868         run.path("src")
 869     }
 870
 871     fn make_run(run: RunConfig<'_>) {
 872         run.builder.ensure(Src);
 873     }
 874
 875     /// Creates the `rust-src` installer component
 876     fn run(self, builder: &Builder<'_>) -> PathBuf {
 877         builder.info("Dist src");
 878
 879         let name = pkgname(builder, "rust-src");
 880         let image = tmpdir(builder).join(format!("{}-image", name));
 881         let _ = fs::remove_dir_all(&image);
 882
 883         let dst = image.join("lib/rustlib/src");
 884         let dst_src = dst.join("rust");
 885         t!(fs::create_dir_all(&dst_src));
 886
 887         let src_files = [
 888             "Cargo.lock",
 889         ];
 890         // This is the reduced set of paths which will become the rust-src component
 891         // (essentially libstd and all of its path dependencies)
 892         let std_src_dirs = [
 893             "src/build_helper",
 894             "src/liballoc",
 895             "src/libcore",
 896             "src/libpanic_abort",
 897             "src/libpanic_unwind",
 898             "src/librustc_asan",
 899             "src/librustc_lsan",
 900             "src/librustc_msan",
 901             "src/librustc_tsan",
 902             "src/libstd",
 903             "src/libunwind",
 904             "src/libtest",
 905             "src/libterm",
 906             "src/libprofiler_builtins",
 907             "src/stdarch",
 908             "src/libproc_macro",
 909             "src/tools/rustc-std-workspace-core",
 910             "src/tools/rustc-std-workspace-alloc",
 911             "src/tools/rustc-std-workspace-std",
 912             "src/librustc",
 913             "src/libsyntax",
 914         ];
 915
 916         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 917         for file in src_files.iter() {
 918             builder.copy(&builder.src.join(file), &dst_src.join(file));
 919         }
 920
 921         // Create source tarball in rust-installer format
 922         let mut cmd = rust_installer(builder);
 923         cmd.arg("generate")
 924            .arg("--product-name=Rust")
 925            .arg("--rel-manifest-dir=rustlib")
 926            .arg("--success-message=Awesome-Source.")
 927            .arg("--image-dir").arg(&image)
 928            .arg("--work-dir").arg(&tmpdir(builder))
 929            .arg("--output-dir").arg(&distdir(builder))
 930            .arg(format!("--package-name={}", name))
 931            .arg("--component-name=rust-src")
 932            .arg("--legacy-manifest-dirs=rustlib,cargo");
 933         builder.run(&mut cmd);
 934
 935         builder.remove_dir(&image);
 936         distdir(builder).join(&format!("{}.tar.gz", name))
 937     }
 938 }
 939
 940 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 941 pub struct PlainSourceTarball;
 942
 943 impl Step for PlainSourceTarball {
 944     /// Produces the location of the tarball generated
 945     type Output = PathBuf;
 946     const DEFAULT: bool = true;
 947     const ONLY_HOSTS: bool = true;
 948
 949     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 950         let builder = run.builder;
 951         run.path("src").default_condition(builder.config.rust_dist_src)
 952     }
 953
 954     fn make_run(run: RunConfig<'_>) {
 955         run.builder.ensure(PlainSourceTarball);
 956     }
 957
 958     /// Creates the plain source tarball
 959     fn run(self, builder: &Builder<'_>) -> PathBuf {
 960         builder.info("Create plain source tarball");
 961
 962         // Make sure that the root folder of tarball has the correct name
 963         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 964         let plain_dst_src = tmpdir(builder).join(&plain_name);
 965         let _ = fs::remove_dir_all(&plain_dst_src);
 966         t!(fs::create_dir_all(&plain_dst_src));
 967
 968         // This is the set of root paths which will become part of the source package
 969         let src_files = [
 970             "COPYRIGHT",
 971             "LICENSE-APACHE",
 972             "LICENSE-MIT",
 973             "CONTRIBUTING.md",
 974             "README.md",
 975             "RELEASES.md",
 976             "configure",
 977             "x.py",
 978             "config.toml.example",
 979             "Cargo.toml",
 980             "Cargo.lock",
 981         ];
 982         let src_dirs = [
 983             "src",
 984         ];
 985
 986         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 987
 988         // Copy the files normally
 989         for item in &src_files {
 990             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 991         }
 992
 993         // Create the version file
 994         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 995         if let Some(sha) = builder.rust_sha() {
 996             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 997         }
 998
 999         // If we're building from git sources, we need to vendor a complete distribution.
1000         if builder.rust_info.is_git() {
1001             // Vendor all Cargo dependencies
1002             let mut cmd = Command::new(&builder.initial_cargo);
1003             cmd.arg("vendor")
1004                .current_dir(&plain_dst_src);
1005             builder.run(&mut cmd);
1006         }
1007
1008         // Create plain source tarball
1009         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1010         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1011         tarball.set_extension(""); // strip .gz
1012         tarball.set_extension(""); // strip .tar
1013         if let Some(dir) = tarball.parent() {
1014             builder.create_dir(&dir);
1015         }
1016         builder.info("running installer");
1017         let mut cmd = rust_installer(builder);
1018         cmd.arg("tarball")
1019            .arg("--input").arg(&plain_name)
1020            .arg("--output").arg(&tarball)
1021            .arg("--work-dir=.")
1022            .current_dir(tmpdir(builder));
1023         builder.run(&mut cmd);
1024         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1025     }
1026 }
1027
1028 // We have to run a few shell scripts, which choke quite a bit on both `\`
1029 // characters and on `C:\` paths, so normalize both of them away.
1030 pub fn sanitize_sh(path: &Path) -> String {
1031     let path = path.to_str().unwrap().replace("\\", "/");
1032     return change_drive(&path).unwrap_or(path);
1033
1034     fn change_drive(s: &str) -> Option<String> {
1035         let mut ch = s.chars();
1036         let drive = ch.next().unwrap_or('C');
1037         if ch.next() != Some(':') {
1038             return None
1039         }
1040         if ch.next() != Some('/') {
1041             return None
1042         }
1043         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1044     }
1045 }
1046
1047 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1048 pub struct Cargo {
1049     pub compiler: Compiler,
1050     pub target: Interned<String>,
1051 }
1052
1053 impl Step for Cargo {
1054     type Output = PathBuf;
1055     const ONLY_HOSTS: bool = true;
1056
1057     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1058         run.path("cargo")
1059     }
1060
1061     fn make_run(run: RunConfig<'_>) {
1062         run.builder.ensure(Cargo {
1063             compiler: run.builder.compiler_for(
1064                 run.builder.top_stage,
1065                 run.builder.config.build,
1066                 run.target,
1067             ),
1068             target: run.target,
1069         });
1070     }
1071
1072     fn run(self, builder: &Builder<'_>) -> PathBuf {
1073         let compiler = self.compiler;
1074         let target = self.target;
1075
1076         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1077         let src = builder.src.join("src/tools/cargo");
1078         let etc = src.join("src/etc");
1079         let release_num = builder.release_num("cargo");
1080         let name = pkgname(builder, "cargo");
1081         let version = builder.cargo_info.version(builder, &release_num);
1082
1083         let tmp = tmpdir(builder);
1084         let image = tmp.join("cargo-image");
1085         drop(fs::remove_dir_all(&image));
1086         builder.create_dir(&image);
1087
1088         // Prepare the image directory
1089         builder.create_dir(&image.join("share/zsh/site-functions"));
1090         builder.create_dir(&image.join("etc/bash_completion.d"));
1091         let cargo = builder.ensure(tool::Cargo { compiler, target });
1092         builder.install(&cargo, &image.join("bin"), 0o755);
1093         for man in t!(etc.join("man").read_dir()) {
1094             let man = t!(man);
1095             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1096         }
1097         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1098         builder.copy(&etc.join("cargo.bashcomp.sh"),
1099              &image.join("etc/bash_completion.d/cargo"));
1100         let doc = image.join("share/doc/cargo");
1101         builder.install(&src.join("README.md"), &doc, 0o644);
1102         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1103         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1104         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1105
1106         // Prepare the overlay
1107         let overlay = tmp.join("cargo-overlay");
1108         drop(fs::remove_dir_all(&overlay));
1109         builder.create_dir(&overlay);
1110         builder.install(&src.join("README.md"), &overlay, 0o644);
1111         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1112         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1113         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1114         builder.create(&overlay.join("version"), &version);
1115
1116         // Generate the installer tarball
1117         let mut cmd = rust_installer(builder);
1118         cmd.arg("generate")
1119            .arg("--product-name=Rust")
1120            .arg("--rel-manifest-dir=rustlib")
1121            .arg("--success-message=Rust-is-ready-to-roll.")
1122            .arg("--image-dir").arg(&image)
1123            .arg("--work-dir").arg(&tmpdir(builder))
1124            .arg("--output-dir").arg(&distdir(builder))
1125            .arg("--non-installed-overlay").arg(&overlay)
1126            .arg(format!("--package-name={}-{}", name, target))
1127            .arg("--component-name=cargo")
1128            .arg("--legacy-manifest-dirs=rustlib,cargo");
1129         builder.run(&mut cmd);
1130         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1131     }
1132 }
1133
1134 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1135 pub struct Rls {
1136     pub compiler: Compiler,
1137     pub target: Interned<String>,
1138 }
1139
1140 impl Step for Rls {
1141     type Output = Option<PathBuf>;
1142     const ONLY_HOSTS: bool = true;
1143
1144     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1145         run.path("rls")
1146     }
1147
1148     fn make_run(run: RunConfig<'_>) {
1149         run.builder.ensure(Rls {
1150             compiler: run.builder.compiler_for(
1151                 run.builder.top_stage,
1152                 run.builder.config.build,
1153                 run.target,
1154             ),
1155             target: run.target,
1156         });
1157     }
1158
1159     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1160         let compiler = self.compiler;
1161         let target = self.target;
1162         assert!(builder.config.extended);
1163
1164         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1165         let src = builder.src.join("src/tools/rls");
1166         let release_num = builder.release_num("rls");
1167         let name = pkgname(builder, "rls");
1168         let version = builder.rls_info.version(builder, &release_num);
1169
1170         let tmp = tmpdir(builder);
1171         let image = tmp.join("rls-image");
1172         drop(fs::remove_dir_all(&image));
1173         t!(fs::create_dir_all(&image));
1174
1175         // Prepare the image directory
1176         // We expect RLS to build, because we've exited this step above if tool
1177         // state for RLS isn't testing.
1178         let rls = builder.ensure(tool::Rls {
1179             compiler,
1180             target,
1181             extra_features: Vec::new(),
1182         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1183
1184         builder.install(&rls, &image.join("bin"), 0o755);
1185         let doc = image.join("share/doc/rls");
1186         builder.install(&src.join("README.md"), &doc, 0o644);
1187         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1188         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1189
1190         // Prepare the overlay
1191         let overlay = tmp.join("rls-overlay");
1192         drop(fs::remove_dir_all(&overlay));
1193         t!(fs::create_dir_all(&overlay));
1194         builder.install(&src.join("README.md"), &overlay, 0o644);
1195         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1196         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1197         builder.create(&overlay.join("version"), &version);
1198
1199         // Generate the installer tarball
1200         let mut cmd = rust_installer(builder);
1201         cmd.arg("generate")
1202            .arg("--product-name=Rust")
1203            .arg("--rel-manifest-dir=rustlib")
1204            .arg("--success-message=RLS-ready-to-serve.")
1205            .arg("--image-dir").arg(&image)
1206            .arg("--work-dir").arg(&tmpdir(builder))
1207            .arg("--output-dir").arg(&distdir(builder))
1208            .arg("--non-installed-overlay").arg(&overlay)
1209            .arg(format!("--package-name={}-{}", name, target))
1210            .arg("--legacy-manifest-dirs=rustlib,cargo")
1211            .arg("--component-name=rls-preview");
1212
1213         builder.run(&mut cmd);
1214         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1215     }
1216 }
1217
1218 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1219 pub struct Clippy {
1220     pub compiler: Compiler,
1221     pub target: Interned<String>,
1222 }
1223
1224 impl Step for Clippy {
1225     type Output = Option<PathBuf>;
1226     const ONLY_HOSTS: bool = true;
1227
1228     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1229         run.path("clippy")
1230     }
1231
1232     fn make_run(run: RunConfig<'_>) {
1233         run.builder.ensure(Clippy {
1234             compiler: run.builder.compiler_for(
1235                 run.builder.top_stage,
1236                 run.builder.config.build,
1237                 run.target,
1238             ),
1239             target: run.target,
1240         });
1241     }
1242
1243     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1244         let compiler = self.compiler;
1245         let target = self.target;
1246         assert!(builder.config.extended);
1247
1248         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1249         let src = builder.src.join("src/tools/clippy");
1250         let release_num = builder.release_num("clippy");
1251         let name = pkgname(builder, "clippy");
1252         let version = builder.clippy_info.version(builder, &release_num);
1253
1254         let tmp = tmpdir(builder);
1255         let image = tmp.join("clippy-image");
1256         drop(fs::remove_dir_all(&image));
1257         builder.create_dir(&image);
1258
1259         // Prepare the image directory
1260         // We expect clippy to build, because we've exited this step above if tool
1261         // state for clippy isn't testing.
1262         let clippy = builder.ensure(tool::Clippy {
1263             compiler,
1264             target,
1265             extra_features: Vec::new(),
1266         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1267         let cargoclippy = builder.ensure(tool::CargoClippy {
1268             compiler,
1269             target, extra_features: Vec::new()
1270         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1271
1272         builder.install(&clippy, &image.join("bin"), 0o755);
1273         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1274         let doc = image.join("share/doc/clippy");
1275         builder.install(&src.join("README.md"), &doc, 0o644);
1276         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1277         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1278
1279         // Prepare the overlay
1280         let overlay = tmp.join("clippy-overlay");
1281         drop(fs::remove_dir_all(&overlay));
1282         t!(fs::create_dir_all(&overlay));
1283         builder.install(&src.join("README.md"), &overlay, 0o644);
1284         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1285         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1286         builder.create(&overlay.join("version"), &version);
1287
1288         // Generate the installer tarball
1289         let mut cmd = rust_installer(builder);
1290         cmd.arg("generate")
1291            .arg("--product-name=Rust")
1292            .arg("--rel-manifest-dir=rustlib")
1293            .arg("--success-message=clippy-ready-to-serve.")
1294            .arg("--image-dir").arg(&image)
1295            .arg("--work-dir").arg(&tmpdir(builder))
1296            .arg("--output-dir").arg(&distdir(builder))
1297            .arg("--non-installed-overlay").arg(&overlay)
1298            .arg(format!("--package-name={}-{}", name, target))
1299            .arg("--legacy-manifest-dirs=rustlib,cargo")
1300            .arg("--component-name=clippy-preview");
1301
1302         builder.run(&mut cmd);
1303         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1304     }
1305 }
1306
1307 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1308 pub struct Miri {
1309     pub compiler: Compiler,
1310     pub target: Interned<String>,
1311 }
1312
1313 impl Step for Miri {
1314     type Output = Option<PathBuf>;
1315     const ONLY_HOSTS: bool = true;
1316
1317     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1318         run.path("miri")
1319     }
1320
1321     fn make_run(run: RunConfig<'_>) {
1322         run.builder.ensure(Miri {
1323             compiler: run.builder.compiler_for(
1324                 run.builder.top_stage,
1325                 run.builder.config.build,
1326                 run.target,
1327             ),
1328             target: run.target,
1329         });
1330     }
1331
1332     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1333         let compiler = self.compiler;
1334         let target = self.target;
1335         assert!(builder.config.extended);
1336
1337         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1338         let src = builder.src.join("src/tools/miri");
1339         let release_num = builder.release_num("miri");
1340         let name = pkgname(builder, "miri");
1341         let version = builder.miri_info.version(builder, &release_num);
1342
1343         let tmp = tmpdir(builder);
1344         let image = tmp.join("miri-image");
1345         drop(fs::remove_dir_all(&image));
1346         builder.create_dir(&image);
1347
1348         // Prepare the image directory
1349         // We expect miri to build, because we've exited this step above if tool
1350         // state for miri isn't testing.
1351         let miri = builder.ensure(tool::Miri {
1352             compiler,
1353             target,
1354             extra_features: Vec::new(),
1355         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1356         let cargomiri = builder.ensure(tool::CargoMiri {
1357             compiler,
1358             target,
1359             extra_features: Vec::new()
1360         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1361
1362         builder.install(&miri, &image.join("bin"), 0o755);
1363         builder.install(&cargomiri, &image.join("bin"), 0o755);
1364         let doc = image.join("share/doc/miri");
1365         builder.install(&src.join("README.md"), &doc, 0o644);
1366         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1367         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1368
1369         // Prepare the overlay
1370         let overlay = tmp.join("miri-overlay");
1371         drop(fs::remove_dir_all(&overlay));
1372         t!(fs::create_dir_all(&overlay));
1373         builder.install(&src.join("README.md"), &overlay, 0o644);
1374         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1375         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1376         builder.create(&overlay.join("version"), &version);
1377
1378         // Generate the installer tarball
1379         let mut cmd = rust_installer(builder);
1380         cmd.arg("generate")
1381            .arg("--product-name=Rust")
1382            .arg("--rel-manifest-dir=rustlib")
1383            .arg("--success-message=miri-ready-to-serve.")
1384            .arg("--image-dir").arg(&image)
1385            .arg("--work-dir").arg(&tmpdir(builder))
1386            .arg("--output-dir").arg(&distdir(builder))
1387            .arg("--non-installed-overlay").arg(&overlay)
1388            .arg(format!("--package-name={}-{}", name, target))
1389            .arg("--legacy-manifest-dirs=rustlib,cargo")
1390            .arg("--component-name=miri-preview");
1391
1392         builder.run(&mut cmd);
1393         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1394     }
1395 }
1396
1397 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1398 pub struct Rustfmt {
1399     pub compiler: Compiler,
1400     pub target: Interned<String>,
1401 }
1402
1403 impl Step for Rustfmt {
1404     type Output = Option<PathBuf>;
1405     const ONLY_HOSTS: bool = true;
1406
1407     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1408         run.path("rustfmt")
1409     }
1410
1411     fn make_run(run: RunConfig<'_>) {
1412         run.builder.ensure(Rustfmt {
1413             compiler: run.builder.compiler_for(
1414                 run.builder.top_stage,
1415                 run.builder.config.build,
1416                 run.target,
1417             ),
1418             target: run.target,
1419         });
1420     }
1421
1422     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1423         let compiler = self.compiler;
1424         let target = self.target;
1425
1426         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1427         let src = builder.src.join("src/tools/rustfmt");
1428         let release_num = builder.release_num("rustfmt");
1429         let name = pkgname(builder, "rustfmt");
1430         let version = builder.rustfmt_info.version(builder, &release_num);
1431
1432         let tmp = tmpdir(builder);
1433         let image = tmp.join("rustfmt-image");
1434         drop(fs::remove_dir_all(&image));
1435         builder.create_dir(&image);
1436
1437         // Prepare the image directory
1438         let rustfmt = builder.ensure(tool::Rustfmt {
1439             compiler,
1440             target,
1441             extra_features: Vec::new(),
1442         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1443         let cargofmt = builder.ensure(tool::Cargofmt {
1444             compiler,
1445             target,
1446             extra_features: Vec::new(),
1447         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1448
1449         builder.install(&rustfmt, &image.join("bin"), 0o755);
1450         builder.install(&cargofmt, &image.join("bin"), 0o755);
1451         let doc = image.join("share/doc/rustfmt");
1452         builder.install(&src.join("README.md"), &doc, 0o644);
1453         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1454         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1455
1456         // Prepare the overlay
1457         let overlay = tmp.join("rustfmt-overlay");
1458         drop(fs::remove_dir_all(&overlay));
1459         builder.create_dir(&overlay);
1460         builder.install(&src.join("README.md"), &overlay, 0o644);
1461         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1462         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1463         builder.create(&overlay.join("version"), &version);
1464
1465         // Generate the installer tarball
1466         let mut cmd = rust_installer(builder);
1467         cmd.arg("generate")
1468            .arg("--product-name=Rust")
1469            .arg("--rel-manifest-dir=rustlib")
1470            .arg("--success-message=rustfmt-ready-to-fmt.")
1471            .arg("--image-dir").arg(&image)
1472            .arg("--work-dir").arg(&tmpdir(builder))
1473            .arg("--output-dir").arg(&distdir(builder))
1474            .arg("--non-installed-overlay").arg(&overlay)
1475            .arg(format!("--package-name={}-{}", name, target))
1476            .arg("--legacy-manifest-dirs=rustlib,cargo")
1477            .arg("--component-name=rustfmt-preview");
1478
1479         builder.run(&mut cmd);
1480         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1481     }
1482 }
1483
1484 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1485 pub struct Extended {
1486     stage: u32,
1487     host: Interned<String>,
1488     target: Interned<String>,
1489 }
1490
1491 impl Step for Extended {
1492     type Output = ();
1493     const DEFAULT: bool = true;
1494     const ONLY_HOSTS: bool = true;
1495
1496     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1497         let builder = run.builder;
1498         run.path("extended").default_condition(builder.config.extended)
1499     }
1500
1501     fn make_run(run: RunConfig<'_>) {
1502         run.builder.ensure(Extended {
1503             stage: run.builder.top_stage,
1504             host: run.builder.config.build,
1505             target: run.target,
1506         });
1507     }
1508
1509     /// Creates a combined installer for the specified target in the provided stage.
1510     fn run(self, builder: &Builder<'_>) {
1511         let target = self.target;
1512         let stage = self.stage;
1513         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1514
1515         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1516
1517         let rustc_installer = builder.ensure(Rustc {
1518             compiler: builder.compiler(stage, target),
1519         });
1520         let cargo_installer = builder.ensure(Cargo { compiler, target });
1521         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1522         let rls_installer = builder.ensure(Rls { compiler, target });
1523         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1524         let clippy_installer = builder.ensure(Clippy { compiler, target });
1525         let miri_installer = builder.ensure(Miri { compiler, target });
1526         let lldb_installer = builder.ensure(Lldb { target });
1527         let mingw_installer = builder.ensure(Mingw { host: target });
1528         let analysis_installer = builder.ensure(Analysis { compiler, target });
1529
1530         let docs_installer = builder.ensure(Docs { host: target, });
1531         let std_installer = builder.ensure(Std {
1532             compiler: builder.compiler(stage, target),
1533             target,
1534         });
1535
1536         let tmp = tmpdir(builder);
1537         let overlay = tmp.join("extended-overlay");
1538         let etc = builder.src.join("src/etc/installer");
1539         let work = tmp.join("work");
1540
1541         let _ = fs::remove_dir_all(&overlay);
1542         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1543         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1544         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1545         let version = builder.rust_version();
1546         builder.create(&overlay.join("version"), &version);
1547         if let Some(sha) = builder.rust_sha() {
1548             builder.create(&overlay.join("git-commit-hash"), &sha);
1549         }
1550         builder.install(&etc.join("README.md"), &overlay, 0o644);
1551
1552         // When rust-std package split from rustc, we needed to ensure that during
1553         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1554         // the std files during uninstall. To do this ensure that rustc comes
1555         // before rust-std in the list below.
1556         let mut tarballs = Vec::new();
1557         tarballs.push(rustc_installer);
1558         tarballs.push(cargo_installer);
1559         tarballs.extend(rls_installer.clone());
1560         tarballs.extend(clippy_installer.clone());
1561         tarballs.extend(miri_installer.clone());
1562         tarballs.extend(rustfmt_installer.clone());
1563         tarballs.extend(llvm_tools_installer);
1564         tarballs.extend(lldb_installer);
1565         tarballs.push(analysis_installer);
1566         tarballs.push(std_installer);
1567         if builder.config.docs {
1568             tarballs.push(docs_installer);
1569         }
1570         if target.contains("pc-windows-gnu") {
1571             tarballs.push(mingw_installer.unwrap());
1572         }
1573         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1574         for tarball in &tarballs[1..] {
1575             input_tarballs.push(",");
1576             input_tarballs.push(tarball);
1577         }
1578
1579         let mut cmd = rust_installer(builder);
1580         cmd.arg("combine")
1581             .arg("--product-name=Rust")
1582             .arg("--rel-manifest-dir=rustlib")
1583             .arg("--success-message=Rust-is-ready-to-roll.")
1584             .arg("--work-dir").arg(&work)
1585             .arg("--output-dir").arg(&distdir(builder))
1586             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1587             .arg("--legacy-manifest-dirs=rustlib,cargo")
1588             .arg("--input-tarballs").arg(input_tarballs)
1589             .arg("--non-installed-overlay").arg(&overlay);
1590         builder.run(&mut cmd);
1591
1592         let mut license = String::new();
1593         license += &builder.read(&builder.src.join("COPYRIGHT"));
1594         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1595         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1596         license.push_str("\n");
1597         license.push_str("\n");
1598
1599         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1600         let mut rtf = rtf.to_string();
1601         rtf.push_str("\n");
1602         for line in license.lines() {
1603             rtf.push_str(line);
1604             rtf.push_str("\\line ");
1605         }
1606         rtf.push_str("}");
1607
1608         fn filter(contents: &str, marker: &str) -> String {
1609             let start = format!("tool-{}-start", marker);
1610             let end = format!("tool-{}-end", marker);
1611             let mut lines = Vec::new();
1612             let mut omitted = false;
1613             for line in contents.lines() {
1614                 if line.contains(&start) {
1615                     omitted = true;
1616                 } else if line.contains(&end) {
1617                     omitted = false;
1618                 } else if !omitted {
1619                     lines.push(line);
1620                 }
1621             }
1622
1623             lines.join("\n")
1624         }
1625
1626         let xform = |p: &Path| {
1627             let mut contents = t!(fs::read_to_string(p));
1628             if rls_installer.is_none() {
1629                 contents = filter(&contents, "rls");
1630             }
1631             if clippy_installer.is_none() {
1632                 contents = filter(&contents, "clippy");
1633             }
1634             if miri_installer.is_none() {
1635                 contents = filter(&contents, "miri");
1636             }
1637             if rustfmt_installer.is_none() {
1638                 contents = filter(&contents, "rustfmt");
1639             }
1640             let ret = tmp.join(p.file_name().unwrap());
1641             t!(fs::write(&ret, &contents));
1642             ret
1643         };
1644
1645         if target.contains("apple-darwin") {
1646             let pkg = tmp.join("pkg");
1647             let _ = fs::remove_dir_all(&pkg);
1648
1649             let pkgbuild = |component: &str| {
1650                 let mut cmd = Command::new("pkgbuild");
1651                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1652                     .arg("--scripts").arg(pkg.join(component))
1653                     .arg("--nopayload")
1654                     .arg(pkg.join(component).with_extension("pkg"));
1655                 builder.run(&mut cmd);
1656             };
1657
1658             let prepare = |name: &str| {
1659                 builder.create_dir(&pkg.join(name));
1660                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1661                         &pkg.join(name));
1662                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1663                 pkgbuild(name);
1664             };
1665             prepare("rustc");
1666             prepare("cargo");
1667             prepare("rust-docs");
1668             prepare("rust-std");
1669             prepare("rust-analysis");
1670
1671             if rls_installer.is_some() {
1672                 prepare("rls");
1673             }
1674             if clippy_installer.is_some() {
1675                 prepare("clippy");
1676             }
1677             if miri_installer.is_some() {
1678                 prepare("miri");
1679             }
1680
1681             // create an 'uninstall' package
1682             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1683             pkgbuild("uninstall");
1684
1685             builder.create_dir(&pkg.join("res"));
1686             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1687             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1688             let mut cmd = Command::new("productbuild");
1689             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1690                 .arg("--resources").arg(pkg.join("res"))
1691                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1692                                                     pkgname(builder, "rust"),
1693                                                     target)))
1694                 .arg("--package-path").arg(&pkg);
1695             builder.run(&mut cmd);
1696         }
1697
1698         if target.contains("windows") {
1699             let exe = tmp.join("exe");
1700             let _ = fs::remove_dir_all(&exe);
1701
1702             let prepare = |name: &str| {
1703                 builder.create_dir(&exe.join(name));
1704                 let dir = if name == "rust-std" || name == "rust-analysis" {
1705                     format!("{}-{}", name, target)
1706                 } else if name == "rls" {
1707                     "rls-preview".to_string()
1708                 } else if name == "clippy" {
1709                     "clippy-preview".to_string()
1710                 } else if name == "miri" {
1711                     "miri-preview".to_string()
1712                 } else {
1713                     name.to_string()
1714                 };
1715                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1716                             .join(dir),
1717                         &exe.join(name));
1718                 builder.remove(&exe.join(name).join("manifest.in"));
1719             };
1720             prepare("rustc");
1721             prepare("cargo");
1722             prepare("rust-analysis");
1723             prepare("rust-docs");
1724             prepare("rust-std");
1725             if rls_installer.is_some() {
1726                 prepare("rls");
1727             }
1728             if clippy_installer.is_some() {
1729                 prepare("clippy");
1730             }
1731             if miri_installer.is_some() {
1732                 prepare("miri");
1733             }
1734             if target.contains("windows-gnu") {
1735                 prepare("rust-mingw");
1736             }
1737
1738             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1739             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1740             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1741             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1742             builder.create(&exe.join("LICENSE.txt"), &license);
1743
1744             // Generate exe installer
1745             let mut cmd = Command::new("iscc");
1746             cmd.arg("rust.iss")
1747                 .current_dir(&exe);
1748             if target.contains("windows-gnu") {
1749                 cmd.arg("/dMINGW");
1750             }
1751             add_env(builder, &mut cmd, target);
1752             builder.run(&mut cmd);
1753             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1754                     &distdir(builder),
1755                     0o755);
1756
1757             // Generate msi installer
1758             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1759             let heat = wix.join("bin/heat.exe");
1760             let candle = wix.join("bin/candle.exe");
1761             let light = wix.join("bin/light.exe");
1762
1763             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1764             builder.run(Command::new(&heat)
1765                             .current_dir(&exe)
1766                             .arg("dir")
1767                             .arg("rustc")
1768                             .args(&heat_flags)
1769                             .arg("-cg").arg("RustcGroup")
1770                             .arg("-dr").arg("Rustc")
1771                             .arg("-var").arg("var.RustcDir")
1772                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1773             builder.run(Command::new(&heat)
1774                             .current_dir(&exe)
1775                             .arg("dir")
1776                             .arg("rust-docs")
1777                             .args(&heat_flags)
1778                             .arg("-cg").arg("DocsGroup")
1779                             .arg("-dr").arg("Docs")
1780                             .arg("-var").arg("var.DocsDir")
1781                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1782                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1783             builder.run(Command::new(&heat)
1784                             .current_dir(&exe)
1785                             .arg("dir")
1786                             .arg("cargo")
1787                             .args(&heat_flags)
1788                             .arg("-cg").arg("CargoGroup")
1789                             .arg("-dr").arg("Cargo")
1790                             .arg("-var").arg("var.CargoDir")
1791                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1792                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1793             builder.run(Command::new(&heat)
1794                             .current_dir(&exe)
1795                             .arg("dir")
1796                             .arg("rust-std")
1797                             .args(&heat_flags)
1798                             .arg("-cg").arg("StdGroup")
1799                             .arg("-dr").arg("Std")
1800                             .arg("-var").arg("var.StdDir")
1801                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1802             if rls_installer.is_some() {
1803                 builder.run(Command::new(&heat)
1804                                 .current_dir(&exe)
1805                                 .arg("dir")
1806                                 .arg("rls")
1807                                 .args(&heat_flags)
1808                                 .arg("-cg").arg("RlsGroup")
1809                                 .arg("-dr").arg("Rls")
1810                                 .arg("-var").arg("var.RlsDir")
1811                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1812                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1813             }
1814             if clippy_installer.is_some() {
1815                 builder.run(Command::new(&heat)
1816                                 .current_dir(&exe)
1817                                 .arg("dir")
1818                                 .arg("clippy")
1819                                 .args(&heat_flags)
1820                                 .arg("-cg").arg("ClippyGroup")
1821                                 .arg("-dr").arg("Clippy")
1822                                 .arg("-var").arg("var.ClippyDir")
1823                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1824                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1825             }
1826             if miri_installer.is_some() {
1827                 builder.run(Command::new(&heat)
1828                                 .current_dir(&exe)
1829                                 .arg("dir")
1830                                 .arg("miri")
1831                                 .args(&heat_flags)
1832                                 .arg("-cg").arg("MiriGroup")
1833                                 .arg("-dr").arg("Miri")
1834                                 .arg("-var").arg("var.MiriDir")
1835                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1836                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1837             }
1838             builder.run(Command::new(&heat)
1839                             .current_dir(&exe)
1840                             .arg("dir")
1841                             .arg("rust-analysis")
1842                             .args(&heat_flags)
1843                             .arg("-cg").arg("AnalysisGroup")
1844                             .arg("-dr").arg("Analysis")
1845                             .arg("-var").arg("var.AnalysisDir")
1846                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1847                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1848             if target.contains("windows-gnu") {
1849                 builder.run(Command::new(&heat)
1850                                 .current_dir(&exe)
1851                                 .arg("dir")
1852                                 .arg("rust-mingw")
1853                                 .args(&heat_flags)
1854                                 .arg("-cg").arg("GccGroup")
1855                                 .arg("-dr").arg("Gcc")
1856                                 .arg("-var").arg("var.GccDir")
1857                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1858             }
1859
1860             let candle = |input: &Path| {
1861                 let output = exe.join(input.file_stem().unwrap())
1862                                 .with_extension("wixobj");
1863                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1864                 let mut cmd = Command::new(&candle);
1865                 cmd.current_dir(&exe)
1866                     .arg("-nologo")
1867                     .arg("-dRustcDir=rustc")
1868                     .arg("-dDocsDir=rust-docs")
1869                     .arg("-dCargoDir=cargo")
1870                     .arg("-dStdDir=rust-std")
1871                     .arg("-dAnalysisDir=rust-analysis")
1872                     .arg("-arch").arg(&arch)
1873                     .arg("-out").arg(&output)
1874                     .arg(&input);
1875                 add_env(builder, &mut cmd, target);
1876
1877                 if rls_installer.is_some() {
1878                     cmd.arg("-dRlsDir=rls");
1879                 }
1880                 if clippy_installer.is_some() {
1881                     cmd.arg("-dClippyDir=clippy");
1882                 }
1883                 if miri_installer.is_some() {
1884                     cmd.arg("-dMiriDir=miri");
1885                 }
1886                 if target.contains("windows-gnu") {
1887                     cmd.arg("-dGccDir=rust-mingw");
1888                 }
1889                 builder.run(&mut cmd);
1890             };
1891             candle(&xform(&etc.join("msi/rust.wxs")));
1892             candle(&etc.join("msi/ui.wxs"));
1893             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1894             candle("RustcGroup.wxs".as_ref());
1895             candle("DocsGroup.wxs".as_ref());
1896             candle("CargoGroup.wxs".as_ref());
1897             candle("StdGroup.wxs".as_ref());
1898             if rls_installer.is_some() {
1899                 candle("RlsGroup.wxs".as_ref());
1900             }
1901             if clippy_installer.is_some() {
1902                 candle("ClippyGroup.wxs".as_ref());
1903             }
1904             if miri_installer.is_some() {
1905                 candle("MiriGroup.wxs".as_ref());
1906             }
1907             candle("AnalysisGroup.wxs".as_ref());
1908
1909             if target.contains("windows-gnu") {
1910                 candle("GccGroup.wxs".as_ref());
1911             }
1912
1913             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1914             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1915             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1916
1917             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1918             let mut cmd = Command::new(&light);
1919             cmd.arg("-nologo")
1920                 .arg("-ext").arg("WixUIExtension")
1921                 .arg("-ext").arg("WixUtilExtension")
1922                 .arg("-out").arg(exe.join(&filename))
1923                 .arg("rust.wixobj")
1924                 .arg("ui.wixobj")
1925                 .arg("rustwelcomedlg.wixobj")
1926                 .arg("RustcGroup.wixobj")
1927                 .arg("DocsGroup.wixobj")
1928                 .arg("CargoGroup.wixobj")
1929                 .arg("StdGroup.wixobj")
1930                 .arg("AnalysisGroup.wixobj")
1931                 .current_dir(&exe);
1932
1933             if rls_installer.is_some() {
1934                 cmd.arg("RlsGroup.wixobj");
1935             }
1936             if clippy_installer.is_some() {
1937                 cmd.arg("ClippyGroup.wixobj");
1938             }
1939             if miri_installer.is_some() {
1940                 cmd.arg("MiriGroup.wixobj");
1941             }
1942
1943             if target.contains("windows-gnu") {
1944                 cmd.arg("GccGroup.wixobj");
1945             }
1946             // ICE57 wrongly complains about the shortcuts
1947             cmd.arg("-sice:ICE57");
1948
1949             builder.run(&mut cmd);
1950
1951             if !builder.config.dry_run {
1952                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1953             }
1954         }
1955     }
1956 }
1957
1958 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1959     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1960     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1961        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1962        .env("CFG_RELEASE", builder.rust_release())
1963        .env("CFG_VER_MAJOR", parts.next().unwrap())
1964        .env("CFG_VER_MINOR", parts.next().unwrap())
1965        .env("CFG_VER_PATCH", parts.next().unwrap())
1966        .env("CFG_VER_BUILD", "0") // just needed to build
1967        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1968        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1969        .env("CFG_BUILD", target)
1970        .env("CFG_CHANNEL", &builder.config.channel);
1971
1972     if target.contains("windows-gnu") {
1973        cmd.env("CFG_MINGW", "1")
1974           .env("CFG_ABI", "GNU");
1975     } else {
1976        cmd.env("CFG_MINGW", "0")
1977           .env("CFG_ABI", "MSVC");
1978     }
1979
1980     if target.contains("x86_64") {
1981        cmd.env("CFG_PLATFORM", "x64");
1982     } else {
1983        cmd.env("CFG_PLATFORM", "x86");
1984     }
1985 }
1986
1987 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1988 pub struct HashSign;
1989
1990 impl Step for HashSign {
1991     type Output = ();
1992     const ONLY_HOSTS: bool = true;
1993
1994     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1995         run.path("hash-and-sign")
1996     }
1997
1998     fn make_run(run: RunConfig<'_>) {
1999         run.builder.ensure(HashSign);
2000     }
2001
2002     fn run(self, builder: &Builder<'_>) {
2003         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2004         if builder.config.dry_run {
2005             return;
2006         }
2007         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2008             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2009         });
2010         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2011             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2012         });
2013         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2014             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2015         });
2016         let pass = t!(fs::read_to_string(&file));
2017
2018         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2019
2020         cmd.arg(sign);
2021         cmd.arg(distdir(builder));
2022         cmd.arg(today.trim());
2023         cmd.arg(builder.rust_package_vers());
2024         cmd.arg(addr);
2025         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2026         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2027         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2028         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2029         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2030         cmd.arg(builder.llvm_tools_package_vers());
2031         cmd.arg(builder.lldb_package_vers());
2032
2033         builder.create_dir(&distdir(builder));
2034
2035         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2036         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2037         let status = t!(child.wait());
2038         assert!(status.success());
2039     }
2040 }
2041
2042 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2043 // LLVM tools are linked dynamically.
2044 // Note: This function does no yet support Windows but we also don't support
2045 //       linking LLVM tools dynamically on Windows yet.
2046 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2047                                 target: Interned<String>,
2048                                 sysroot: &Path) {
2049     let src_libdir = builder
2050         .llvm_out(target)
2051         .join("lib");
2052     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2053     t!(fs::create_dir_all(&dst_libdir));
2054
2055     if target.contains("apple-darwin") {
2056         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2057         if llvm_dylib_path.exists() {
2058             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2059         }
2060         return
2061     }
2062
2063     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2064     // Since tools link to the latter rather than the former, we have to
2065     // follow the symlink to find out what to distribute.
2066     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2067     if llvm_dylib_path.exists() {
2068         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2069             panic!("dist: Error calling canonicalize path `{}`: {}",
2070                    llvm_dylib_path.display(), e);
2071         });
2072
2073
2074         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2075     }
2076 }
2077
2078 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2079 pub struct LlvmTools {
2080     pub target: Interned<String>,
2081 }
2082
2083 impl Step for LlvmTools {
2084     type Output = Option<PathBuf>;
2085     const ONLY_HOSTS: bool = true;
2086
2087     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2088         run.path("llvm-tools")
2089     }
2090
2091     fn make_run(run: RunConfig<'_>) {
2092         run.builder.ensure(LlvmTools {
2093             target: run.target,
2094         });
2095     }
2096
2097     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2098         let target = self.target;
2099         assert!(builder.config.extended);
2100
2101         /* run only if llvm-config isn't used */
2102         if let Some(config) = builder.config.target_config.get(&target) {
2103             if let Some(ref _s) = config.llvm_config {
2104                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2105                     target));
2106                 return None;
2107             }
2108         }
2109
2110         builder.info(&format!("Dist LlvmTools ({})", target));
2111         let src = builder.src.join("src/llvm-project/llvm");
2112         let name = pkgname(builder, "llvm-tools");
2113
2114         let tmp = tmpdir(builder);
2115         let image = tmp.join("llvm-tools-image");
2116         drop(fs::remove_dir_all(&image));
2117
2118         // Prepare the image directory
2119         let src_bindir = builder
2120             .llvm_out(target)
2121             .join("bin");
2122         let dst_bindir = image.join("lib/rustlib")
2123             .join(&*target)
2124             .join("bin");
2125         t!(fs::create_dir_all(&dst_bindir));
2126         for tool in LLVM_TOOLS {
2127             let exe = src_bindir.join(exe(tool, &target));
2128             builder.install(&exe, &dst_bindir, 0o755);
2129         }
2130
2131         // Prepare the overlay
2132         let overlay = tmp.join("llvm-tools-overlay");
2133         drop(fs::remove_dir_all(&overlay));
2134         builder.create_dir(&overlay);
2135         builder.install(&src.join("README.txt"), &overlay, 0o644);
2136         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2137         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2138
2139         // Generate the installer tarball
2140         let mut cmd = rust_installer(builder);
2141         cmd.arg("generate")
2142             .arg("--product-name=Rust")
2143             .arg("--rel-manifest-dir=rustlib")
2144             .arg("--success-message=llvm-tools-installed.")
2145             .arg("--image-dir").arg(&image)
2146             .arg("--work-dir").arg(&tmpdir(builder))
2147             .arg("--output-dir").arg(&distdir(builder))
2148             .arg("--non-installed-overlay").arg(&overlay)
2149             .arg(format!("--package-name={}-{}", name, target))
2150             .arg("--legacy-manifest-dirs=rustlib,cargo")
2151             .arg("--component-name=llvm-tools-preview");
2152
2153
2154         builder.run(&mut cmd);
2155         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2156     }
2157 }
2158
2159 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2160 pub struct Lldb {
2161     pub target: Interned<String>,
2162 }
2163
2164 impl Step for Lldb {
2165     type Output = Option<PathBuf>;
2166     const ONLY_HOSTS: bool = true;
2167     const DEFAULT: bool = true;
2168
2169     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2170         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2171     }
2172
2173     fn make_run(run: RunConfig<'_>) {
2174         run.builder.ensure(Lldb {
2175             target: run.target,
2176         });
2177     }
2178
2179     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2180         let target = self.target;
2181
2182         if builder.config.dry_run {
2183             return None;
2184         }
2185
2186         let bindir = builder
2187             .llvm_out(target)
2188             .join("bin");
2189         let lldb_exe = bindir.join(exe("lldb", &target));
2190         if !lldb_exe.exists() {
2191             return None;
2192         }
2193
2194         builder.info(&format!("Dist Lldb ({})", target));
2195         let src = builder.src.join("src/llvm-project/lldb");
2196         let name = pkgname(builder, "lldb");
2197
2198         let tmp = tmpdir(builder);
2199         let image = tmp.join("lldb-image");
2200         drop(fs::remove_dir_all(&image));
2201
2202         // Prepare the image directory
2203         let root = image.join("lib/rustlib").join(&*target);
2204         let dst = root.join("bin");
2205         t!(fs::create_dir_all(&dst));
2206         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2207             let exe = bindir.join(exe(program, &target));
2208             builder.install(&exe, &dst, 0o755);
2209         }
2210
2211         // The libraries.
2212         let libdir = builder.llvm_out(target).join("lib");
2213         let dst = root.join("lib");
2214         t!(fs::create_dir_all(&dst));
2215         for entry in t!(fs::read_dir(&libdir)) {
2216             let entry = entry.unwrap();
2217             if let Ok(name) = entry.file_name().into_string() {
2218                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2219                     if t!(entry.file_type()).is_symlink() {
2220                         builder.copy_to_folder(&entry.path(), &dst);
2221                     } else {
2222                        builder.install(&entry.path(), &dst, 0o755);
2223                     }
2224                 }
2225             }
2226         }
2227
2228         // The lldb scripts might be installed in lib/python$version
2229         // or in lib64/python$version.  If lib64 exists, use it;
2230         // otherwise lib.
2231         let libdir = builder.llvm_out(target).join("lib64");
2232         let (libdir, libdir_name) = if libdir.exists() {
2233             (libdir, "lib64")
2234         } else {
2235             (builder.llvm_out(target).join("lib"), "lib")
2236         };
2237         for entry in t!(fs::read_dir(&libdir)) {
2238             let entry = t!(entry);
2239             if let Ok(name) = entry.file_name().into_string() {
2240                 if name.starts_with("python") {
2241                     let dst = root.join(libdir_name)
2242                         .join(entry.file_name());
2243                     t!(fs::create_dir_all(&dst));
2244                     builder.cp_r(&entry.path(), &dst);
2245                     break;
2246                 }
2247             }
2248         }
2249
2250         // Prepare the overlay
2251         let overlay = tmp.join("lldb-overlay");
2252         drop(fs::remove_dir_all(&overlay));
2253         builder.create_dir(&overlay);
2254         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2255         builder.create(&overlay.join("version"), &builder.lldb_vers());
2256
2257         // Generate the installer tarball
2258         let mut cmd = rust_installer(builder);
2259         cmd.arg("generate")
2260             .arg("--product-name=Rust")
2261             .arg("--rel-manifest-dir=rustlib")
2262             .arg("--success-message=lldb-installed.")
2263             .arg("--image-dir").arg(&image)
2264             .arg("--work-dir").arg(&tmpdir(builder))
2265             .arg("--output-dir").arg(&distdir(builder))
2266             .arg("--non-installed-overlay").arg(&overlay)
2267             .arg(format!("--package-name={}-{}", name, target))
2268             .arg("--legacy-manifest-dirs=rustlib,cargo")
2269             .arg("--component-name=lldb-preview");
2270
2271
2272         builder.run(&mut cmd);
2273         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2274     }
2275 }