src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472
 473             // Copy rustc/rustdoc binaries
 474             t!(fs::create_dir_all(image.join("bin")));
 475             builder.cp_r(&src.join("bin"), &image.join("bin"));
 476
 477             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 478
 479             let libdir_relative = builder.libdir_relative(compiler);
 480
 481             // Copy runtime DLLs needed by the compiler
 482             if libdir_relative.to_str() != Some("bin") {
 483                 let libdir = builder.rustc_libdir(compiler);
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             // Don't use custom libdir here because ^lib/ will be resolved again
 489                             // with installer
 490                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap()
 499                 .strip_prefix(builder.sysroot_libdir_relative(compiler)).unwrap();
 500             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 501             let backends_dst = image.join("lib").join(&backends_rel);
 502
 503             t!(fs::create_dir_all(&backends_dst));
 504             builder.cp_r(&backends_src, &backends_dst);
 505
 506             // Copy libLLVM.so to the lib dir as well, if needed. While not
 507             // technically needed by rustc itself it's needed by lots of other
 508             // components like the llvm tools and LLD. LLD is included below and
 509             // tools/LLDB come later, so let's just throw it in the rustc
 510             // component for now.
 511             maybe_install_llvm_dylib(builder, host, image);
 512
 513             // Copy over lld if it's there
 514             if builder.config.lld_enabled {
 515                 let exe = exe("rust-lld", &compiler.host);
 516                 let src = builder.sysroot_libdir(compiler, host)
 517                     .parent()
 518                     .unwrap()
 519                     .join("bin")
 520                     .join(&exe);
 521                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 522                 let dst = image.join("lib/rustlib")
 523                     .join(&*host)
 524                     .join("bin")
 525                     .join(&exe);
 526                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 527                 builder.copy(&src, &dst);
 528             }
 529
 530             // Man pages
 531             t!(fs::create_dir_all(image.join("share/man/man1")));
 532             let man_src = builder.src.join("src/doc/man");
 533             let man_dst = image.join("share/man/man1");
 534
 535             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 536             let time = env::var("SOURCE_DATE_EPOCH")
 537                 .map(|timestamp| {
 538                     let epoch = timestamp.parse().map_err(|err| {
 539                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 540                     }).unwrap();
 541
 542                     time::at(Timespec::new(epoch, 0))
 543                 })
 544                 .unwrap_or_else(|_| time::now());
 545
 546             let month_year = t!(time::strftime("%B %Y", &time));
 547             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 548             // to hardlink, and we don't want to edit the source templates
 549             for file_entry in builder.read_dir(&man_src) {
 550                 let page_src = file_entry.path();
 551                 let page_dst = man_dst.join(file_entry.file_name());
 552                 t!(fs::copy(&page_src, &page_dst));
 553                 // template in month/year and version number
 554                 builder.replace_in_file(&page_dst,
 555                                 &[("<INSERT DATE HERE>", &month_year),
 556                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 557             }
 558
 559             // Debugger scripts
 560             builder.ensure(DebuggerScripts {
 561                 sysroot: INTERNER.intern_path(image.to_owned()),
 562                 host,
 563             });
 564
 565             // Misc license info
 566             let cp = |file: &str| {
 567                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 568             };
 569             cp("COPYRIGHT");
 570             cp("LICENSE-APACHE");
 571             cp("LICENSE-MIT");
 572             cp("README.md");
 573         }
 574     }
 575 }
 576
 577 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 578 pub struct DebuggerScripts {
 579     pub sysroot: Interned<PathBuf>,
 580     pub host: Interned<String>,
 581 }
 582
 583 impl Step for DebuggerScripts {
 584     type Output = ();
 585
 586     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 587         run.path("src/lldb_batchmode.py")
 588     }
 589
 590     fn make_run(run: RunConfig<'_>) {
 591         run.builder.ensure(DebuggerScripts {
 592             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 593             host: run.target,
 594         });
 595     }
 596
 597     /// Copies debugger scripts for `target` into the `sysroot` specified.
 598     fn run(self, builder: &Builder<'_>) {
 599         let host = self.host;
 600         let sysroot = self.sysroot;
 601         let dst = sysroot.join("lib/rustlib/etc");
 602         t!(fs::create_dir_all(&dst));
 603         let cp_debugger_script = |file: &str| {
 604             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 605         };
 606         if host.contains("windows-msvc") {
 607             // windbg debugger scripts
 608             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 609                 0o755);
 610
 611             cp_debugger_script("natvis/intrinsic.natvis");
 612             cp_debugger_script("natvis/liballoc.natvis");
 613             cp_debugger_script("natvis/libcore.natvis");
 614         } else {
 615             cp_debugger_script("debugger_pretty_printers_common.py");
 616
 617             // gdb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 619                     0o755);
 620             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 621                     0o755);
 622
 623             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 624             cp_debugger_script("gdb_rust_pretty_printing.py");
 625
 626             // lldb debugger scripts
 627             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 628                     0o755);
 629
 630             cp_debugger_script("lldb_rust_formatters.py");
 631         }
 632     }
 633 }
 634
 635 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 636 pub struct Std {
 637     pub compiler: Compiler,
 638     pub target: Interned<String>,
 639 }
 640
 641 impl Step for Std {
 642     type Output = PathBuf;
 643     const DEFAULT: bool = true;
 644
 645     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 646         run.path("src/libstd")
 647     }
 648
 649     fn make_run(run: RunConfig<'_>) {
 650         run.builder.ensure(Std {
 651             compiler: run.builder.compiler_for(
 652                 run.builder.top_stage,
 653                 run.builder.config.build,
 654                 run.target,
 655             ),
 656             target: run.target,
 657         });
 658     }
 659
 660     fn run(self, builder: &Builder<'_>) -> PathBuf {
 661         let compiler = self.compiler;
 662         let target = self.target;
 663
 664         let name = pkgname(builder, "rust-std");
 665         builder.info(&format!("Dist std stage{} ({} -> {})",
 666             compiler.stage, &compiler.host, target));
 667
 668         // The only true set of target libraries came from the build triple, so
 669         // let's reduce redundant work by only producing archives from that host.
 670         if compiler.host != builder.config.build {
 671             builder.info("\tskipping, not a build host");
 672             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 673         }
 674
 675         // We want to package up as many target libraries as possible
 676         // for the `rust-std` package, so if this is a host target we
 677         // depend on librustc and otherwise we just depend on libtest.
 678         if builder.hosts.iter().any(|t| t == target) {
 679             builder.ensure(compile::Rustc { compiler, target });
 680         } else {
 681             builder.ensure(compile::Std { compiler, target });
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             // Find the actual compiler (handling the full bootstrap option) which
 741             // produced the save-analysis data because that data isn't copied
 742             // through the sysroot uplifting.
 743             compiler: run.builder.compiler_for(
 744                 run.builder.top_stage,
 745                 run.builder.config.build,
 746                 run.target,
 747             ),
 748             target: run.target,
 749         });
 750     }
 751
 752     /// Creates a tarball of save-analysis metadata, if available.
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756         assert!(builder.config.extended);
 757         builder.info("Dist analysis");
 758         let name = pkgname(builder, "rust-analysis");
 759
 760         if &compiler.host != builder.config.build {
 761             builder.info("\tskipping, not a build host");
 762             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 763         }
 764
 765         builder.ensure(Std { compiler, target });
 766
 767         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 768
 769         let src = builder.stage_out(compiler, Mode::Std)
 770             .join(target).join(builder.cargo_dir()).join("deps");
 771
 772         let image_src = src.join("save-analysis");
 773         let dst = image.join("lib/rustlib").join(target).join("analysis");
 774         t!(fs::create_dir_all(&dst));
 775         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 776         builder.cp_r(&image_src, &dst);
 777
 778         let mut cmd = rust_installer(builder);
 779         cmd.arg("generate")
 780            .arg("--product-name=Rust")
 781            .arg("--rel-manifest-dir=rustlib")
 782            .arg("--success-message=save-analysis-saved.")
 783            .arg("--image-dir").arg(&image)
 784            .arg("--work-dir").arg(&tmpdir(builder))
 785            .arg("--output-dir").arg(&distdir(builder))
 786            .arg(format!("--package-name={}-{}", name, target))
 787            .arg(format!("--component-name=rust-analysis-{}", target))
 788            .arg("--legacy-manifest-dirs=rustlib,cargo");
 789         builder.run(&mut cmd);
 790         builder.remove_dir(&image);
 791         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 792     }
 793 }
 794
 795 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 796     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 797         let spath = match path.to_str() {
 798             Some(path) => path,
 799             None => return false,
 800         };
 801         if spath.ends_with("~") || spath.ends_with(".pyc") {
 802             return false
 803         }
 804
 805         const LLVM_PROJECTS: &[&str] = &[
 806             "llvm-project/clang", "llvm-project\\clang",
 807             "llvm-project/libunwind", "llvm-project\\libunwind",
 808             "llvm-project/lld", "llvm-project\\lld",
 809             "llvm-project/lldb", "llvm-project\\lldb",
 810             "llvm-project/llvm", "llvm-project\\llvm",
 811         ];
 812         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 813             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 814         {
 815             return false;
 816         }
 817
 818         const LLVM_TEST: &[&str] = &[
 819             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 820             "llvm-emscripten/test", "llvm-emscripten\\test",
 821         ];
 822         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 823             (spath.ends_with(".ll") ||
 824              spath.ends_with(".td") ||
 825              spath.ends_with(".s")) {
 826             return false
 827         }
 828         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 829             return false
 830         }
 831
 832         let full_path = Path::new(dir).join(path);
 833         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 834             return false;
 835         }
 836
 837         let excludes = [
 838             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 839             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 840             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 841             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 842         ];
 843         !path.iter()
 844              .map(|s| s.to_str().unwrap())
 845              .any(|s| excludes.contains(&s))
 846     }
 847
 848     // Copy the directories using our filter
 849     for item in src_dirs {
 850         let dst = &dst_dir.join(item);
 851         t!(fs::create_dir_all(dst));
 852         builder.cp_filtered(
 853             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 854     }
 855 }
 856
 857 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 858 pub struct Src;
 859
 860 impl Step for Src {
 861     /// The output path of the src installer tarball
 862     type Output = PathBuf;
 863     const DEFAULT: bool = true;
 864     const ONLY_HOSTS: bool = true;
 865
 866     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 867         run.path("src")
 868     }
 869
 870     fn make_run(run: RunConfig<'_>) {
 871         run.builder.ensure(Src);
 872     }
 873
 874     /// Creates the `rust-src` installer component
 875     fn run(self, builder: &Builder<'_>) -> PathBuf {
 876         builder.info("Dist src");
 877
 878         let name = pkgname(builder, "rust-src");
 879         let image = tmpdir(builder).join(format!("{}-image", name));
 880         let _ = fs::remove_dir_all(&image);
 881
 882         let dst = image.join("lib/rustlib/src");
 883         let dst_src = dst.join("rust");
 884         t!(fs::create_dir_all(&dst_src));
 885
 886         let src_files = [
 887             "Cargo.lock",
 888         ];
 889         // This is the reduced set of paths which will become the rust-src component
 890         // (essentially libstd and all of its path dependencies)
 891         let std_src_dirs = [
 892             "src/build_helper",
 893             "src/liballoc",
 894             "src/libcore",
 895             "src/libpanic_abort",
 896             "src/libpanic_unwind",
 897             "src/librustc_asan",
 898             "src/librustc_lsan",
 899             "src/librustc_msan",
 900             "src/librustc_tsan",
 901             "src/libstd",
 902             "src/libunwind",
 903             "src/libtest",
 904             "src/libterm",
 905             "src/libprofiler_builtins",
 906             "src/stdarch",
 907             "src/libproc_macro",
 908             "src/tools/rustc-std-workspace-core",
 909             "src/tools/rustc-std-workspace-alloc",
 910             "src/tools/rustc-std-workspace-std",
 911             "src/librustc",
 912             "src/libsyntax",
 913         ];
 914
 915         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 916         for file in src_files.iter() {
 917             builder.copy(&builder.src.join(file), &dst_src.join(file));
 918         }
 919
 920         // Create source tarball in rust-installer format
 921         let mut cmd = rust_installer(builder);
 922         cmd.arg("generate")
 923            .arg("--product-name=Rust")
 924            .arg("--rel-manifest-dir=rustlib")
 925            .arg("--success-message=Awesome-Source.")
 926            .arg("--image-dir").arg(&image)
 927            .arg("--work-dir").arg(&tmpdir(builder))
 928            .arg("--output-dir").arg(&distdir(builder))
 929            .arg(format!("--package-name={}", name))
 930            .arg("--component-name=rust-src")
 931            .arg("--legacy-manifest-dirs=rustlib,cargo");
 932         builder.run(&mut cmd);
 933
 934         builder.remove_dir(&image);
 935         distdir(builder).join(&format!("{}.tar.gz", name))
 936     }
 937 }
 938
 939 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 940 pub struct PlainSourceTarball;
 941
 942 impl Step for PlainSourceTarball {
 943     /// Produces the location of the tarball generated
 944     type Output = PathBuf;
 945     const DEFAULT: bool = true;
 946     const ONLY_HOSTS: bool = true;
 947
 948     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 949         let builder = run.builder;
 950         run.path("src").default_condition(builder.config.rust_dist_src)
 951     }
 952
 953     fn make_run(run: RunConfig<'_>) {
 954         run.builder.ensure(PlainSourceTarball);
 955     }
 956
 957     /// Creates the plain source tarball
 958     fn run(self, builder: &Builder<'_>) -> PathBuf {
 959         builder.info("Create plain source tarball");
 960
 961         // Make sure that the root folder of tarball has the correct name
 962         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 963         let plain_dst_src = tmpdir(builder).join(&plain_name);
 964         let _ = fs::remove_dir_all(&plain_dst_src);
 965         t!(fs::create_dir_all(&plain_dst_src));
 966
 967         // This is the set of root paths which will become part of the source package
 968         let src_files = [
 969             "COPYRIGHT",
 970             "LICENSE-APACHE",
 971             "LICENSE-MIT",
 972             "CONTRIBUTING.md",
 973             "README.md",
 974             "RELEASES.md",
 975             "configure",
 976             "x.py",
 977             "config.toml.example",
 978             "Cargo.toml",
 979             "Cargo.lock",
 980         ];
 981         let src_dirs = [
 982             "src",
 983         ];
 984
 985         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 986
 987         // Copy the files normally
 988         for item in &src_files {
 989             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 990         }
 991
 992         // Create the version file
 993         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 994         if let Some(sha) = builder.rust_sha() {
 995             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 996         }
 997
 998         // If we're building from git sources, we need to vendor a complete distribution.
 999         if builder.rust_info.is_git() {
1000             // Vendor all Cargo dependencies
1001             let mut cmd = Command::new(&builder.initial_cargo);
1002             cmd.arg("vendor")
1003                .current_dir(&plain_dst_src);
1004             builder.run(&mut cmd);
1005         }
1006
1007         // Create plain source tarball
1008         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1009         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1010         tarball.set_extension(""); // strip .gz
1011         tarball.set_extension(""); // strip .tar
1012         if let Some(dir) = tarball.parent() {
1013             builder.create_dir(&dir);
1014         }
1015         builder.info("running installer");
1016         let mut cmd = rust_installer(builder);
1017         cmd.arg("tarball")
1018            .arg("--input").arg(&plain_name)
1019            .arg("--output").arg(&tarball)
1020            .arg("--work-dir=.")
1021            .current_dir(tmpdir(builder));
1022         builder.run(&mut cmd);
1023         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1024     }
1025 }
1026
1027 // We have to run a few shell scripts, which choke quite a bit on both `\`
1028 // characters and on `C:\` paths, so normalize both of them away.
1029 pub fn sanitize_sh(path: &Path) -> String {
1030     let path = path.to_str().unwrap().replace("\\", "/");
1031     return change_drive(&path).unwrap_or(path);
1032
1033     fn change_drive(s: &str) -> Option<String> {
1034         let mut ch = s.chars();
1035         let drive = ch.next().unwrap_or('C');
1036         if ch.next() != Some(':') {
1037             return None
1038         }
1039         if ch.next() != Some('/') {
1040             return None
1041         }
1042         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1043     }
1044 }
1045
1046 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1047 pub struct Cargo {
1048     pub compiler: Compiler,
1049     pub target: Interned<String>,
1050 }
1051
1052 impl Step for Cargo {
1053     type Output = PathBuf;
1054     const ONLY_HOSTS: bool = true;
1055
1056     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1057         run.path("cargo")
1058     }
1059
1060     fn make_run(run: RunConfig<'_>) {
1061         run.builder.ensure(Cargo {
1062             compiler: run.builder.compiler_for(
1063                 run.builder.top_stage,
1064                 run.builder.config.build,
1065                 run.target,
1066             ),
1067             target: run.target,
1068         });
1069     }
1070
1071     fn run(self, builder: &Builder<'_>) -> PathBuf {
1072         let compiler = self.compiler;
1073         let target = self.target;
1074
1075         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1076         let src = builder.src.join("src/tools/cargo");
1077         let etc = src.join("src/etc");
1078         let release_num = builder.release_num("cargo");
1079         let name = pkgname(builder, "cargo");
1080         let version = builder.cargo_info.version(builder, &release_num);
1081
1082         let tmp = tmpdir(builder);
1083         let image = tmp.join("cargo-image");
1084         drop(fs::remove_dir_all(&image));
1085         builder.create_dir(&image);
1086
1087         // Prepare the image directory
1088         builder.create_dir(&image.join("share/zsh/site-functions"));
1089         builder.create_dir(&image.join("etc/bash_completion.d"));
1090         let cargo = builder.ensure(tool::Cargo { compiler, target });
1091         builder.install(&cargo, &image.join("bin"), 0o755);
1092         for man in t!(etc.join("man").read_dir()) {
1093             let man = t!(man);
1094             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1095         }
1096         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1097         builder.copy(&etc.join("cargo.bashcomp.sh"),
1098              &image.join("etc/bash_completion.d/cargo"));
1099         let doc = image.join("share/doc/cargo");
1100         builder.install(&src.join("README.md"), &doc, 0o644);
1101         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1102         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1103         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1104
1105         // Prepare the overlay
1106         let overlay = tmp.join("cargo-overlay");
1107         drop(fs::remove_dir_all(&overlay));
1108         builder.create_dir(&overlay);
1109         builder.install(&src.join("README.md"), &overlay, 0o644);
1110         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1111         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1112         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1113         builder.create(&overlay.join("version"), &version);
1114
1115         // Generate the installer tarball
1116         let mut cmd = rust_installer(builder);
1117         cmd.arg("generate")
1118            .arg("--product-name=Rust")
1119            .arg("--rel-manifest-dir=rustlib")
1120            .arg("--success-message=Rust-is-ready-to-roll.")
1121            .arg("--image-dir").arg(&image)
1122            .arg("--work-dir").arg(&tmpdir(builder))
1123            .arg("--output-dir").arg(&distdir(builder))
1124            .arg("--non-installed-overlay").arg(&overlay)
1125            .arg(format!("--package-name={}-{}", name, target))
1126            .arg("--component-name=cargo")
1127            .arg("--legacy-manifest-dirs=rustlib,cargo");
1128         builder.run(&mut cmd);
1129         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1130     }
1131 }
1132
1133 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1134 pub struct Rls {
1135     pub compiler: Compiler,
1136     pub target: Interned<String>,
1137 }
1138
1139 impl Step for Rls {
1140     type Output = Option<PathBuf>;
1141     const ONLY_HOSTS: bool = true;
1142
1143     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1144         run.path("rls")
1145     }
1146
1147     fn make_run(run: RunConfig<'_>) {
1148         run.builder.ensure(Rls {
1149             compiler: run.builder.compiler_for(
1150                 run.builder.top_stage,
1151                 run.builder.config.build,
1152                 run.target,
1153             ),
1154             target: run.target,
1155         });
1156     }
1157
1158     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1159         let compiler = self.compiler;
1160         let target = self.target;
1161         assert!(builder.config.extended);
1162
1163         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1164         let src = builder.src.join("src/tools/rls");
1165         let release_num = builder.release_num("rls");
1166         let name = pkgname(builder, "rls");
1167         let version = builder.rls_info.version(builder, &release_num);
1168
1169         let tmp = tmpdir(builder);
1170         let image = tmp.join("rls-image");
1171         drop(fs::remove_dir_all(&image));
1172         t!(fs::create_dir_all(&image));
1173
1174         // Prepare the image directory
1175         // We expect RLS to build, because we've exited this step above if tool
1176         // state for RLS isn't testing.
1177         let rls = builder.ensure(tool::Rls {
1178             compiler,
1179             target,
1180             extra_features: Vec::new(),
1181         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1182
1183         builder.install(&rls, &image.join("bin"), 0o755);
1184         let doc = image.join("share/doc/rls");
1185         builder.install(&src.join("README.md"), &doc, 0o644);
1186         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1187         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1188
1189         // Prepare the overlay
1190         let overlay = tmp.join("rls-overlay");
1191         drop(fs::remove_dir_all(&overlay));
1192         t!(fs::create_dir_all(&overlay));
1193         builder.install(&src.join("README.md"), &overlay, 0o644);
1194         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1195         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1196         builder.create(&overlay.join("version"), &version);
1197
1198         // Generate the installer tarball
1199         let mut cmd = rust_installer(builder);
1200         cmd.arg("generate")
1201            .arg("--product-name=Rust")
1202            .arg("--rel-manifest-dir=rustlib")
1203            .arg("--success-message=RLS-ready-to-serve.")
1204            .arg("--image-dir").arg(&image)
1205            .arg("--work-dir").arg(&tmpdir(builder))
1206            .arg("--output-dir").arg(&distdir(builder))
1207            .arg("--non-installed-overlay").arg(&overlay)
1208            .arg(format!("--package-name={}-{}", name, target))
1209            .arg("--legacy-manifest-dirs=rustlib,cargo")
1210            .arg("--component-name=rls-preview");
1211
1212         builder.run(&mut cmd);
1213         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1214     }
1215 }
1216
1217 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1218 pub struct Clippy {
1219     pub compiler: Compiler,
1220     pub target: Interned<String>,
1221 }
1222
1223 impl Step for Clippy {
1224     type Output = Option<PathBuf>;
1225     const ONLY_HOSTS: bool = true;
1226
1227     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1228         run.path("clippy")
1229     }
1230
1231     fn make_run(run: RunConfig<'_>) {
1232         run.builder.ensure(Clippy {
1233             compiler: run.builder.compiler_for(
1234                 run.builder.top_stage,
1235                 run.builder.config.build,
1236                 run.target,
1237             ),
1238             target: run.target,
1239         });
1240     }
1241
1242     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1243         let compiler = self.compiler;
1244         let target = self.target;
1245         assert!(builder.config.extended);
1246
1247         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1248         let src = builder.src.join("src/tools/clippy");
1249         let release_num = builder.release_num("clippy");
1250         let name = pkgname(builder, "clippy");
1251         let version = builder.clippy_info.version(builder, &release_num);
1252
1253         let tmp = tmpdir(builder);
1254         let image = tmp.join("clippy-image");
1255         drop(fs::remove_dir_all(&image));
1256         builder.create_dir(&image);
1257
1258         // Prepare the image directory
1259         // We expect clippy to build, because we've exited this step above if tool
1260         // state for clippy isn't testing.
1261         let clippy = builder.ensure(tool::Clippy {
1262             compiler,
1263             target,
1264             extra_features: Vec::new(),
1265         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1266         let cargoclippy = builder.ensure(tool::CargoClippy {
1267             compiler,
1268             target, extra_features: Vec::new()
1269         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1270
1271         builder.install(&clippy, &image.join("bin"), 0o755);
1272         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1273         let doc = image.join("share/doc/clippy");
1274         builder.install(&src.join("README.md"), &doc, 0o644);
1275         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1276         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1277
1278         // Prepare the overlay
1279         let overlay = tmp.join("clippy-overlay");
1280         drop(fs::remove_dir_all(&overlay));
1281         t!(fs::create_dir_all(&overlay));
1282         builder.install(&src.join("README.md"), &overlay, 0o644);
1283         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1284         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1285         builder.create(&overlay.join("version"), &version);
1286
1287         // Generate the installer tarball
1288         let mut cmd = rust_installer(builder);
1289         cmd.arg("generate")
1290            .arg("--product-name=Rust")
1291            .arg("--rel-manifest-dir=rustlib")
1292            .arg("--success-message=clippy-ready-to-serve.")
1293            .arg("--image-dir").arg(&image)
1294            .arg("--work-dir").arg(&tmpdir(builder))
1295            .arg("--output-dir").arg(&distdir(builder))
1296            .arg("--non-installed-overlay").arg(&overlay)
1297            .arg(format!("--package-name={}-{}", name, target))
1298            .arg("--legacy-manifest-dirs=rustlib,cargo")
1299            .arg("--component-name=clippy-preview");
1300
1301         builder.run(&mut cmd);
1302         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1303     }
1304 }
1305
1306 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1307 pub struct Miri {
1308     pub compiler: Compiler,
1309     pub target: Interned<String>,
1310 }
1311
1312 impl Step for Miri {
1313     type Output = Option<PathBuf>;
1314     const ONLY_HOSTS: bool = true;
1315
1316     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1317         run.path("miri")
1318     }
1319
1320     fn make_run(run: RunConfig<'_>) {
1321         run.builder.ensure(Miri {
1322             compiler: run.builder.compiler_for(
1323                 run.builder.top_stage,
1324                 run.builder.config.build,
1325                 run.target,
1326             ),
1327             target: run.target,
1328         });
1329     }
1330
1331     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1332         let compiler = self.compiler;
1333         let target = self.target;
1334         assert!(builder.config.extended);
1335
1336         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1337         let src = builder.src.join("src/tools/miri");
1338         let release_num = builder.release_num("miri");
1339         let name = pkgname(builder, "miri");
1340         let version = builder.miri_info.version(builder, &release_num);
1341
1342         let tmp = tmpdir(builder);
1343         let image = tmp.join("miri-image");
1344         drop(fs::remove_dir_all(&image));
1345         builder.create_dir(&image);
1346
1347         // Prepare the image directory
1348         // We expect miri to build, because we've exited this step above if tool
1349         // state for miri isn't testing.
1350         let miri = builder.ensure(tool::Miri {
1351             compiler,
1352             target,
1353             extra_features: Vec::new(),
1354         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1355         let cargomiri = builder.ensure(tool::CargoMiri {
1356             compiler,
1357             target,
1358             extra_features: Vec::new()
1359         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1360
1361         builder.install(&miri, &image.join("bin"), 0o755);
1362         builder.install(&cargomiri, &image.join("bin"), 0o755);
1363         let doc = image.join("share/doc/miri");
1364         builder.install(&src.join("README.md"), &doc, 0o644);
1365         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1366         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1367
1368         // Prepare the overlay
1369         let overlay = tmp.join("miri-overlay");
1370         drop(fs::remove_dir_all(&overlay));
1371         t!(fs::create_dir_all(&overlay));
1372         builder.install(&src.join("README.md"), &overlay, 0o644);
1373         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1374         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1375         builder.create(&overlay.join("version"), &version);
1376
1377         // Generate the installer tarball
1378         let mut cmd = rust_installer(builder);
1379         cmd.arg("generate")
1380            .arg("--product-name=Rust")
1381            .arg("--rel-manifest-dir=rustlib")
1382            .arg("--success-message=miri-ready-to-serve.")
1383            .arg("--image-dir").arg(&image)
1384            .arg("--work-dir").arg(&tmpdir(builder))
1385            .arg("--output-dir").arg(&distdir(builder))
1386            .arg("--non-installed-overlay").arg(&overlay)
1387            .arg(format!("--package-name={}-{}", name, target))
1388            .arg("--legacy-manifest-dirs=rustlib,cargo")
1389            .arg("--component-name=miri-preview");
1390
1391         builder.run(&mut cmd);
1392         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1393     }
1394 }
1395
1396 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1397 pub struct Rustfmt {
1398     pub compiler: Compiler,
1399     pub target: Interned<String>,
1400 }
1401
1402 impl Step for Rustfmt {
1403     type Output = Option<PathBuf>;
1404     const ONLY_HOSTS: bool = true;
1405
1406     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1407         run.path("rustfmt")
1408     }
1409
1410     fn make_run(run: RunConfig<'_>) {
1411         run.builder.ensure(Rustfmt {
1412             compiler: run.builder.compiler_for(
1413                 run.builder.top_stage,
1414                 run.builder.config.build,
1415                 run.target,
1416             ),
1417             target: run.target,
1418         });
1419     }
1420
1421     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1422         let compiler = self.compiler;
1423         let target = self.target;
1424
1425         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1426         let src = builder.src.join("src/tools/rustfmt");
1427         let release_num = builder.release_num("rustfmt");
1428         let name = pkgname(builder, "rustfmt");
1429         let version = builder.rustfmt_info.version(builder, &release_num);
1430
1431         let tmp = tmpdir(builder);
1432         let image = tmp.join("rustfmt-image");
1433         drop(fs::remove_dir_all(&image));
1434         builder.create_dir(&image);
1435
1436         // Prepare the image directory
1437         let rustfmt = builder.ensure(tool::Rustfmt {
1438             compiler,
1439             target,
1440             extra_features: Vec::new(),
1441         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1442         let cargofmt = builder.ensure(tool::Cargofmt {
1443             compiler,
1444             target,
1445             extra_features: Vec::new(),
1446         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1447
1448         builder.install(&rustfmt, &image.join("bin"), 0o755);
1449         builder.install(&cargofmt, &image.join("bin"), 0o755);
1450         let doc = image.join("share/doc/rustfmt");
1451         builder.install(&src.join("README.md"), &doc, 0o644);
1452         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1453         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1454
1455         // Prepare the overlay
1456         let overlay = tmp.join("rustfmt-overlay");
1457         drop(fs::remove_dir_all(&overlay));
1458         builder.create_dir(&overlay);
1459         builder.install(&src.join("README.md"), &overlay, 0o644);
1460         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1461         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1462         builder.create(&overlay.join("version"), &version);
1463
1464         // Generate the installer tarball
1465         let mut cmd = rust_installer(builder);
1466         cmd.arg("generate")
1467            .arg("--product-name=Rust")
1468            .arg("--rel-manifest-dir=rustlib")
1469            .arg("--success-message=rustfmt-ready-to-fmt.")
1470            .arg("--image-dir").arg(&image)
1471            .arg("--work-dir").arg(&tmpdir(builder))
1472            .arg("--output-dir").arg(&distdir(builder))
1473            .arg("--non-installed-overlay").arg(&overlay)
1474            .arg(format!("--package-name={}-{}", name, target))
1475            .arg("--legacy-manifest-dirs=rustlib,cargo")
1476            .arg("--component-name=rustfmt-preview");
1477
1478         builder.run(&mut cmd);
1479         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1480     }
1481 }
1482
1483 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1484 pub struct Extended {
1485     stage: u32,
1486     host: Interned<String>,
1487     target: Interned<String>,
1488 }
1489
1490 impl Step for Extended {
1491     type Output = ();
1492     const DEFAULT: bool = true;
1493     const ONLY_HOSTS: bool = true;
1494
1495     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1496         let builder = run.builder;
1497         run.path("extended").default_condition(builder.config.extended)
1498     }
1499
1500     fn make_run(run: RunConfig<'_>) {
1501         run.builder.ensure(Extended {
1502             stage: run.builder.top_stage,
1503             host: run.builder.config.build,
1504             target: run.target,
1505         });
1506     }
1507
1508     /// Creates a combined installer for the specified target in the provided stage.
1509     fn run(self, builder: &Builder<'_>) {
1510         let target = self.target;
1511         let stage = self.stage;
1512         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1513
1514         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1515
1516         let rustc_installer = builder.ensure(Rustc {
1517             compiler: builder.compiler(stage, target),
1518         });
1519         let cargo_installer = builder.ensure(Cargo { compiler, target });
1520         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1521         let rls_installer = builder.ensure(Rls { compiler, target });
1522         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1523         let clippy_installer = builder.ensure(Clippy { compiler, target });
1524         let miri_installer = builder.ensure(Miri { compiler, target });
1525         let lldb_installer = builder.ensure(Lldb { target });
1526         let mingw_installer = builder.ensure(Mingw { host: target });
1527         let analysis_installer = builder.ensure(Analysis { compiler, target });
1528
1529         let docs_installer = builder.ensure(Docs { host: target, });
1530         let std_installer = builder.ensure(Std {
1531             compiler: builder.compiler(stage, target),
1532             target,
1533         });
1534
1535         let tmp = tmpdir(builder);
1536         let overlay = tmp.join("extended-overlay");
1537         let etc = builder.src.join("src/etc/installer");
1538         let work = tmp.join("work");
1539
1540         let _ = fs::remove_dir_all(&overlay);
1541         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1542         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1543         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1544         let version = builder.rust_version();
1545         builder.create(&overlay.join("version"), &version);
1546         if let Some(sha) = builder.rust_sha() {
1547             builder.create(&overlay.join("git-commit-hash"), &sha);
1548         }
1549         builder.install(&etc.join("README.md"), &overlay, 0o644);
1550
1551         // When rust-std package split from rustc, we needed to ensure that during
1552         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1553         // the std files during uninstall. To do this ensure that rustc comes
1554         // before rust-std in the list below.
1555         let mut tarballs = Vec::new();
1556         tarballs.push(rustc_installer);
1557         tarballs.push(cargo_installer);
1558         tarballs.extend(rls_installer.clone());
1559         tarballs.extend(clippy_installer.clone());
1560         tarballs.extend(miri_installer.clone());
1561         tarballs.extend(rustfmt_installer.clone());
1562         tarballs.extend(llvm_tools_installer);
1563         tarballs.extend(lldb_installer);
1564         tarballs.push(analysis_installer);
1565         tarballs.push(std_installer);
1566         if builder.config.docs {
1567             tarballs.push(docs_installer);
1568         }
1569         if target.contains("pc-windows-gnu") {
1570             tarballs.push(mingw_installer.unwrap());
1571         }
1572         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1573         for tarball in &tarballs[1..] {
1574             input_tarballs.push(",");
1575             input_tarballs.push(tarball);
1576         }
1577
1578         let mut cmd = rust_installer(builder);
1579         cmd.arg("combine")
1580             .arg("--product-name=Rust")
1581             .arg("--rel-manifest-dir=rustlib")
1582             .arg("--success-message=Rust-is-ready-to-roll.")
1583             .arg("--work-dir").arg(&work)
1584             .arg("--output-dir").arg(&distdir(builder))
1585             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1586             .arg("--legacy-manifest-dirs=rustlib,cargo")
1587             .arg("--input-tarballs").arg(input_tarballs)
1588             .arg("--non-installed-overlay").arg(&overlay);
1589         builder.run(&mut cmd);
1590
1591         let mut license = String::new();
1592         license += &builder.read(&builder.src.join("COPYRIGHT"));
1593         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1594         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1595         license.push_str("\n");
1596         license.push_str("\n");
1597
1598         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1599         let mut rtf = rtf.to_string();
1600         rtf.push_str("\n");
1601         for line in license.lines() {
1602             rtf.push_str(line);
1603             rtf.push_str("\\line ");
1604         }
1605         rtf.push_str("}");
1606
1607         fn filter(contents: &str, marker: &str) -> String {
1608             let start = format!("tool-{}-start", marker);
1609             let end = format!("tool-{}-end", marker);
1610             let mut lines = Vec::new();
1611             let mut omitted = false;
1612             for line in contents.lines() {
1613                 if line.contains(&start) {
1614                     omitted = true;
1615                 } else if line.contains(&end) {
1616                     omitted = false;
1617                 } else if !omitted {
1618                     lines.push(line);
1619                 }
1620             }
1621
1622             lines.join("\n")
1623         }
1624
1625         let xform = |p: &Path| {
1626             let mut contents = t!(fs::read_to_string(p));
1627             if rls_installer.is_none() {
1628                 contents = filter(&contents, "rls");
1629             }
1630             if clippy_installer.is_none() {
1631                 contents = filter(&contents, "clippy");
1632             }
1633             if miri_installer.is_none() {
1634                 contents = filter(&contents, "miri");
1635             }
1636             if rustfmt_installer.is_none() {
1637                 contents = filter(&contents, "rustfmt");
1638             }
1639             let ret = tmp.join(p.file_name().unwrap());
1640             t!(fs::write(&ret, &contents));
1641             ret
1642         };
1643
1644         if target.contains("apple-darwin") {
1645             let pkg = tmp.join("pkg");
1646             let _ = fs::remove_dir_all(&pkg);
1647
1648             let pkgbuild = |component: &str| {
1649                 let mut cmd = Command::new("pkgbuild");
1650                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1651                     .arg("--scripts").arg(pkg.join(component))
1652                     .arg("--nopayload")
1653                     .arg(pkg.join(component).with_extension("pkg"));
1654                 builder.run(&mut cmd);
1655             };
1656
1657             let prepare = |name: &str| {
1658                 builder.create_dir(&pkg.join(name));
1659                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1660                         &pkg.join(name));
1661                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1662                 pkgbuild(name);
1663             };
1664             prepare("rustc");
1665             prepare("cargo");
1666             prepare("rust-docs");
1667             prepare("rust-std");
1668             prepare("rust-analysis");
1669
1670             if rls_installer.is_some() {
1671                 prepare("rls");
1672             }
1673             if clippy_installer.is_some() {
1674                 prepare("clippy");
1675             }
1676             if miri_installer.is_some() {
1677                 prepare("miri");
1678             }
1679
1680             // create an 'uninstall' package
1681             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1682             pkgbuild("uninstall");
1683
1684             builder.create_dir(&pkg.join("res"));
1685             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1686             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1687             let mut cmd = Command::new("productbuild");
1688             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1689                 .arg("--resources").arg(pkg.join("res"))
1690                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1691                                                     pkgname(builder, "rust"),
1692                                                     target)))
1693                 .arg("--package-path").arg(&pkg);
1694             builder.run(&mut cmd);
1695         }
1696
1697         if target.contains("windows") {
1698             let exe = tmp.join("exe");
1699             let _ = fs::remove_dir_all(&exe);
1700
1701             let prepare = |name: &str| {
1702                 builder.create_dir(&exe.join(name));
1703                 let dir = if name == "rust-std" || name == "rust-analysis" {
1704                     format!("{}-{}", name, target)
1705                 } else if name == "rls" {
1706                     "rls-preview".to_string()
1707                 } else if name == "clippy" {
1708                     "clippy-preview".to_string()
1709                 } else if name == "miri" {
1710                     "miri-preview".to_string()
1711                 } else {
1712                     name.to_string()
1713                 };
1714                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1715                             .join(dir),
1716                         &exe.join(name));
1717                 builder.remove(&exe.join(name).join("manifest.in"));
1718             };
1719             prepare("rustc");
1720             prepare("cargo");
1721             prepare("rust-analysis");
1722             prepare("rust-docs");
1723             prepare("rust-std");
1724             if rls_installer.is_some() {
1725                 prepare("rls");
1726             }
1727             if clippy_installer.is_some() {
1728                 prepare("clippy");
1729             }
1730             if miri_installer.is_some() {
1731                 prepare("miri");
1732             }
1733             if target.contains("windows-gnu") {
1734                 prepare("rust-mingw");
1735             }
1736
1737             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1738             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1739             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1740             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1741             builder.create(&exe.join("LICENSE.txt"), &license);
1742
1743             // Generate exe installer
1744             let mut cmd = Command::new("iscc");
1745             cmd.arg("rust.iss")
1746                 .current_dir(&exe);
1747             if target.contains("windows-gnu") {
1748                 cmd.arg("/dMINGW");
1749             }
1750             add_env(builder, &mut cmd, target);
1751             builder.run(&mut cmd);
1752             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1753                     &distdir(builder),
1754                     0o755);
1755
1756             // Generate msi installer
1757             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1758             let heat = wix.join("bin/heat.exe");
1759             let candle = wix.join("bin/candle.exe");
1760             let light = wix.join("bin/light.exe");
1761
1762             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1763             builder.run(Command::new(&heat)
1764                             .current_dir(&exe)
1765                             .arg("dir")
1766                             .arg("rustc")
1767                             .args(&heat_flags)
1768                             .arg("-cg").arg("RustcGroup")
1769                             .arg("-dr").arg("Rustc")
1770                             .arg("-var").arg("var.RustcDir")
1771                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1772             builder.run(Command::new(&heat)
1773                             .current_dir(&exe)
1774                             .arg("dir")
1775                             .arg("rust-docs")
1776                             .args(&heat_flags)
1777                             .arg("-cg").arg("DocsGroup")
1778                             .arg("-dr").arg("Docs")
1779                             .arg("-var").arg("var.DocsDir")
1780                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1781                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1782             builder.run(Command::new(&heat)
1783                             .current_dir(&exe)
1784                             .arg("dir")
1785                             .arg("cargo")
1786                             .args(&heat_flags)
1787                             .arg("-cg").arg("CargoGroup")
1788                             .arg("-dr").arg("Cargo")
1789                             .arg("-var").arg("var.CargoDir")
1790                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1791                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1792             builder.run(Command::new(&heat)
1793                             .current_dir(&exe)
1794                             .arg("dir")
1795                             .arg("rust-std")
1796                             .args(&heat_flags)
1797                             .arg("-cg").arg("StdGroup")
1798                             .arg("-dr").arg("Std")
1799                             .arg("-var").arg("var.StdDir")
1800                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1801             if rls_installer.is_some() {
1802                 builder.run(Command::new(&heat)
1803                                 .current_dir(&exe)
1804                                 .arg("dir")
1805                                 .arg("rls")
1806                                 .args(&heat_flags)
1807                                 .arg("-cg").arg("RlsGroup")
1808                                 .arg("-dr").arg("Rls")
1809                                 .arg("-var").arg("var.RlsDir")
1810                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1811                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1812             }
1813             if clippy_installer.is_some() {
1814                 builder.run(Command::new(&heat)
1815                                 .current_dir(&exe)
1816                                 .arg("dir")
1817                                 .arg("clippy")
1818                                 .args(&heat_flags)
1819                                 .arg("-cg").arg("ClippyGroup")
1820                                 .arg("-dr").arg("Clippy")
1821                                 .arg("-var").arg("var.ClippyDir")
1822                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1823                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1824             }
1825             if miri_installer.is_some() {
1826                 builder.run(Command::new(&heat)
1827                                 .current_dir(&exe)
1828                                 .arg("dir")
1829                                 .arg("miri")
1830                                 .args(&heat_flags)
1831                                 .arg("-cg").arg("MiriGroup")
1832                                 .arg("-dr").arg("Miri")
1833                                 .arg("-var").arg("var.MiriDir")
1834                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1835                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1836             }
1837             builder.run(Command::new(&heat)
1838                             .current_dir(&exe)
1839                             .arg("dir")
1840                             .arg("rust-analysis")
1841                             .args(&heat_flags)
1842                             .arg("-cg").arg("AnalysisGroup")
1843                             .arg("-dr").arg("Analysis")
1844                             .arg("-var").arg("var.AnalysisDir")
1845                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1846                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1847             if target.contains("windows-gnu") {
1848                 builder.run(Command::new(&heat)
1849                                 .current_dir(&exe)
1850                                 .arg("dir")
1851                                 .arg("rust-mingw")
1852                                 .args(&heat_flags)
1853                                 .arg("-cg").arg("GccGroup")
1854                                 .arg("-dr").arg("Gcc")
1855                                 .arg("-var").arg("var.GccDir")
1856                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1857             }
1858
1859             let candle = |input: &Path| {
1860                 let output = exe.join(input.file_stem().unwrap())
1861                                 .with_extension("wixobj");
1862                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1863                 let mut cmd = Command::new(&candle);
1864                 cmd.current_dir(&exe)
1865                     .arg("-nologo")
1866                     .arg("-dRustcDir=rustc")
1867                     .arg("-dDocsDir=rust-docs")
1868                     .arg("-dCargoDir=cargo")
1869                     .arg("-dStdDir=rust-std")
1870                     .arg("-dAnalysisDir=rust-analysis")
1871                     .arg("-arch").arg(&arch)
1872                     .arg("-out").arg(&output)
1873                     .arg(&input);
1874                 add_env(builder, &mut cmd, target);
1875
1876                 if rls_installer.is_some() {
1877                     cmd.arg("-dRlsDir=rls");
1878                 }
1879                 if clippy_installer.is_some() {
1880                     cmd.arg("-dClippyDir=clippy");
1881                 }
1882                 if miri_installer.is_some() {
1883                     cmd.arg("-dMiriDir=miri");
1884                 }
1885                 if target.contains("windows-gnu") {
1886                     cmd.arg("-dGccDir=rust-mingw");
1887                 }
1888                 builder.run(&mut cmd);
1889             };
1890             candle(&xform(&etc.join("msi/rust.wxs")));
1891             candle(&etc.join("msi/ui.wxs"));
1892             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1893             candle("RustcGroup.wxs".as_ref());
1894             candle("DocsGroup.wxs".as_ref());
1895             candle("CargoGroup.wxs".as_ref());
1896             candle("StdGroup.wxs".as_ref());
1897             if rls_installer.is_some() {
1898                 candle("RlsGroup.wxs".as_ref());
1899             }
1900             if clippy_installer.is_some() {
1901                 candle("ClippyGroup.wxs".as_ref());
1902             }
1903             if miri_installer.is_some() {
1904                 candle("MiriGroup.wxs".as_ref());
1905             }
1906             candle("AnalysisGroup.wxs".as_ref());
1907
1908             if target.contains("windows-gnu") {
1909                 candle("GccGroup.wxs".as_ref());
1910             }
1911
1912             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1913             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1914             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1915
1916             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1917             let mut cmd = Command::new(&light);
1918             cmd.arg("-nologo")
1919                 .arg("-ext").arg("WixUIExtension")
1920                 .arg("-ext").arg("WixUtilExtension")
1921                 .arg("-out").arg(exe.join(&filename))
1922                 .arg("rust.wixobj")
1923                 .arg("ui.wixobj")
1924                 .arg("rustwelcomedlg.wixobj")
1925                 .arg("RustcGroup.wixobj")
1926                 .arg("DocsGroup.wixobj")
1927                 .arg("CargoGroup.wixobj")
1928                 .arg("StdGroup.wixobj")
1929                 .arg("AnalysisGroup.wixobj")
1930                 .current_dir(&exe);
1931
1932             if rls_installer.is_some() {
1933                 cmd.arg("RlsGroup.wixobj");
1934             }
1935             if clippy_installer.is_some() {
1936                 cmd.arg("ClippyGroup.wixobj");
1937             }
1938             if miri_installer.is_some() {
1939                 cmd.arg("MiriGroup.wixobj");
1940             }
1941
1942             if target.contains("windows-gnu") {
1943                 cmd.arg("GccGroup.wixobj");
1944             }
1945             // ICE57 wrongly complains about the shortcuts
1946             cmd.arg("-sice:ICE57");
1947
1948             builder.run(&mut cmd);
1949
1950             if !builder.config.dry_run {
1951                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1952             }
1953         }
1954     }
1955 }
1956
1957 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1958     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1959     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1960        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1961        .env("CFG_RELEASE", builder.rust_release())
1962        .env("CFG_VER_MAJOR", parts.next().unwrap())
1963        .env("CFG_VER_MINOR", parts.next().unwrap())
1964        .env("CFG_VER_PATCH", parts.next().unwrap())
1965        .env("CFG_VER_BUILD", "0") // just needed to build
1966        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1967        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1968        .env("CFG_BUILD", target)
1969        .env("CFG_CHANNEL", &builder.config.channel);
1970
1971     if target.contains("windows-gnu") {
1972        cmd.env("CFG_MINGW", "1")
1973           .env("CFG_ABI", "GNU");
1974     } else {
1975        cmd.env("CFG_MINGW", "0")
1976           .env("CFG_ABI", "MSVC");
1977     }
1978
1979     if target.contains("x86_64") {
1980        cmd.env("CFG_PLATFORM", "x64");
1981     } else {
1982        cmd.env("CFG_PLATFORM", "x86");
1983     }
1984 }
1985
1986 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1987 pub struct HashSign;
1988
1989 impl Step for HashSign {
1990     type Output = ();
1991     const ONLY_HOSTS: bool = true;
1992
1993     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1994         run.path("hash-and-sign")
1995     }
1996
1997     fn make_run(run: RunConfig<'_>) {
1998         run.builder.ensure(HashSign);
1999     }
2000
2001     fn run(self, builder: &Builder<'_>) {
2002         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2003         if builder.config.dry_run {
2004             return;
2005         }
2006         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2007             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2008         });
2009         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2010             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2011         });
2012         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2013             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2014         });
2015         let pass = t!(fs::read_to_string(&file));
2016
2017         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2018
2019         cmd.arg(sign);
2020         cmd.arg(distdir(builder));
2021         cmd.arg(today.trim());
2022         cmd.arg(builder.rust_package_vers());
2023         cmd.arg(addr);
2024         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2025         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2026         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2027         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2028         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2029         cmd.arg(builder.llvm_tools_package_vers());
2030         cmd.arg(builder.lldb_package_vers());
2031
2032         builder.create_dir(&distdir(builder));
2033
2034         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2035         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2036         let status = t!(child.wait());
2037         assert!(status.success());
2038     }
2039 }
2040
2041 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2042 // LLVM tools are linked dynamically.
2043 // Note: This function does no yet support Windows but we also don't support
2044 //       linking LLVM tools dynamically on Windows yet.
2045 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2046                                 target: Interned<String>,
2047                                 sysroot: &Path) {
2048     let src_libdir = builder
2049         .llvm_out(target)
2050         .join("lib");
2051     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2052     t!(fs::create_dir_all(&dst_libdir));
2053
2054     if target.contains("apple-darwin") {
2055         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2056         if llvm_dylib_path.exists() {
2057             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2058         }
2059         return
2060     }
2061
2062     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2063     // Since tools link to the latter rather than the former, we have to
2064     // follow the symlink to find out what to distribute.
2065     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2066     if llvm_dylib_path.exists() {
2067         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2068             panic!("dist: Error calling canonicalize path `{}`: {}",
2069                    llvm_dylib_path.display(), e);
2070         });
2071
2072
2073         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2074     }
2075 }
2076
2077 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2078 pub struct LlvmTools {
2079     pub target: Interned<String>,
2080 }
2081
2082 impl Step for LlvmTools {
2083     type Output = Option<PathBuf>;
2084     const ONLY_HOSTS: bool = true;
2085
2086     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2087         run.path("llvm-tools")
2088     }
2089
2090     fn make_run(run: RunConfig<'_>) {
2091         run.builder.ensure(LlvmTools {
2092             target: run.target,
2093         });
2094     }
2095
2096     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2097         let target = self.target;
2098         assert!(builder.config.extended);
2099
2100         /* run only if llvm-config isn't used */
2101         if let Some(config) = builder.config.target_config.get(&target) {
2102             if let Some(ref _s) = config.llvm_config {
2103                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2104                     target));
2105                 return None;
2106             }
2107         }
2108
2109         builder.info(&format!("Dist LlvmTools ({})", target));
2110         let src = builder.src.join("src/llvm-project/llvm");
2111         let name = pkgname(builder, "llvm-tools");
2112
2113         let tmp = tmpdir(builder);
2114         let image = tmp.join("llvm-tools-image");
2115         drop(fs::remove_dir_all(&image));
2116
2117         // Prepare the image directory
2118         let src_bindir = builder
2119             .llvm_out(target)
2120             .join("bin");
2121         let dst_bindir = image.join("lib/rustlib")
2122             .join(&*target)
2123             .join("bin");
2124         t!(fs::create_dir_all(&dst_bindir));
2125         for tool in LLVM_TOOLS {
2126             let exe = src_bindir.join(exe(tool, &target));
2127             builder.install(&exe, &dst_bindir, 0o755);
2128         }
2129
2130         // Prepare the overlay
2131         let overlay = tmp.join("llvm-tools-overlay");
2132         drop(fs::remove_dir_all(&overlay));
2133         builder.create_dir(&overlay);
2134         builder.install(&src.join("README.txt"), &overlay, 0o644);
2135         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2136         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2137
2138         // Generate the installer tarball
2139         let mut cmd = rust_installer(builder);
2140         cmd.arg("generate")
2141             .arg("--product-name=Rust")
2142             .arg("--rel-manifest-dir=rustlib")
2143             .arg("--success-message=llvm-tools-installed.")
2144             .arg("--image-dir").arg(&image)
2145             .arg("--work-dir").arg(&tmpdir(builder))
2146             .arg("--output-dir").arg(&distdir(builder))
2147             .arg("--non-installed-overlay").arg(&overlay)
2148             .arg(format!("--package-name={}-{}", name, target))
2149             .arg("--legacy-manifest-dirs=rustlib,cargo")
2150             .arg("--component-name=llvm-tools-preview");
2151
2152
2153         builder.run(&mut cmd);
2154         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2155     }
2156 }
2157
2158 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2159 pub struct Lldb {
2160     pub target: Interned<String>,
2161 }
2162
2163 impl Step for Lldb {
2164     type Output = Option<PathBuf>;
2165     const ONLY_HOSTS: bool = true;
2166     const DEFAULT: bool = true;
2167
2168     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2169         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2170     }
2171
2172     fn make_run(run: RunConfig<'_>) {
2173         run.builder.ensure(Lldb {
2174             target: run.target,
2175         });
2176     }
2177
2178     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2179         let target = self.target;
2180
2181         if builder.config.dry_run {
2182             return None;
2183         }
2184
2185         let bindir = builder
2186             .llvm_out(target)
2187             .join("bin");
2188         let lldb_exe = bindir.join(exe("lldb", &target));
2189         if !lldb_exe.exists() {
2190             return None;
2191         }
2192
2193         builder.info(&format!("Dist Lldb ({})", target));
2194         let src = builder.src.join("src/llvm-project/lldb");
2195         let name = pkgname(builder, "lldb");
2196
2197         let tmp = tmpdir(builder);
2198         let image = tmp.join("lldb-image");
2199         drop(fs::remove_dir_all(&image));
2200
2201         // Prepare the image directory
2202         let root = image.join("lib/rustlib").join(&*target);
2203         let dst = root.join("bin");
2204         t!(fs::create_dir_all(&dst));
2205         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2206             let exe = bindir.join(exe(program, &target));
2207             builder.install(&exe, &dst, 0o755);
2208         }
2209
2210         // The libraries.
2211         let libdir = builder.llvm_out(target).join("lib");
2212         let dst = root.join("lib");
2213         t!(fs::create_dir_all(&dst));
2214         for entry in t!(fs::read_dir(&libdir)) {
2215             let entry = entry.unwrap();
2216             if let Ok(name) = entry.file_name().into_string() {
2217                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2218                     if t!(entry.file_type()).is_symlink() {
2219                         builder.copy_to_folder(&entry.path(), &dst);
2220                     } else {
2221                        builder.install(&entry.path(), &dst, 0o755);
2222                     }
2223                 }
2224             }
2225         }
2226
2227         // The lldb scripts might be installed in lib/python$version
2228         // or in lib64/python$version.  If lib64 exists, use it;
2229         // otherwise lib.
2230         let libdir = builder.llvm_out(target).join("lib64");
2231         let (libdir, libdir_name) = if libdir.exists() {
2232             (libdir, "lib64")
2233         } else {
2234             (builder.llvm_out(target).join("lib"), "lib")
2235         };
2236         for entry in t!(fs::read_dir(&libdir)) {
2237             let entry = t!(entry);
2238             if let Ok(name) = entry.file_name().into_string() {
2239                 if name.starts_with("python") {
2240                     let dst = root.join(libdir_name)
2241                         .join(entry.file_name());
2242                     t!(fs::create_dir_all(&dst));
2243                     builder.cp_r(&entry.path(), &dst);
2244                     break;
2245                 }
2246             }
2247         }
2248
2249         // Prepare the overlay
2250         let overlay = tmp.join("lldb-overlay");
2251         drop(fs::remove_dir_all(&overlay));
2252         builder.create_dir(&overlay);
2253         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2254         builder.create(&overlay.join("version"), &builder.lldb_vers());
2255
2256         // Generate the installer tarball
2257         let mut cmd = rust_installer(builder);
2258         cmd.arg("generate")
2259             .arg("--product-name=Rust")
2260             .arg("--rel-manifest-dir=rustlib")
2261             .arg("--success-message=lldb-installed.")
2262             .arg("--image-dir").arg(&image)
2263             .arg("--work-dir").arg(&tmpdir(builder))
2264             .arg("--output-dir").arg(&distdir(builder))
2265             .arg("--non-installed-overlay").arg(&overlay)
2266             .arg(format!("--package-name={}-{}", name, target))
2267             .arg("--legacy-manifest-dirs=rustlib,cargo")
2268             .arg("--component-name=lldb-preview");
2269
2270
2271         builder.run(&mut cmd);
2272         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2273     }
2274 }