src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472             let libdir = builder.rustc_libdir(compiler);
 473
 474             // Copy rustc/rustdoc binaries
 475             t!(fs::create_dir_all(image.join("bin")));
 476             builder.cp_r(&src.join("bin"), &image.join("bin"));
 477
 478             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 479
 480             let libdir_relative = builder.libdir_relative(compiler);
 481
 482             // Copy runtime DLLs needed by the compiler
 483             if libdir_relative.to_str() != Some("bin") {
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             builder.install(&entry.path(), &image.join(&libdir_relative), 0o644);
 489                         }
 490                     }
 491                 }
 492             }
 493
 494             // Copy over the codegen backends
 495             let backends_src = builder.sysroot_codegen_backends(compiler);
 496             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 497             let backends_dst = image.join(&backends_rel);
 498             t!(fs::create_dir_all(&backends_dst));
 499             builder.cp_r(&backends_src, &backends_dst);
 500
 501             // Copy libLLVM.so to the lib dir as well, if needed. While not
 502             // technically needed by rustc itself it's needed by lots of other
 503             // components like the llvm tools and LLD. LLD is included below and
 504             // tools/LLDB come later, so let's just throw it in the rustc
 505             // component for now.
 506             maybe_install_llvm_dylib(builder, host, image);
 507
 508             // Copy over lld if it's there
 509             if builder.config.lld_enabled {
 510                 let exe = exe("rust-lld", &compiler.host);
 511                 let src = builder.sysroot_libdir(compiler, host)
 512                     .parent()
 513                     .unwrap()
 514                     .join("bin")
 515                     .join(&exe);
 516                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 517                 let dst = image.join("lib/rustlib")
 518                     .join(&*host)
 519                     .join("bin")
 520                     .join(&exe);
 521                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 522                 builder.copy(&src, &dst);
 523             }
 524
 525             // Man pages
 526             t!(fs::create_dir_all(image.join("share/man/man1")));
 527             let man_src = builder.src.join("src/doc/man");
 528             let man_dst = image.join("share/man/man1");
 529
 530             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 531             let time = env::var("SOURCE_DATE_EPOCH")
 532                 .map(|timestamp| {
 533                     let epoch = timestamp.parse().map_err(|err| {
 534                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 535                     }).unwrap();
 536
 537                     time::at(Timespec::new(epoch, 0))
 538                 })
 539                 .unwrap_or_else(|_| time::now());
 540
 541             let month_year = t!(time::strftime("%B %Y", &time));
 542             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 543             // to hardlink, and we don't want to edit the source templates
 544             for file_entry in builder.read_dir(&man_src) {
 545                 let page_src = file_entry.path();
 546                 let page_dst = man_dst.join(file_entry.file_name());
 547                 t!(fs::copy(&page_src, &page_dst));
 548                 // template in month/year and version number
 549                 builder.replace_in_file(&page_dst,
 550                                 &[("<INSERT DATE HERE>", &month_year),
 551                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 552             }
 553
 554             // Debugger scripts
 555             builder.ensure(DebuggerScripts {
 556                 sysroot: INTERNER.intern_path(image.to_owned()),
 557                 host,
 558             });
 559
 560             // Misc license info
 561             let cp = |file: &str| {
 562                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 563             };
 564             cp("COPYRIGHT");
 565             cp("LICENSE-APACHE");
 566             cp("LICENSE-MIT");
 567             cp("README.md");
 568         }
 569     }
 570 }
 571
 572 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 573 pub struct DebuggerScripts {
 574     pub sysroot: Interned<PathBuf>,
 575     pub host: Interned<String>,
 576 }
 577
 578 impl Step for DebuggerScripts {
 579     type Output = ();
 580
 581     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 582         run.path("src/lldb_batchmode.py")
 583     }
 584
 585     fn make_run(run: RunConfig<'_>) {
 586         run.builder.ensure(DebuggerScripts {
 587             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 588             host: run.target,
 589         });
 590     }
 591
 592     /// Copies debugger scripts for `target` into the `sysroot` specified.
 593     fn run(self, builder: &Builder<'_>) {
 594         let host = self.host;
 595         let sysroot = self.sysroot;
 596         let dst = sysroot.join("lib/rustlib/etc");
 597         t!(fs::create_dir_all(&dst));
 598         let cp_debugger_script = |file: &str| {
 599             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 600         };
 601         if host.contains("windows-msvc") {
 602             // windbg debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 604                 0o755);
 605
 606             cp_debugger_script("natvis/intrinsic.natvis");
 607             cp_debugger_script("natvis/liballoc.natvis");
 608             cp_debugger_script("natvis/libcore.natvis");
 609         } else {
 610             cp_debugger_script("debugger_pretty_printers_common.py");
 611
 612             // gdb debugger scripts
 613             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 614                     0o755);
 615             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 616                     0o755);
 617
 618             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 619             cp_debugger_script("gdb_rust_pretty_printing.py");
 620
 621             // lldb debugger scripts
 622             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 623                     0o755);
 624
 625             cp_debugger_script("lldb_rust_formatters.py");
 626         }
 627     }
 628 }
 629
 630 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 631 pub struct Std {
 632     pub compiler: Compiler,
 633     pub target: Interned<String>,
 634 }
 635
 636 impl Step for Std {
 637     type Output = PathBuf;
 638     const DEFAULT: bool = true;
 639
 640     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 641         run.path("src/libstd")
 642     }
 643
 644     fn make_run(run: RunConfig<'_>) {
 645         run.builder.ensure(Std {
 646             compiler: run.builder.compiler_for(
 647                 run.builder.top_stage,
 648                 run.builder.config.build,
 649                 run.target,
 650             ),
 651             target: run.target,
 652         });
 653     }
 654
 655     fn run(self, builder: &Builder<'_>) -> PathBuf {
 656         let compiler = self.compiler;
 657         let target = self.target;
 658
 659         let name = pkgname(builder, "rust-std");
 660         builder.info(&format!("Dist std stage{} ({} -> {})",
 661             compiler.stage, &compiler.host, target));
 662
 663         // The only true set of target libraries came from the build triple, so
 664         // let's reduce redundant work by only producing archives from that host.
 665         if compiler.host != builder.config.build {
 666             builder.info("\tskipping, not a build host");
 667             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 668         }
 669
 670         // We want to package up as many target libraries as possible
 671         // for the `rust-std` package, so if this is a host target we
 672         // depend on librustc and otherwise we just depend on libtest.
 673         if builder.hosts.iter().any(|t| t == target) {
 674             builder.ensure(compile::Rustc { compiler, target });
 675         } else {
 676             if builder.no_std(target) == Some(true) {
 677                 // the `test` doesn't compile for no-std targets
 678                 builder.ensure(compile::Std { compiler, target });
 679             } else {
 680                 builder.ensure(compile::Test { compiler, target });
 681             }
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             // Find the actual compiler (handling the full bootstrap option) which
 741             // produced the save-analysis data because that data isn't copied
 742             // through the sysroot uplifting.
 743             compiler: run.builder.compiler_for(
 744                 run.builder.top_stage,
 745                 run.builder.config.build,
 746                 run.target,
 747             ),
 748             target: run.target,
 749         });
 750     }
 751
 752     /// Creates a tarball of save-analysis metadata, if available.
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756         assert!(builder.config.extended);
 757         builder.info("Dist analysis");
 758         let name = pkgname(builder, "rust-analysis");
 759
 760         if &compiler.host != builder.config.build {
 761             builder.info("\tskipping, not a build host");
 762             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 763         }
 764
 765         builder.ensure(Std { compiler, target });
 766
 767         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 768
 769         let src = builder.stage_out(compiler, Mode::Std)
 770             .join(target).join(builder.cargo_dir()).join("deps");
 771
 772         let image_src = src.join("save-analysis");
 773         let dst = image.join("lib/rustlib").join(target).join("analysis");
 774         t!(fs::create_dir_all(&dst));
 775         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 776         builder.cp_r(&image_src, &dst);
 777
 778         let mut cmd = rust_installer(builder);
 779         cmd.arg("generate")
 780            .arg("--product-name=Rust")
 781            .arg("--rel-manifest-dir=rustlib")
 782            .arg("--success-message=save-analysis-saved.")
 783            .arg("--image-dir").arg(&image)
 784            .arg("--work-dir").arg(&tmpdir(builder))
 785            .arg("--output-dir").arg(&distdir(builder))
 786            .arg(format!("--package-name={}-{}", name, target))
 787            .arg(format!("--component-name=rust-analysis-{}", target))
 788            .arg("--legacy-manifest-dirs=rustlib,cargo");
 789         builder.run(&mut cmd);
 790         builder.remove_dir(&image);
 791         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 792     }
 793 }
 794
 795 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 796     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 797         let spath = match path.to_str() {
 798             Some(path) => path,
 799             None => return false,
 800         };
 801         if spath.ends_with("~") || spath.ends_with(".pyc") {
 802             return false
 803         }
 804
 805         const LLVM_PROJECTS: &[&str] = &[
 806             "llvm-project/clang", "llvm-project\\clang",
 807             "llvm-project/lld", "llvm-project\\lld",
 808             "llvm-project/lldb", "llvm-project\\lldb",
 809             "llvm-project/llvm", "llvm-project\\llvm",
 810         ];
 811         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 812             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 813         {
 814             return false;
 815         }
 816
 817         const LLVM_TEST: &[&str] = &[
 818             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 819             "llvm-emscripten/test", "llvm-emscripten\\test",
 820         ];
 821         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 822             (spath.ends_with(".ll") ||
 823              spath.ends_with(".td") ||
 824              spath.ends_with(".s")) {
 825             return false
 826         }
 827         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 828             return false
 829         }
 830
 831         let full_path = Path::new(dir).join(path);
 832         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 833             return false;
 834         }
 835
 836         let excludes = [
 837             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 838             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 839             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 840             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 841         ];
 842         !path.iter()
 843              .map(|s| s.to_str().unwrap())
 844              .any(|s| excludes.contains(&s))
 845     }
 846
 847     // Copy the directories using our filter
 848     for item in src_dirs {
 849         let dst = &dst_dir.join(item);
 850         t!(fs::create_dir_all(dst));
 851         builder.cp_filtered(
 852             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 853     }
 854 }
 855
 856 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 857 pub struct Src;
 858
 859 impl Step for Src {
 860     /// The output path of the src installer tarball
 861     type Output = PathBuf;
 862     const DEFAULT: bool = true;
 863     const ONLY_HOSTS: bool = true;
 864
 865     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 866         run.path("src")
 867     }
 868
 869     fn make_run(run: RunConfig<'_>) {
 870         run.builder.ensure(Src);
 871     }
 872
 873     /// Creates the `rust-src` installer component
 874     fn run(self, builder: &Builder<'_>) -> PathBuf {
 875         builder.info("Dist src");
 876
 877         let name = pkgname(builder, "rust-src");
 878         let image = tmpdir(builder).join(format!("{}-image", name));
 879         let _ = fs::remove_dir_all(&image);
 880
 881         let dst = image.join("lib/rustlib/src");
 882         let dst_src = dst.join("rust");
 883         t!(fs::create_dir_all(&dst_src));
 884
 885         let src_files = [
 886             "Cargo.lock",
 887         ];
 888         // This is the reduced set of paths which will become the rust-src component
 889         // (essentially libstd and all of its path dependencies)
 890         let std_src_dirs = [
 891             "src/build_helper",
 892             "src/liballoc",
 893             "src/libcore",
 894             "src/libpanic_abort",
 895             "src/libpanic_unwind",
 896             "src/librustc_asan",
 897             "src/librustc_lsan",
 898             "src/librustc_msan",
 899             "src/librustc_tsan",
 900             "src/libstd",
 901             "src/libunwind",
 902             "src/libtest",
 903             "src/libterm",
 904             "src/libprofiler_builtins",
 905             "src/stdsimd",
 906             "src/libproc_macro",
 907             "src/tools/rustc-std-workspace-core",
 908             "src/tools/rustc-std-workspace-alloc",
 909             "src/librustc",
 910             "src/libsyntax",
 911         ];
 912
 913         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 914         for file in src_files.iter() {
 915             builder.copy(&builder.src.join(file), &dst_src.join(file));
 916         }
 917
 918         // Create source tarball in rust-installer format
 919         let mut cmd = rust_installer(builder);
 920         cmd.arg("generate")
 921            .arg("--product-name=Rust")
 922            .arg("--rel-manifest-dir=rustlib")
 923            .arg("--success-message=Awesome-Source.")
 924            .arg("--image-dir").arg(&image)
 925            .arg("--work-dir").arg(&tmpdir(builder))
 926            .arg("--output-dir").arg(&distdir(builder))
 927            .arg(format!("--package-name={}", name))
 928            .arg("--component-name=rust-src")
 929            .arg("--legacy-manifest-dirs=rustlib,cargo");
 930         builder.run(&mut cmd);
 931
 932         builder.remove_dir(&image);
 933         distdir(builder).join(&format!("{}.tar.gz", name))
 934     }
 935 }
 936
 937 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 938
 939 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 940 pub struct PlainSourceTarball;
 941
 942 impl Step for PlainSourceTarball {
 943     /// Produces the location of the tarball generated
 944     type Output = PathBuf;
 945     const DEFAULT: bool = true;
 946     const ONLY_HOSTS: bool = true;
 947
 948     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 949         let builder = run.builder;
 950         run.path("src").default_condition(builder.config.rust_dist_src)
 951     }
 952
 953     fn make_run(run: RunConfig<'_>) {
 954         run.builder.ensure(PlainSourceTarball);
 955     }
 956
 957     /// Creates the plain source tarball
 958     fn run(self, builder: &Builder<'_>) -> PathBuf {
 959         builder.info("Create plain source tarball");
 960
 961         // Make sure that the root folder of tarball has the correct name
 962         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 963         let plain_dst_src = tmpdir(builder).join(&plain_name);
 964         let _ = fs::remove_dir_all(&plain_dst_src);
 965         t!(fs::create_dir_all(&plain_dst_src));
 966
 967         // This is the set of root paths which will become part of the source package
 968         let src_files = [
 969             "COPYRIGHT",
 970             "LICENSE-APACHE",
 971             "LICENSE-MIT",
 972             "CONTRIBUTING.md",
 973             "README.md",
 974             "RELEASES.md",
 975             "configure",
 976             "x.py",
 977             "config.toml.example",
 978             "Cargo.toml",
 979             "Cargo.lock",
 980         ];
 981         let src_dirs = [
 982             "src",
 983         ];
 984
 985         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 986
 987         // Copy the files normally
 988         for item in &src_files {
 989             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 990         }
 991
 992         // Create the version file
 993         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 994         if let Some(sha) = builder.rust_sha() {
 995             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 996         }
 997
 998         // If we're building from git sources, we need to vendor a complete distribution.
 999         if builder.rust_info.is_git() {
1000             // Get cargo-vendor installed, if it isn't already.
1001             let mut has_cargo_vendor = false;
1002             let mut cmd = Command::new(&builder.initial_cargo);
1003             for line in output(cmd.arg("install").arg("--list")).lines() {
1004                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
1005             }
1006             if !has_cargo_vendor {
1007                 let mut cmd = builder.cargo(
1008                     builder.compiler(0, builder.config.build),
1009                     Mode::ToolBootstrap,
1010                     builder.config.build,
1011                     "install"
1012                 );
1013                 cmd.arg("--force")
1014                    .arg("--debug")
1015                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1016                    .arg("cargo-vendor");
1017                 builder.run(&mut cmd);
1018             }
1019
1020             // Vendor all Cargo dependencies
1021             let mut cmd = Command::new(&builder.initial_cargo);
1022             cmd.arg("vendor")
1023                .current_dir(&plain_dst_src);
1024             builder.run(&mut cmd);
1025         }
1026
1027         // Create plain source tarball
1028         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1029         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1030         tarball.set_extension(""); // strip .gz
1031         tarball.set_extension(""); // strip .tar
1032         if let Some(dir) = tarball.parent() {
1033             builder.create_dir(&dir);
1034         }
1035         builder.info("running installer");
1036         let mut cmd = rust_installer(builder);
1037         cmd.arg("tarball")
1038            .arg("--input").arg(&plain_name)
1039            .arg("--output").arg(&tarball)
1040            .arg("--work-dir=.")
1041            .current_dir(tmpdir(builder));
1042         builder.run(&mut cmd);
1043         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1044     }
1045 }
1046
1047 // We have to run a few shell scripts, which choke quite a bit on both `\`
1048 // characters and on `C:\` paths, so normalize both of them away.
1049 pub fn sanitize_sh(path: &Path) -> String {
1050     let path = path.to_str().unwrap().replace("\\", "/");
1051     return change_drive(&path).unwrap_or(path);
1052
1053     fn change_drive(s: &str) -> Option<String> {
1054         let mut ch = s.chars();
1055         let drive = ch.next().unwrap_or('C');
1056         if ch.next() != Some(':') {
1057             return None
1058         }
1059         if ch.next() != Some('/') {
1060             return None
1061         }
1062         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1063     }
1064 }
1065
1066 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1067 pub struct Cargo {
1068     pub compiler: Compiler,
1069     pub target: Interned<String>,
1070 }
1071
1072 impl Step for Cargo {
1073     type Output = PathBuf;
1074     const ONLY_HOSTS: bool = true;
1075
1076     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1077         run.path("cargo")
1078     }
1079
1080     fn make_run(run: RunConfig<'_>) {
1081         run.builder.ensure(Cargo {
1082             compiler: run.builder.compiler_for(
1083                 run.builder.top_stage,
1084                 run.builder.config.build,
1085                 run.target,
1086             ),
1087             target: run.target,
1088         });
1089     }
1090
1091     fn run(self, builder: &Builder<'_>) -> PathBuf {
1092         let compiler = self.compiler;
1093         let target = self.target;
1094
1095         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1096         let src = builder.src.join("src/tools/cargo");
1097         let etc = src.join("src/etc");
1098         let release_num = builder.release_num("cargo");
1099         let name = pkgname(builder, "cargo");
1100         let version = builder.cargo_info.version(builder, &release_num);
1101
1102         let tmp = tmpdir(builder);
1103         let image = tmp.join("cargo-image");
1104         drop(fs::remove_dir_all(&image));
1105         builder.create_dir(&image);
1106
1107         // Prepare the image directory
1108         builder.create_dir(&image.join("share/zsh/site-functions"));
1109         builder.create_dir(&image.join("etc/bash_completion.d"));
1110         let cargo = builder.ensure(tool::Cargo { compiler, target });
1111         builder.install(&cargo, &image.join("bin"), 0o755);
1112         for man in t!(etc.join("man").read_dir()) {
1113             let man = t!(man);
1114             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1115         }
1116         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1117         builder.copy(&etc.join("cargo.bashcomp.sh"),
1118              &image.join("etc/bash_completion.d/cargo"));
1119         let doc = image.join("share/doc/cargo");
1120         builder.install(&src.join("README.md"), &doc, 0o644);
1121         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1122         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1123         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1124
1125         // Prepare the overlay
1126         let overlay = tmp.join("cargo-overlay");
1127         drop(fs::remove_dir_all(&overlay));
1128         builder.create_dir(&overlay);
1129         builder.install(&src.join("README.md"), &overlay, 0o644);
1130         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1131         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1132         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1133         builder.create(&overlay.join("version"), &version);
1134
1135         // Generate the installer tarball
1136         let mut cmd = rust_installer(builder);
1137         cmd.arg("generate")
1138            .arg("--product-name=Rust")
1139            .arg("--rel-manifest-dir=rustlib")
1140            .arg("--success-message=Rust-is-ready-to-roll.")
1141            .arg("--image-dir").arg(&image)
1142            .arg("--work-dir").arg(&tmpdir(builder))
1143            .arg("--output-dir").arg(&distdir(builder))
1144            .arg("--non-installed-overlay").arg(&overlay)
1145            .arg(format!("--package-name={}-{}", name, target))
1146            .arg("--component-name=cargo")
1147            .arg("--legacy-manifest-dirs=rustlib,cargo");
1148         builder.run(&mut cmd);
1149         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1150     }
1151 }
1152
1153 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1154 pub struct Rls {
1155     pub compiler: Compiler,
1156     pub target: Interned<String>,
1157 }
1158
1159 impl Step for Rls {
1160     type Output = Option<PathBuf>;
1161     const ONLY_HOSTS: bool = true;
1162
1163     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1164         run.path("rls")
1165     }
1166
1167     fn make_run(run: RunConfig<'_>) {
1168         run.builder.ensure(Rls {
1169             compiler: run.builder.compiler_for(
1170                 run.builder.top_stage,
1171                 run.builder.config.build,
1172                 run.target,
1173             ),
1174             target: run.target,
1175         });
1176     }
1177
1178     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1179         let compiler = self.compiler;
1180         let target = self.target;
1181         assert!(builder.config.extended);
1182
1183         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1184         let src = builder.src.join("src/tools/rls");
1185         let release_num = builder.release_num("rls");
1186         let name = pkgname(builder, "rls");
1187         let version = builder.rls_info.version(builder, &release_num);
1188
1189         let tmp = tmpdir(builder);
1190         let image = tmp.join("rls-image");
1191         drop(fs::remove_dir_all(&image));
1192         t!(fs::create_dir_all(&image));
1193
1194         // Prepare the image directory
1195         // We expect RLS to build, because we've exited this step above if tool
1196         // state for RLS isn't testing.
1197         let rls = builder.ensure(tool::Rls {
1198             compiler,
1199             target,
1200             extra_features: Vec::new(),
1201         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1202
1203         builder.install(&rls, &image.join("bin"), 0o755);
1204         let doc = image.join("share/doc/rls");
1205         builder.install(&src.join("README.md"), &doc, 0o644);
1206         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1207         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1208
1209         // Prepare the overlay
1210         let overlay = tmp.join("rls-overlay");
1211         drop(fs::remove_dir_all(&overlay));
1212         t!(fs::create_dir_all(&overlay));
1213         builder.install(&src.join("README.md"), &overlay, 0o644);
1214         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1215         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1216         builder.create(&overlay.join("version"), &version);
1217
1218         // Generate the installer tarball
1219         let mut cmd = rust_installer(builder);
1220         cmd.arg("generate")
1221            .arg("--product-name=Rust")
1222            .arg("--rel-manifest-dir=rustlib")
1223            .arg("--success-message=RLS-ready-to-serve.")
1224            .arg("--image-dir").arg(&image)
1225            .arg("--work-dir").arg(&tmpdir(builder))
1226            .arg("--output-dir").arg(&distdir(builder))
1227            .arg("--non-installed-overlay").arg(&overlay)
1228            .arg(format!("--package-name={}-{}", name, target))
1229            .arg("--legacy-manifest-dirs=rustlib,cargo")
1230            .arg("--component-name=rls-preview");
1231
1232         builder.run(&mut cmd);
1233         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1234     }
1235 }
1236
1237 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1238 pub struct Clippy {
1239     pub compiler: Compiler,
1240     pub target: Interned<String>,
1241 }
1242
1243 impl Step for Clippy {
1244     type Output = Option<PathBuf>;
1245     const ONLY_HOSTS: bool = true;
1246
1247     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1248         run.path("clippy")
1249     }
1250
1251     fn make_run(run: RunConfig<'_>) {
1252         run.builder.ensure(Clippy {
1253             compiler: run.builder.compiler_for(
1254                 run.builder.top_stage,
1255                 run.builder.config.build,
1256                 run.target,
1257             ),
1258             target: run.target,
1259         });
1260     }
1261
1262     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1263         let compiler = self.compiler;
1264         let target = self.target;
1265         assert!(builder.config.extended);
1266
1267         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1268         let src = builder.src.join("src/tools/clippy");
1269         let release_num = builder.release_num("clippy");
1270         let name = pkgname(builder, "clippy");
1271         let version = builder.clippy_info.version(builder, &release_num);
1272
1273         let tmp = tmpdir(builder);
1274         let image = tmp.join("clippy-image");
1275         drop(fs::remove_dir_all(&image));
1276         builder.create_dir(&image);
1277
1278         // Prepare the image directory
1279         // We expect clippy to build, because we've exited this step above if tool
1280         // state for clippy isn't testing.
1281         let clippy = builder.ensure(tool::Clippy {
1282             compiler,
1283             target,
1284             extra_features: Vec::new(),
1285         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1286         let cargoclippy = builder.ensure(tool::CargoClippy {
1287             compiler,
1288             target, extra_features: Vec::new()
1289         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1290
1291         builder.install(&clippy, &image.join("bin"), 0o755);
1292         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1293         let doc = image.join("share/doc/clippy");
1294         builder.install(&src.join("README.md"), &doc, 0o644);
1295         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1296         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1297
1298         // Prepare the overlay
1299         let overlay = tmp.join("clippy-overlay");
1300         drop(fs::remove_dir_all(&overlay));
1301         t!(fs::create_dir_all(&overlay));
1302         builder.install(&src.join("README.md"), &overlay, 0o644);
1303         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1304         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1305         builder.create(&overlay.join("version"), &version);
1306
1307         // Generate the installer tarball
1308         let mut cmd = rust_installer(builder);
1309         cmd.arg("generate")
1310            .arg("--product-name=Rust")
1311            .arg("--rel-manifest-dir=rustlib")
1312            .arg("--success-message=clippy-ready-to-serve.")
1313            .arg("--image-dir").arg(&image)
1314            .arg("--work-dir").arg(&tmpdir(builder))
1315            .arg("--output-dir").arg(&distdir(builder))
1316            .arg("--non-installed-overlay").arg(&overlay)
1317            .arg(format!("--package-name={}-{}", name, target))
1318            .arg("--legacy-manifest-dirs=rustlib,cargo")
1319            .arg("--component-name=clippy-preview");
1320
1321         builder.run(&mut cmd);
1322         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1323     }
1324 }
1325
1326 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1327 pub struct Miri {
1328     pub compiler: Compiler,
1329     pub target: Interned<String>,
1330 }
1331
1332 impl Step for Miri {
1333     type Output = Option<PathBuf>;
1334     const ONLY_HOSTS: bool = true;
1335
1336     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1337         run.path("miri")
1338     }
1339
1340     fn make_run(run: RunConfig<'_>) {
1341         run.builder.ensure(Miri {
1342             compiler: run.builder.compiler_for(
1343                 run.builder.top_stage,
1344                 run.builder.config.build,
1345                 run.target,
1346             ),
1347             target: run.target,
1348         });
1349     }
1350
1351     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1352         let compiler = self.compiler;
1353         let target = self.target;
1354         assert!(builder.config.extended);
1355
1356         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1357         let src = builder.src.join("src/tools/miri");
1358         let release_num = builder.release_num("miri");
1359         let name = pkgname(builder, "miri");
1360         let version = builder.miri_info.version(builder, &release_num);
1361
1362         let tmp = tmpdir(builder);
1363         let image = tmp.join("miri-image");
1364         drop(fs::remove_dir_all(&image));
1365         builder.create_dir(&image);
1366
1367         // Prepare the image directory
1368         // We expect miri to build, because we've exited this step above if tool
1369         // state for miri isn't testing.
1370         let miri = builder.ensure(tool::Miri {
1371             compiler,
1372             target,
1373             extra_features: Vec::new(),
1374         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1375         let cargomiri = builder.ensure(tool::CargoMiri {
1376             compiler,
1377             target,
1378             extra_features: Vec::new()
1379         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1380
1381         builder.install(&miri, &image.join("bin"), 0o755);
1382         builder.install(&cargomiri, &image.join("bin"), 0o755);
1383         let doc = image.join("share/doc/miri");
1384         builder.install(&src.join("README.md"), &doc, 0o644);
1385         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1386         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1387
1388         // Prepare the overlay
1389         let overlay = tmp.join("miri-overlay");
1390         drop(fs::remove_dir_all(&overlay));
1391         t!(fs::create_dir_all(&overlay));
1392         builder.install(&src.join("README.md"), &overlay, 0o644);
1393         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1394         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1395         builder.create(&overlay.join("version"), &version);
1396
1397         // Generate the installer tarball
1398         let mut cmd = rust_installer(builder);
1399         cmd.arg("generate")
1400            .arg("--product-name=Rust")
1401            .arg("--rel-manifest-dir=rustlib")
1402            .arg("--success-message=miri-ready-to-serve.")
1403            .arg("--image-dir").arg(&image)
1404            .arg("--work-dir").arg(&tmpdir(builder))
1405            .arg("--output-dir").arg(&distdir(builder))
1406            .arg("--non-installed-overlay").arg(&overlay)
1407            .arg(format!("--package-name={}-{}", name, target))
1408            .arg("--legacy-manifest-dirs=rustlib,cargo")
1409            .arg("--component-name=miri-preview");
1410
1411         builder.run(&mut cmd);
1412         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1413     }
1414 }
1415
1416 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1417 pub struct Rustfmt {
1418     pub compiler: Compiler,
1419     pub target: Interned<String>,
1420 }
1421
1422 impl Step for Rustfmt {
1423     type Output = Option<PathBuf>;
1424     const ONLY_HOSTS: bool = true;
1425
1426     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1427         run.path("rustfmt")
1428     }
1429
1430     fn make_run(run: RunConfig<'_>) {
1431         run.builder.ensure(Rustfmt {
1432             compiler: run.builder.compiler_for(
1433                 run.builder.top_stage,
1434                 run.builder.config.build,
1435                 run.target,
1436             ),
1437             target: run.target,
1438         });
1439     }
1440
1441     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1442         let compiler = self.compiler;
1443         let target = self.target;
1444
1445         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1446         let src = builder.src.join("src/tools/rustfmt");
1447         let release_num = builder.release_num("rustfmt");
1448         let name = pkgname(builder, "rustfmt");
1449         let version = builder.rustfmt_info.version(builder, &release_num);
1450
1451         let tmp = tmpdir(builder);
1452         let image = tmp.join("rustfmt-image");
1453         drop(fs::remove_dir_all(&image));
1454         builder.create_dir(&image);
1455
1456         // Prepare the image directory
1457         let rustfmt = builder.ensure(tool::Rustfmt {
1458             compiler,
1459             target,
1460             extra_features: Vec::new(),
1461         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1462         let cargofmt = builder.ensure(tool::Cargofmt {
1463             compiler,
1464             target,
1465             extra_features: Vec::new(),
1466         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1467
1468         builder.install(&rustfmt, &image.join("bin"), 0o755);
1469         builder.install(&cargofmt, &image.join("bin"), 0o755);
1470         let doc = image.join("share/doc/rustfmt");
1471         builder.install(&src.join("README.md"), &doc, 0o644);
1472         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1473         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1474
1475         // Prepare the overlay
1476         let overlay = tmp.join("rustfmt-overlay");
1477         drop(fs::remove_dir_all(&overlay));
1478         builder.create_dir(&overlay);
1479         builder.install(&src.join("README.md"), &overlay, 0o644);
1480         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1481         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1482         builder.create(&overlay.join("version"), &version);
1483
1484         // Generate the installer tarball
1485         let mut cmd = rust_installer(builder);
1486         cmd.arg("generate")
1487            .arg("--product-name=Rust")
1488            .arg("--rel-manifest-dir=rustlib")
1489            .arg("--success-message=rustfmt-ready-to-fmt.")
1490            .arg("--image-dir").arg(&image)
1491            .arg("--work-dir").arg(&tmpdir(builder))
1492            .arg("--output-dir").arg(&distdir(builder))
1493            .arg("--non-installed-overlay").arg(&overlay)
1494            .arg(format!("--package-name={}-{}", name, target))
1495            .arg("--legacy-manifest-dirs=rustlib,cargo")
1496            .arg("--component-name=rustfmt-preview");
1497
1498         builder.run(&mut cmd);
1499         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1500     }
1501 }
1502
1503 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1504 pub struct Extended {
1505     stage: u32,
1506     host: Interned<String>,
1507     target: Interned<String>,
1508 }
1509
1510 impl Step for Extended {
1511     type Output = ();
1512     const DEFAULT: bool = true;
1513     const ONLY_HOSTS: bool = true;
1514
1515     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1516         let builder = run.builder;
1517         run.path("extended").default_condition(builder.config.extended)
1518     }
1519
1520     fn make_run(run: RunConfig<'_>) {
1521         run.builder.ensure(Extended {
1522             stage: run.builder.top_stage,
1523             host: run.builder.config.build,
1524             target: run.target,
1525         });
1526     }
1527
1528     /// Creates a combined installer for the specified target in the provided stage.
1529     fn run(self, builder: &Builder<'_>) {
1530         let target = self.target;
1531         let stage = self.stage;
1532         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1533
1534         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1535
1536         let rustc_installer = builder.ensure(Rustc {
1537             compiler: builder.compiler(stage, target),
1538         });
1539         let cargo_installer = builder.ensure(Cargo { compiler, target });
1540         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1541         let rls_installer = builder.ensure(Rls { compiler, target });
1542         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1543         let clippy_installer = builder.ensure(Clippy { compiler, target });
1544         let miri_installer = builder.ensure(Miri { compiler, target });
1545         let lldb_installer = builder.ensure(Lldb { target });
1546         let mingw_installer = builder.ensure(Mingw { host: target });
1547         let analysis_installer = builder.ensure(Analysis { compiler, target });
1548
1549         let docs_installer = builder.ensure(Docs { host: target, });
1550         let std_installer = builder.ensure(Std {
1551             compiler: builder.compiler(stage, target),
1552             target,
1553         });
1554
1555         let tmp = tmpdir(builder);
1556         let overlay = tmp.join("extended-overlay");
1557         let etc = builder.src.join("src/etc/installer");
1558         let work = tmp.join("work");
1559
1560         let _ = fs::remove_dir_all(&overlay);
1561         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1562         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1563         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1564         let version = builder.rust_version();
1565         builder.create(&overlay.join("version"), &version);
1566         if let Some(sha) = builder.rust_sha() {
1567             builder.create(&overlay.join("git-commit-hash"), &sha);
1568         }
1569         builder.install(&etc.join("README.md"), &overlay, 0o644);
1570
1571         // When rust-std package split from rustc, we needed to ensure that during
1572         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1573         // the std files during uninstall. To do this ensure that rustc comes
1574         // before rust-std in the list below.
1575         let mut tarballs = Vec::new();
1576         tarballs.push(rustc_installer);
1577         tarballs.push(cargo_installer);
1578         tarballs.extend(rls_installer.clone());
1579         tarballs.extend(clippy_installer.clone());
1580         tarballs.extend(miri_installer.clone());
1581         tarballs.extend(rustfmt_installer.clone());
1582         tarballs.extend(llvm_tools_installer);
1583         tarballs.extend(lldb_installer);
1584         tarballs.push(analysis_installer);
1585         tarballs.push(std_installer);
1586         if builder.config.docs {
1587             tarballs.push(docs_installer);
1588         }
1589         if target.contains("pc-windows-gnu") {
1590             tarballs.push(mingw_installer.unwrap());
1591         }
1592         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1593         for tarball in &tarballs[1..] {
1594             input_tarballs.push(",");
1595             input_tarballs.push(tarball);
1596         }
1597
1598         let mut cmd = rust_installer(builder);
1599         cmd.arg("combine")
1600             .arg("--product-name=Rust")
1601             .arg("--rel-manifest-dir=rustlib")
1602             .arg("--success-message=Rust-is-ready-to-roll.")
1603             .arg("--work-dir").arg(&work)
1604             .arg("--output-dir").arg(&distdir(builder))
1605             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1606             .arg("--legacy-manifest-dirs=rustlib,cargo")
1607             .arg("--input-tarballs").arg(input_tarballs)
1608             .arg("--non-installed-overlay").arg(&overlay);
1609         builder.run(&mut cmd);
1610
1611         let mut license = String::new();
1612         license += &builder.read(&builder.src.join("COPYRIGHT"));
1613         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1614         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1615         license.push_str("\n");
1616         license.push_str("\n");
1617
1618         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1619         let mut rtf = rtf.to_string();
1620         rtf.push_str("\n");
1621         for line in license.lines() {
1622             rtf.push_str(line);
1623             rtf.push_str("\\line ");
1624         }
1625         rtf.push_str("}");
1626
1627         fn filter(contents: &str, marker: &str) -> String {
1628             let start = format!("tool-{}-start", marker);
1629             let end = format!("tool-{}-end", marker);
1630             let mut lines = Vec::new();
1631             let mut omitted = false;
1632             for line in contents.lines() {
1633                 if line.contains(&start) {
1634                     omitted = true;
1635                 } else if line.contains(&end) {
1636                     omitted = false;
1637                 } else if !omitted {
1638                     lines.push(line);
1639                 }
1640             }
1641
1642             lines.join("\n")
1643         }
1644
1645         let xform = |p: &Path| {
1646             let mut contents = t!(fs::read_to_string(p));
1647             if rls_installer.is_none() {
1648                 contents = filter(&contents, "rls");
1649             }
1650             if clippy_installer.is_none() {
1651                 contents = filter(&contents, "clippy");
1652             }
1653             if miri_installer.is_none() {
1654                 contents = filter(&contents, "miri");
1655             }
1656             if rustfmt_installer.is_none() {
1657                 contents = filter(&contents, "rustfmt");
1658             }
1659             let ret = tmp.join(p.file_name().unwrap());
1660             t!(fs::write(&ret, &contents));
1661             ret
1662         };
1663
1664         if target.contains("apple-darwin") {
1665             let pkg = tmp.join("pkg");
1666             let _ = fs::remove_dir_all(&pkg);
1667
1668             let pkgbuild = |component: &str| {
1669                 let mut cmd = Command::new("pkgbuild");
1670                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1671                     .arg("--scripts").arg(pkg.join(component))
1672                     .arg("--nopayload")
1673                     .arg(pkg.join(component).with_extension("pkg"));
1674                 builder.run(&mut cmd);
1675             };
1676
1677             let prepare = |name: &str| {
1678                 builder.create_dir(&pkg.join(name));
1679                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1680                         &pkg.join(name));
1681                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1682                 pkgbuild(name);
1683             };
1684             prepare("rustc");
1685             prepare("cargo");
1686             prepare("rust-docs");
1687             prepare("rust-std");
1688             prepare("rust-analysis");
1689
1690             if rls_installer.is_some() {
1691                 prepare("rls");
1692             }
1693             if clippy_installer.is_some() {
1694                 prepare("clippy");
1695             }
1696             if miri_installer.is_some() {
1697                 prepare("miri");
1698             }
1699
1700             // create an 'uninstall' package
1701             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1702             pkgbuild("uninstall");
1703
1704             builder.create_dir(&pkg.join("res"));
1705             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1706             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1707             let mut cmd = Command::new("productbuild");
1708             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1709                 .arg("--resources").arg(pkg.join("res"))
1710                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1711                                                     pkgname(builder, "rust"),
1712                                                     target)))
1713                 .arg("--package-path").arg(&pkg);
1714             builder.run(&mut cmd);
1715         }
1716
1717         if target.contains("windows") {
1718             let exe = tmp.join("exe");
1719             let _ = fs::remove_dir_all(&exe);
1720
1721             let prepare = |name: &str| {
1722                 builder.create_dir(&exe.join(name));
1723                 let dir = if name == "rust-std" || name == "rust-analysis" {
1724                     format!("{}-{}", name, target)
1725                 } else if name == "rls" {
1726                     "rls-preview".to_string()
1727                 } else if name == "clippy" {
1728                     "clippy-preview".to_string()
1729                 } else if name == "miri" {
1730                     "miri-preview".to_string()
1731                 } else {
1732                     name.to_string()
1733                 };
1734                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1735                             .join(dir),
1736                         &exe.join(name));
1737                 builder.remove(&exe.join(name).join("manifest.in"));
1738             };
1739             prepare("rustc");
1740             prepare("cargo");
1741             prepare("rust-analysis");
1742             prepare("rust-docs");
1743             prepare("rust-std");
1744             if rls_installer.is_some() {
1745                 prepare("rls");
1746             }
1747             if clippy_installer.is_some() {
1748                 prepare("clippy");
1749             }
1750             if miri_installer.is_some() {
1751                 prepare("miri");
1752             }
1753             if target.contains("windows-gnu") {
1754                 prepare("rust-mingw");
1755             }
1756
1757             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1758             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1759             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1760             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1761             builder.create(&exe.join("LICENSE.txt"), &license);
1762
1763             // Generate exe installer
1764             let mut cmd = Command::new("iscc");
1765             cmd.arg("rust.iss")
1766                 .current_dir(&exe);
1767             if target.contains("windows-gnu") {
1768                 cmd.arg("/dMINGW");
1769             }
1770             add_env(builder, &mut cmd, target);
1771             builder.run(&mut cmd);
1772             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1773                     &distdir(builder),
1774                     0o755);
1775
1776             // Generate msi installer
1777             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1778             let heat = wix.join("bin/heat.exe");
1779             let candle = wix.join("bin/candle.exe");
1780             let light = wix.join("bin/light.exe");
1781
1782             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1783             builder.run(Command::new(&heat)
1784                             .current_dir(&exe)
1785                             .arg("dir")
1786                             .arg("rustc")
1787                             .args(&heat_flags)
1788                             .arg("-cg").arg("RustcGroup")
1789                             .arg("-dr").arg("Rustc")
1790                             .arg("-var").arg("var.RustcDir")
1791                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1792             builder.run(Command::new(&heat)
1793                             .current_dir(&exe)
1794                             .arg("dir")
1795                             .arg("rust-docs")
1796                             .args(&heat_flags)
1797                             .arg("-cg").arg("DocsGroup")
1798                             .arg("-dr").arg("Docs")
1799                             .arg("-var").arg("var.DocsDir")
1800                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1801                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1802             builder.run(Command::new(&heat)
1803                             .current_dir(&exe)
1804                             .arg("dir")
1805                             .arg("cargo")
1806                             .args(&heat_flags)
1807                             .arg("-cg").arg("CargoGroup")
1808                             .arg("-dr").arg("Cargo")
1809                             .arg("-var").arg("var.CargoDir")
1810                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1811                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1812             builder.run(Command::new(&heat)
1813                             .current_dir(&exe)
1814                             .arg("dir")
1815                             .arg("rust-std")
1816                             .args(&heat_flags)
1817                             .arg("-cg").arg("StdGroup")
1818                             .arg("-dr").arg("Std")
1819                             .arg("-var").arg("var.StdDir")
1820                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1821             if rls_installer.is_some() {
1822                 builder.run(Command::new(&heat)
1823                                 .current_dir(&exe)
1824                                 .arg("dir")
1825                                 .arg("rls")
1826                                 .args(&heat_flags)
1827                                 .arg("-cg").arg("RlsGroup")
1828                                 .arg("-dr").arg("Rls")
1829                                 .arg("-var").arg("var.RlsDir")
1830                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1831                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1832             }
1833             if clippy_installer.is_some() {
1834                 builder.run(Command::new(&heat)
1835                                 .current_dir(&exe)
1836                                 .arg("dir")
1837                                 .arg("clippy")
1838                                 .args(&heat_flags)
1839                                 .arg("-cg").arg("ClippyGroup")
1840                                 .arg("-dr").arg("Clippy")
1841                                 .arg("-var").arg("var.ClippyDir")
1842                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1843                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1844             }
1845             if miri_installer.is_some() {
1846                 builder.run(Command::new(&heat)
1847                                 .current_dir(&exe)
1848                                 .arg("dir")
1849                                 .arg("miri")
1850                                 .args(&heat_flags)
1851                                 .arg("-cg").arg("MiriGroup")
1852                                 .arg("-dr").arg("Miri")
1853                                 .arg("-var").arg("var.MiriDir")
1854                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1855                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1856             }
1857             builder.run(Command::new(&heat)
1858                             .current_dir(&exe)
1859                             .arg("dir")
1860                             .arg("rust-analysis")
1861                             .args(&heat_flags)
1862                             .arg("-cg").arg("AnalysisGroup")
1863                             .arg("-dr").arg("Analysis")
1864                             .arg("-var").arg("var.AnalysisDir")
1865                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1866                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1867             if target.contains("windows-gnu") {
1868                 builder.run(Command::new(&heat)
1869                                 .current_dir(&exe)
1870                                 .arg("dir")
1871                                 .arg("rust-mingw")
1872                                 .args(&heat_flags)
1873                                 .arg("-cg").arg("GccGroup")
1874                                 .arg("-dr").arg("Gcc")
1875                                 .arg("-var").arg("var.GccDir")
1876                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1877             }
1878
1879             let candle = |input: &Path| {
1880                 let output = exe.join(input.file_stem().unwrap())
1881                                 .with_extension("wixobj");
1882                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1883                 let mut cmd = Command::new(&candle);
1884                 cmd.current_dir(&exe)
1885                     .arg("-nologo")
1886                     .arg("-dRustcDir=rustc")
1887                     .arg("-dDocsDir=rust-docs")
1888                     .arg("-dCargoDir=cargo")
1889                     .arg("-dStdDir=rust-std")
1890                     .arg("-dAnalysisDir=rust-analysis")
1891                     .arg("-arch").arg(&arch)
1892                     .arg("-out").arg(&output)
1893                     .arg(&input);
1894                 add_env(builder, &mut cmd, target);
1895
1896                 if rls_installer.is_some() {
1897                     cmd.arg("-dRlsDir=rls");
1898                 }
1899                 if clippy_installer.is_some() {
1900                     cmd.arg("-dClippyDir=clippy");
1901                 }
1902                 if miri_installer.is_some() {
1903                     cmd.arg("-dMiriDir=miri");
1904                 }
1905                 if target.contains("windows-gnu") {
1906                     cmd.arg("-dGccDir=rust-mingw");
1907                 }
1908                 builder.run(&mut cmd);
1909             };
1910             candle(&xform(&etc.join("msi/rust.wxs")));
1911             candle(&etc.join("msi/ui.wxs"));
1912             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1913             candle("RustcGroup.wxs".as_ref());
1914             candle("DocsGroup.wxs".as_ref());
1915             candle("CargoGroup.wxs".as_ref());
1916             candle("StdGroup.wxs".as_ref());
1917             if rls_installer.is_some() {
1918                 candle("RlsGroup.wxs".as_ref());
1919             }
1920             if clippy_installer.is_some() {
1921                 candle("ClippyGroup.wxs".as_ref());
1922             }
1923             if miri_installer.is_some() {
1924                 candle("MiriGroup.wxs".as_ref());
1925             }
1926             candle("AnalysisGroup.wxs".as_ref());
1927
1928             if target.contains("windows-gnu") {
1929                 candle("GccGroup.wxs".as_ref());
1930             }
1931
1932             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1933             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1934             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1935
1936             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1937             let mut cmd = Command::new(&light);
1938             cmd.arg("-nologo")
1939                 .arg("-ext").arg("WixUIExtension")
1940                 .arg("-ext").arg("WixUtilExtension")
1941                 .arg("-out").arg(exe.join(&filename))
1942                 .arg("rust.wixobj")
1943                 .arg("ui.wixobj")
1944                 .arg("rustwelcomedlg.wixobj")
1945                 .arg("RustcGroup.wixobj")
1946                 .arg("DocsGroup.wixobj")
1947                 .arg("CargoGroup.wixobj")
1948                 .arg("StdGroup.wixobj")
1949                 .arg("AnalysisGroup.wixobj")
1950                 .current_dir(&exe);
1951
1952             if rls_installer.is_some() {
1953                 cmd.arg("RlsGroup.wixobj");
1954             }
1955             if clippy_installer.is_some() {
1956                 cmd.arg("ClippyGroup.wixobj");
1957             }
1958             if miri_installer.is_some() {
1959                 cmd.arg("MiriGroup.wixobj");
1960             }
1961
1962             if target.contains("windows-gnu") {
1963                 cmd.arg("GccGroup.wixobj");
1964             }
1965             // ICE57 wrongly complains about the shortcuts
1966             cmd.arg("-sice:ICE57");
1967
1968             builder.run(&mut cmd);
1969
1970             if !builder.config.dry_run {
1971                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1972             }
1973         }
1974     }
1975 }
1976
1977 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1978     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1979     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1980        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1981        .env("CFG_RELEASE", builder.rust_release())
1982        .env("CFG_VER_MAJOR", parts.next().unwrap())
1983        .env("CFG_VER_MINOR", parts.next().unwrap())
1984        .env("CFG_VER_PATCH", parts.next().unwrap())
1985        .env("CFG_VER_BUILD", "0") // just needed to build
1986        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1987        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1988        .env("CFG_BUILD", target)
1989        .env("CFG_CHANNEL", &builder.config.channel);
1990
1991     if target.contains("windows-gnu") {
1992        cmd.env("CFG_MINGW", "1")
1993           .env("CFG_ABI", "GNU");
1994     } else {
1995        cmd.env("CFG_MINGW", "0")
1996           .env("CFG_ABI", "MSVC");
1997     }
1998
1999     if target.contains("x86_64") {
2000        cmd.env("CFG_PLATFORM", "x64");
2001     } else {
2002        cmd.env("CFG_PLATFORM", "x86");
2003     }
2004 }
2005
2006 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
2007 pub struct HashSign;
2008
2009 impl Step for HashSign {
2010     type Output = ();
2011     const ONLY_HOSTS: bool = true;
2012
2013     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2014         run.path("hash-and-sign")
2015     }
2016
2017     fn make_run(run: RunConfig<'_>) {
2018         run.builder.ensure(HashSign);
2019     }
2020
2021     fn run(self, builder: &Builder<'_>) {
2022         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2023         if builder.config.dry_run {
2024             return;
2025         }
2026         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2027             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2028         });
2029         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2030             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2031         });
2032         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2033             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2034         });
2035         let pass = t!(fs::read_to_string(&file));
2036
2037         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2038
2039         cmd.arg(sign);
2040         cmd.arg(distdir(builder));
2041         cmd.arg(today.trim());
2042         cmd.arg(builder.rust_package_vers());
2043         cmd.arg(addr);
2044         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2045         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2046         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2047         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2048         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2049         cmd.arg(builder.llvm_tools_package_vers());
2050         cmd.arg(builder.lldb_package_vers());
2051
2052         builder.create_dir(&distdir(builder));
2053
2054         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2055         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2056         let status = t!(child.wait());
2057         assert!(status.success());
2058     }
2059 }
2060
2061 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2062 // LLVM tools are linked dynamically.
2063 // Note: This function does no yet support Windows but we also don't support
2064 //       linking LLVM tools dynamically on Windows yet.
2065 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2066                                 target: Interned<String>,
2067                                 sysroot: &Path) {
2068     let src_libdir = builder
2069         .llvm_out(target)
2070         .join("lib");
2071     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2072     t!(fs::create_dir_all(&dst_libdir));
2073
2074     if target.contains("apple-darwin") {
2075         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2076         if llvm_dylib_path.exists() {
2077             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2078         }
2079         return
2080     }
2081
2082     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2083     // Since tools link to the latter rather than the former, we have to
2084     // follow the symlink to find out what to distribute.
2085     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2086     if llvm_dylib_path.exists() {
2087         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2088             panic!("dist: Error calling canonicalize path `{}`: {}",
2089                    llvm_dylib_path.display(), e);
2090         });
2091
2092
2093         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2094     }
2095 }
2096
2097 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2098 pub struct LlvmTools {
2099     pub target: Interned<String>,
2100 }
2101
2102 impl Step for LlvmTools {
2103     type Output = Option<PathBuf>;
2104     const ONLY_HOSTS: bool = true;
2105
2106     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2107         run.path("llvm-tools")
2108     }
2109
2110     fn make_run(run: RunConfig<'_>) {
2111         run.builder.ensure(LlvmTools {
2112             target: run.target,
2113         });
2114     }
2115
2116     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2117         let target = self.target;
2118         assert!(builder.config.extended);
2119
2120         /* run only if llvm-config isn't used */
2121         if let Some(config) = builder.config.target_config.get(&target) {
2122             if let Some(ref _s) = config.llvm_config {
2123                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2124                     target));
2125                 return None;
2126             }
2127         }
2128
2129         builder.info(&format!("Dist LlvmTools ({})", target));
2130         let src = builder.src.join("src/llvm-project/llvm");
2131         let name = pkgname(builder, "llvm-tools");
2132
2133         let tmp = tmpdir(builder);
2134         let image = tmp.join("llvm-tools-image");
2135         drop(fs::remove_dir_all(&image));
2136
2137         // Prepare the image directory
2138         let src_bindir = builder
2139             .llvm_out(target)
2140             .join("bin");
2141         let dst_bindir = image.join("lib/rustlib")
2142             .join(&*target)
2143             .join("bin");
2144         t!(fs::create_dir_all(&dst_bindir));
2145         for tool in LLVM_TOOLS {
2146             let exe = src_bindir.join(exe(tool, &target));
2147             builder.install(&exe, &dst_bindir, 0o755);
2148         }
2149
2150         // Prepare the overlay
2151         let overlay = tmp.join("llvm-tools-overlay");
2152         drop(fs::remove_dir_all(&overlay));
2153         builder.create_dir(&overlay);
2154         builder.install(&src.join("README.txt"), &overlay, 0o644);
2155         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2156         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2157
2158         // Generate the installer tarball
2159         let mut cmd = rust_installer(builder);
2160         cmd.arg("generate")
2161             .arg("--product-name=Rust")
2162             .arg("--rel-manifest-dir=rustlib")
2163             .arg("--success-message=llvm-tools-installed.")
2164             .arg("--image-dir").arg(&image)
2165             .arg("--work-dir").arg(&tmpdir(builder))
2166             .arg("--output-dir").arg(&distdir(builder))
2167             .arg("--non-installed-overlay").arg(&overlay)
2168             .arg(format!("--package-name={}-{}", name, target))
2169             .arg("--legacy-manifest-dirs=rustlib,cargo")
2170             .arg("--component-name=llvm-tools-preview");
2171
2172
2173         builder.run(&mut cmd);
2174         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2175     }
2176 }
2177
2178 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2179 pub struct Lldb {
2180     pub target: Interned<String>,
2181 }
2182
2183 impl Step for Lldb {
2184     type Output = Option<PathBuf>;
2185     const ONLY_HOSTS: bool = true;
2186     const DEFAULT: bool = true;
2187
2188     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2189         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2190     }
2191
2192     fn make_run(run: RunConfig<'_>) {
2193         run.builder.ensure(Lldb {
2194             target: run.target,
2195         });
2196     }
2197
2198     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2199         let target = self.target;
2200
2201         if builder.config.dry_run {
2202             return None;
2203         }
2204
2205         let bindir = builder
2206             .llvm_out(target)
2207             .join("bin");
2208         let lldb_exe = bindir.join(exe("lldb", &target));
2209         if !lldb_exe.exists() {
2210             return None;
2211         }
2212
2213         builder.info(&format!("Dist Lldb ({})", target));
2214         let src = builder.src.join("src/llvm-project/lldb");
2215         let name = pkgname(builder, "lldb");
2216
2217         let tmp = tmpdir(builder);
2218         let image = tmp.join("lldb-image");
2219         drop(fs::remove_dir_all(&image));
2220
2221         // Prepare the image directory
2222         let root = image.join("lib/rustlib").join(&*target);
2223         let dst = root.join("bin");
2224         t!(fs::create_dir_all(&dst));
2225         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2226             let exe = bindir.join(exe(program, &target));
2227             builder.install(&exe, &dst, 0o755);
2228         }
2229
2230         // The libraries.
2231         let libdir = builder.llvm_out(target).join("lib");
2232         let dst = root.join("lib");
2233         t!(fs::create_dir_all(&dst));
2234         for entry in t!(fs::read_dir(&libdir)) {
2235             let entry = entry.unwrap();
2236             if let Ok(name) = entry.file_name().into_string() {
2237                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2238                     if t!(entry.file_type()).is_symlink() {
2239                         builder.copy_to_folder(&entry.path(), &dst);
2240                     } else {
2241                        builder.install(&entry.path(), &dst, 0o755);
2242                     }
2243                 }
2244             }
2245         }
2246
2247         // The lldb scripts might be installed in lib/python$version
2248         // or in lib64/python$version.  If lib64 exists, use it;
2249         // otherwise lib.
2250         let libdir = builder.llvm_out(target).join("lib64");
2251         let (libdir, libdir_name) = if libdir.exists() {
2252             (libdir, "lib64")
2253         } else {
2254             (builder.llvm_out(target).join("lib"), "lib")
2255         };
2256         for entry in t!(fs::read_dir(&libdir)) {
2257             let entry = t!(entry);
2258             if let Ok(name) = entry.file_name().into_string() {
2259                 if name.starts_with("python") {
2260                     let dst = root.join(libdir_name)
2261                         .join(entry.file_name());
2262                     t!(fs::create_dir_all(&dst));
2263                     builder.cp_r(&entry.path(), &dst);
2264                     break;
2265                 }
2266             }
2267         }
2268
2269         // Prepare the overlay
2270         let overlay = tmp.join("lldb-overlay");
2271         drop(fs::remove_dir_all(&overlay));
2272         builder.create_dir(&overlay);
2273         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2274         builder.create(&overlay.join("version"), &builder.lldb_vers());
2275
2276         // Generate the installer tarball
2277         let mut cmd = rust_installer(builder);
2278         cmd.arg("generate")
2279             .arg("--product-name=Rust")
2280             .arg("--rel-manifest-dir=rustlib")
2281             .arg("--success-message=lldb-installed.")
2282             .arg("--image-dir").arg(&image)
2283             .arg("--work-dir").arg(&tmpdir(builder))
2284             .arg("--output-dir").arg(&distdir(builder))
2285             .arg("--non-installed-overlay").arg(&overlay)
2286             .arg(format!("--package-name={}-{}", name, target))
2287             .arg("--legacy-manifest-dirs=rustlib,cargo")
2288             .arg("--component-name=lldb-preview");
2289
2290
2291         builder.run(&mut cmd);
2292         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2293     }
2294 }