src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472             let libdir = builder.rustc_libdir(compiler);
 473
 474             // Copy rustc/rustdoc binaries
 475             t!(fs::create_dir_all(image.join("bin")));
 476             builder.cp_r(&src.join("bin"), &image.join("bin"));
 477
 478             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 479
 480             let libdir_relative = builder.libdir_relative(compiler);
 481
 482             // Copy runtime DLLs needed by the compiler
 483             if libdir_relative.to_str() != Some("bin") {
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             builder.install(&entry.path(), &image.join(&libdir_relative), 0o644);
 489                         }
 490                     }
 491                 }
 492             }
 493
 494             // Copy over the codegen backends
 495             let backends_src = builder.sysroot_codegen_backends(compiler);
 496             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 497             let backends_dst = image.join(&backends_rel);
 498             t!(fs::create_dir_all(&backends_dst));
 499             builder.cp_r(&backends_src, &backends_dst);
 500
 501             // Copy libLLVM.so to the lib dir as well, if needed. While not
 502             // technically needed by rustc itself it's needed by lots of other
 503             // components like the llvm tools and LLD. LLD is included below and
 504             // tools/LLDB come later, so let's just throw it in the rustc
 505             // component for now.
 506             maybe_install_llvm_dylib(builder, host, image);
 507
 508             // Copy over lld if it's there
 509             if builder.config.lld_enabled {
 510                 let exe = exe("rust-lld", &compiler.host);
 511                 let src = builder.sysroot_libdir(compiler, host)
 512                     .parent()
 513                     .unwrap()
 514                     .join("bin")
 515                     .join(&exe);
 516                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 517                 let dst = image.join("lib/rustlib")
 518                     .join(&*host)
 519                     .join("bin")
 520                     .join(&exe);
 521                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 522                 builder.copy(&src, &dst);
 523             }
 524
 525             // Man pages
 526             t!(fs::create_dir_all(image.join("share/man/man1")));
 527             let man_src = builder.src.join("src/doc/man");
 528             let man_dst = image.join("share/man/man1");
 529
 530             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 531             let time = env::var("SOURCE_DATE_EPOCH")
 532                 .map(|timestamp| {
 533                     let epoch = timestamp.parse().map_err(|err| {
 534                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 535                     }).unwrap();
 536
 537                     time::at(Timespec::new(epoch, 0))
 538                 })
 539                 .unwrap_or_else(|_| time::now());
 540
 541             let month_year = t!(time::strftime("%B %Y", &time));
 542             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 543             // to hardlink, and we don't want to edit the source templates
 544             for file_entry in builder.read_dir(&man_src) {
 545                 let page_src = file_entry.path();
 546                 let page_dst = man_dst.join(file_entry.file_name());
 547                 t!(fs::copy(&page_src, &page_dst));
 548                 // template in month/year and version number
 549                 builder.replace_in_file(&page_dst,
 550                                 &[("<INSERT DATE HERE>", &month_year),
 551                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 552             }
 553
 554             // Debugger scripts
 555             builder.ensure(DebuggerScripts {
 556                 sysroot: INTERNER.intern_path(image.to_owned()),
 557                 host,
 558             });
 559
 560             // Misc license info
 561             let cp = |file: &str| {
 562                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 563             };
 564             cp("COPYRIGHT");
 565             cp("LICENSE-APACHE");
 566             cp("LICENSE-MIT");
 567             cp("README.md");
 568         }
 569     }
 570 }
 571
 572 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 573 pub struct DebuggerScripts {
 574     pub sysroot: Interned<PathBuf>,
 575     pub host: Interned<String>,
 576 }
 577
 578 impl Step for DebuggerScripts {
 579     type Output = ();
 580
 581     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 582         run.path("src/lldb_batchmode.py")
 583     }
 584
 585     fn make_run(run: RunConfig<'_>) {
 586         run.builder.ensure(DebuggerScripts {
 587             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 588             host: run.target,
 589         });
 590     }
 591
 592     /// Copies debugger scripts for `target` into the `sysroot` specified.
 593     fn run(self, builder: &Builder<'_>) {
 594         let host = self.host;
 595         let sysroot = self.sysroot;
 596         let dst = sysroot.join("lib/rustlib/etc");
 597         t!(fs::create_dir_all(&dst));
 598         let cp_debugger_script = |file: &str| {
 599             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 600         };
 601         if host.contains("windows-msvc") {
 602             // windbg debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 604                 0o755);
 605
 606             cp_debugger_script("natvis/intrinsic.natvis");
 607             cp_debugger_script("natvis/liballoc.natvis");
 608             cp_debugger_script("natvis/libcore.natvis");
 609         } else {
 610             cp_debugger_script("debugger_pretty_printers_common.py");
 611
 612             // gdb debugger scripts
 613             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 614                     0o755);
 615             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 616                     0o755);
 617
 618             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 619             cp_debugger_script("gdb_rust_pretty_printing.py");
 620
 621             // lldb debugger scripts
 622             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 623                     0o755);
 624
 625             cp_debugger_script("lldb_rust_formatters.py");
 626         }
 627     }
 628 }
 629
 630 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 631 pub struct Std {
 632     pub compiler: Compiler,
 633     pub target: Interned<String>,
 634 }
 635
 636 impl Step for Std {
 637     type Output = PathBuf;
 638     const DEFAULT: bool = true;
 639
 640     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 641         run.path("src/libstd")
 642     }
 643
 644     fn make_run(run: RunConfig<'_>) {
 645         run.builder.ensure(Std {
 646             compiler: run.builder.compiler_for(
 647                 run.builder.top_stage,
 648                 run.builder.config.build,
 649                 run.target,
 650             ),
 651             target: run.target,
 652         });
 653     }
 654
 655     fn run(self, builder: &Builder<'_>) -> PathBuf {
 656         let compiler = self.compiler;
 657         let target = self.target;
 658
 659         let name = pkgname(builder, "rust-std");
 660         builder.info(&format!("Dist std stage{} ({} -> {})",
 661             compiler.stage, &compiler.host, target));
 662
 663         // The only true set of target libraries came from the build triple, so
 664         // let's reduce redundant work by only producing archives from that host.
 665         if compiler.host != builder.config.build {
 666             builder.info("\tskipping, not a build host");
 667             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 668         }
 669
 670         // We want to package up as many target libraries as possible
 671         // for the `rust-std` package, so if this is a host target we
 672         // depend on librustc and otherwise we just depend on libtest.
 673         if builder.hosts.iter().any(|t| t == target) {
 674             builder.ensure(compile::Rustc { compiler, target });
 675         } else {
 676             if builder.no_std(target) == Some(true) {
 677                 // the `test` doesn't compile for no-std targets
 678                 builder.ensure(compile::Std { compiler, target });
 679             } else {
 680                 builder.ensure(compile::Test { compiler, target });
 681             }
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             // Find the actual compiler (handling the full bootstrap option) which
 741             // produced the save-analysis data because that data isn't copied
 742             // through the sysroot uplifting.
 743             compiler: run.builder.compiler_for(
 744                 run.builder.top_stage,
 745                 run.builder.config.build,
 746                 run.target,
 747             ),
 748             target: run.target,
 749         });
 750     }
 751
 752     /// Creates a tarball of save-analysis metadata, if available.
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756         assert!(builder.config.extended);
 757         builder.info("Dist analysis");
 758         let name = pkgname(builder, "rust-analysis");
 759
 760         if &compiler.host != builder.config.build {
 761             builder.info("\tskipping, not a build host");
 762             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 763         }
 764
 765         builder.ensure(Std { compiler, target });
 766
 767         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 768
 769         let src = builder.stage_out(compiler, Mode::Std)
 770             .join(target).join(builder.cargo_dir()).join("deps");
 771
 772         let image_src = src.join("save-analysis");
 773         let dst = image.join("lib/rustlib").join(target).join("analysis");
 774         t!(fs::create_dir_all(&dst));
 775         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 776         builder.cp_r(&image_src, &dst);
 777
 778         let mut cmd = rust_installer(builder);
 779         cmd.arg("generate")
 780            .arg("--product-name=Rust")
 781            .arg("--rel-manifest-dir=rustlib")
 782            .arg("--success-message=save-analysis-saved.")
 783            .arg("--image-dir").arg(&image)
 784            .arg("--work-dir").arg(&tmpdir(builder))
 785            .arg("--output-dir").arg(&distdir(builder))
 786            .arg(format!("--package-name={}-{}", name, target))
 787            .arg(format!("--component-name=rust-analysis-{}", target))
 788            .arg("--legacy-manifest-dirs=rustlib,cargo");
 789         builder.run(&mut cmd);
 790         builder.remove_dir(&image);
 791         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 792     }
 793 }
 794
 795 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 796     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 797         let spath = match path.to_str() {
 798             Some(path) => path,
 799             None => return false,
 800         };
 801         if spath.ends_with("~") || spath.ends_with(".pyc") {
 802             return false
 803         }
 804
 805         const LLVM_PROJECTS: &[&str] = &[
 806             "llvm-project/clang", "llvm-project\\clang",
 807             "llvm-project/libunwind", "llvm-project\\libunwind",
 808             "llvm-project/lld", "llvm-project\\lld",
 809             "llvm-project/lldb", "llvm-project\\lldb",
 810             "llvm-project/llvm", "llvm-project\\llvm",
 811         ];
 812         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 813             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 814         {
 815             return false;
 816         }
 817
 818         const LLVM_TEST: &[&str] = &[
 819             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 820             "llvm-emscripten/test", "llvm-emscripten\\test",
 821         ];
 822         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 823             (spath.ends_with(".ll") ||
 824              spath.ends_with(".td") ||
 825              spath.ends_with(".s")) {
 826             return false
 827         }
 828         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 829             return false
 830         }
 831
 832         let full_path = Path::new(dir).join(path);
 833         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 834             return false;
 835         }
 836
 837         let excludes = [
 838             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 839             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 840             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 841             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 842         ];
 843         !path.iter()
 844              .map(|s| s.to_str().unwrap())
 845              .any(|s| excludes.contains(&s))
 846     }
 847
 848     // Copy the directories using our filter
 849     for item in src_dirs {
 850         let dst = &dst_dir.join(item);
 851         t!(fs::create_dir_all(dst));
 852         builder.cp_filtered(
 853             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 854     }
 855 }
 856
 857 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 858 pub struct Src;
 859
 860 impl Step for Src {
 861     /// The output path of the src installer tarball
 862     type Output = PathBuf;
 863     const DEFAULT: bool = true;
 864     const ONLY_HOSTS: bool = true;
 865
 866     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 867         run.path("src")
 868     }
 869
 870     fn make_run(run: RunConfig<'_>) {
 871         run.builder.ensure(Src);
 872     }
 873
 874     /// Creates the `rust-src` installer component
 875     fn run(self, builder: &Builder<'_>) -> PathBuf {
 876         builder.info("Dist src");
 877
 878         let name = pkgname(builder, "rust-src");
 879         let image = tmpdir(builder).join(format!("{}-image", name));
 880         let _ = fs::remove_dir_all(&image);
 881
 882         let dst = image.join("lib/rustlib/src");
 883         let dst_src = dst.join("rust");
 884         t!(fs::create_dir_all(&dst_src));
 885
 886         let src_files = [
 887             "Cargo.lock",
 888         ];
 889         // This is the reduced set of paths which will become the rust-src component
 890         // (essentially libstd and all of its path dependencies)
 891         let std_src_dirs = [
 892             "src/build_helper",
 893             "src/liballoc",
 894             "src/libcore",
 895             "src/libpanic_abort",
 896             "src/libpanic_unwind",
 897             "src/librustc_asan",
 898             "src/librustc_lsan",
 899             "src/librustc_msan",
 900             "src/librustc_tsan",
 901             "src/libstd",
 902             "src/libunwind",
 903             "src/libtest",
 904             "src/libterm",
 905             "src/libprofiler_builtins",
 906             "src/stdsimd",
 907             "src/libproc_macro",
 908             "src/tools/rustc-std-workspace-core",
 909             "src/tools/rustc-std-workspace-alloc",
 910             "src/librustc",
 911             "src/libsyntax",
 912         ];
 913
 914         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 915         for file in src_files.iter() {
 916             builder.copy(&builder.src.join(file), &dst_src.join(file));
 917         }
 918
 919         // Create source tarball in rust-installer format
 920         let mut cmd = rust_installer(builder);
 921         cmd.arg("generate")
 922            .arg("--product-name=Rust")
 923            .arg("--rel-manifest-dir=rustlib")
 924            .arg("--success-message=Awesome-Source.")
 925            .arg("--image-dir").arg(&image)
 926            .arg("--work-dir").arg(&tmpdir(builder))
 927            .arg("--output-dir").arg(&distdir(builder))
 928            .arg(format!("--package-name={}", name))
 929            .arg("--component-name=rust-src")
 930            .arg("--legacy-manifest-dirs=rustlib,cargo");
 931         builder.run(&mut cmd);
 932
 933         builder.remove_dir(&image);
 934         distdir(builder).join(&format!("{}.tar.gz", name))
 935     }
 936 }
 937
 938 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 939
 940 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 941 pub struct PlainSourceTarball;
 942
 943 impl Step for PlainSourceTarball {
 944     /// Produces the location of the tarball generated
 945     type Output = PathBuf;
 946     const DEFAULT: bool = true;
 947     const ONLY_HOSTS: bool = true;
 948
 949     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 950         let builder = run.builder;
 951         run.path("src").default_condition(builder.config.rust_dist_src)
 952     }
 953
 954     fn make_run(run: RunConfig<'_>) {
 955         run.builder.ensure(PlainSourceTarball);
 956     }
 957
 958     /// Creates the plain source tarball
 959     fn run(self, builder: &Builder<'_>) -> PathBuf {
 960         builder.info("Create plain source tarball");
 961
 962         // Make sure that the root folder of tarball has the correct name
 963         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 964         let plain_dst_src = tmpdir(builder).join(&plain_name);
 965         let _ = fs::remove_dir_all(&plain_dst_src);
 966         t!(fs::create_dir_all(&plain_dst_src));
 967
 968         // This is the set of root paths which will become part of the source package
 969         let src_files = [
 970             "COPYRIGHT",
 971             "LICENSE-APACHE",
 972             "LICENSE-MIT",
 973             "CONTRIBUTING.md",
 974             "README.md",
 975             "RELEASES.md",
 976             "configure",
 977             "x.py",
 978             "config.toml.example",
 979             "Cargo.toml",
 980             "Cargo.lock",
 981         ];
 982         let src_dirs = [
 983             "src",
 984         ];
 985
 986         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 987
 988         // Copy the files normally
 989         for item in &src_files {
 990             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 991         }
 992
 993         // Create the version file
 994         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 995         if let Some(sha) = builder.rust_sha() {
 996             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 997         }
 998
 999         // If we're building from git sources, we need to vendor a complete distribution.
1000         if builder.rust_info.is_git() {
1001             // Get cargo-vendor installed, if it isn't already.
1002             let mut has_cargo_vendor = false;
1003             let mut cmd = Command::new(&builder.initial_cargo);
1004             for line in output(cmd.arg("install").arg("--list")).lines() {
1005                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
1006             }
1007             if !has_cargo_vendor {
1008                 let mut cmd = builder.cargo(
1009                     builder.compiler(0, builder.config.build),
1010                     Mode::ToolBootstrap,
1011                     builder.config.build,
1012                     "install"
1013                 );
1014                 cmd.arg("--force")
1015                    .arg("--debug")
1016                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1017                    .arg("cargo-vendor");
1018                 builder.run(&mut cmd);
1019             }
1020
1021             // Vendor all Cargo dependencies
1022             let mut cmd = Command::new(&builder.initial_cargo);
1023             cmd.arg("vendor")
1024                .current_dir(&plain_dst_src);
1025             builder.run(&mut cmd);
1026         }
1027
1028         // Create plain source tarball
1029         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1030         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1031         tarball.set_extension(""); // strip .gz
1032         tarball.set_extension(""); // strip .tar
1033         if let Some(dir) = tarball.parent() {
1034             builder.create_dir(&dir);
1035         }
1036         builder.info("running installer");
1037         let mut cmd = rust_installer(builder);
1038         cmd.arg("tarball")
1039            .arg("--input").arg(&plain_name)
1040            .arg("--output").arg(&tarball)
1041            .arg("--work-dir=.")
1042            .current_dir(tmpdir(builder));
1043         builder.run(&mut cmd);
1044         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1045     }
1046 }
1047
1048 // We have to run a few shell scripts, which choke quite a bit on both `\`
1049 // characters and on `C:\` paths, so normalize both of them away.
1050 pub fn sanitize_sh(path: &Path) -> String {
1051     let path = path.to_str().unwrap().replace("\\", "/");
1052     return change_drive(&path).unwrap_or(path);
1053
1054     fn change_drive(s: &str) -> Option<String> {
1055         let mut ch = s.chars();
1056         let drive = ch.next().unwrap_or('C');
1057         if ch.next() != Some(':') {
1058             return None
1059         }
1060         if ch.next() != Some('/') {
1061             return None
1062         }
1063         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1064     }
1065 }
1066
1067 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1068 pub struct Cargo {
1069     pub compiler: Compiler,
1070     pub target: Interned<String>,
1071 }
1072
1073 impl Step for Cargo {
1074     type Output = PathBuf;
1075     const ONLY_HOSTS: bool = true;
1076
1077     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1078         run.path("cargo")
1079     }
1080
1081     fn make_run(run: RunConfig<'_>) {
1082         run.builder.ensure(Cargo {
1083             compiler: run.builder.compiler_for(
1084                 run.builder.top_stage,
1085                 run.builder.config.build,
1086                 run.target,
1087             ),
1088             target: run.target,
1089         });
1090     }
1091
1092     fn run(self, builder: &Builder<'_>) -> PathBuf {
1093         let compiler = self.compiler;
1094         let target = self.target;
1095
1096         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1097         let src = builder.src.join("src/tools/cargo");
1098         let etc = src.join("src/etc");
1099         let release_num = builder.release_num("cargo");
1100         let name = pkgname(builder, "cargo");
1101         let version = builder.cargo_info.version(builder, &release_num);
1102
1103         let tmp = tmpdir(builder);
1104         let image = tmp.join("cargo-image");
1105         drop(fs::remove_dir_all(&image));
1106         builder.create_dir(&image);
1107
1108         // Prepare the image directory
1109         builder.create_dir(&image.join("share/zsh/site-functions"));
1110         builder.create_dir(&image.join("etc/bash_completion.d"));
1111         let cargo = builder.ensure(tool::Cargo { compiler, target });
1112         builder.install(&cargo, &image.join("bin"), 0o755);
1113         for man in t!(etc.join("man").read_dir()) {
1114             let man = t!(man);
1115             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1116         }
1117         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1118         builder.copy(&etc.join("cargo.bashcomp.sh"),
1119              &image.join("etc/bash_completion.d/cargo"));
1120         let doc = image.join("share/doc/cargo");
1121         builder.install(&src.join("README.md"), &doc, 0o644);
1122         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1123         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1124         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1125
1126         // Prepare the overlay
1127         let overlay = tmp.join("cargo-overlay");
1128         drop(fs::remove_dir_all(&overlay));
1129         builder.create_dir(&overlay);
1130         builder.install(&src.join("README.md"), &overlay, 0o644);
1131         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1132         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1133         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1134         builder.create(&overlay.join("version"), &version);
1135
1136         // Generate the installer tarball
1137         let mut cmd = rust_installer(builder);
1138         cmd.arg("generate")
1139            .arg("--product-name=Rust")
1140            .arg("--rel-manifest-dir=rustlib")
1141            .arg("--success-message=Rust-is-ready-to-roll.")
1142            .arg("--image-dir").arg(&image)
1143            .arg("--work-dir").arg(&tmpdir(builder))
1144            .arg("--output-dir").arg(&distdir(builder))
1145            .arg("--non-installed-overlay").arg(&overlay)
1146            .arg(format!("--package-name={}-{}", name, target))
1147            .arg("--component-name=cargo")
1148            .arg("--legacy-manifest-dirs=rustlib,cargo");
1149         builder.run(&mut cmd);
1150         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1151     }
1152 }
1153
1154 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1155 pub struct Rls {
1156     pub compiler: Compiler,
1157     pub target: Interned<String>,
1158 }
1159
1160 impl Step for Rls {
1161     type Output = Option<PathBuf>;
1162     const ONLY_HOSTS: bool = true;
1163
1164     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1165         run.path("rls")
1166     }
1167
1168     fn make_run(run: RunConfig<'_>) {
1169         run.builder.ensure(Rls {
1170             compiler: run.builder.compiler_for(
1171                 run.builder.top_stage,
1172                 run.builder.config.build,
1173                 run.target,
1174             ),
1175             target: run.target,
1176         });
1177     }
1178
1179     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1180         let compiler = self.compiler;
1181         let target = self.target;
1182         assert!(builder.config.extended);
1183
1184         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1185         let src = builder.src.join("src/tools/rls");
1186         let release_num = builder.release_num("rls");
1187         let name = pkgname(builder, "rls");
1188         let version = builder.rls_info.version(builder, &release_num);
1189
1190         let tmp = tmpdir(builder);
1191         let image = tmp.join("rls-image");
1192         drop(fs::remove_dir_all(&image));
1193         t!(fs::create_dir_all(&image));
1194
1195         // Prepare the image directory
1196         // We expect RLS to build, because we've exited this step above if tool
1197         // state for RLS isn't testing.
1198         let rls = builder.ensure(tool::Rls {
1199             compiler,
1200             target,
1201             extra_features: Vec::new(),
1202         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1203
1204         builder.install(&rls, &image.join("bin"), 0o755);
1205         let doc = image.join("share/doc/rls");
1206         builder.install(&src.join("README.md"), &doc, 0o644);
1207         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1208         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1209
1210         // Prepare the overlay
1211         let overlay = tmp.join("rls-overlay");
1212         drop(fs::remove_dir_all(&overlay));
1213         t!(fs::create_dir_all(&overlay));
1214         builder.install(&src.join("README.md"), &overlay, 0o644);
1215         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1216         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1217         builder.create(&overlay.join("version"), &version);
1218
1219         // Generate the installer tarball
1220         let mut cmd = rust_installer(builder);
1221         cmd.arg("generate")
1222            .arg("--product-name=Rust")
1223            .arg("--rel-manifest-dir=rustlib")
1224            .arg("--success-message=RLS-ready-to-serve.")
1225            .arg("--image-dir").arg(&image)
1226            .arg("--work-dir").arg(&tmpdir(builder))
1227            .arg("--output-dir").arg(&distdir(builder))
1228            .arg("--non-installed-overlay").arg(&overlay)
1229            .arg(format!("--package-name={}-{}", name, target))
1230            .arg("--legacy-manifest-dirs=rustlib,cargo")
1231            .arg("--component-name=rls-preview");
1232
1233         builder.run(&mut cmd);
1234         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1235     }
1236 }
1237
1238 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1239 pub struct Clippy {
1240     pub compiler: Compiler,
1241     pub target: Interned<String>,
1242 }
1243
1244 impl Step for Clippy {
1245     type Output = Option<PathBuf>;
1246     const ONLY_HOSTS: bool = true;
1247
1248     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1249         run.path("clippy")
1250     }
1251
1252     fn make_run(run: RunConfig<'_>) {
1253         run.builder.ensure(Clippy {
1254             compiler: run.builder.compiler_for(
1255                 run.builder.top_stage,
1256                 run.builder.config.build,
1257                 run.target,
1258             ),
1259             target: run.target,
1260         });
1261     }
1262
1263     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1264         let compiler = self.compiler;
1265         let target = self.target;
1266         assert!(builder.config.extended);
1267
1268         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1269         let src = builder.src.join("src/tools/clippy");
1270         let release_num = builder.release_num("clippy");
1271         let name = pkgname(builder, "clippy");
1272         let version = builder.clippy_info.version(builder, &release_num);
1273
1274         let tmp = tmpdir(builder);
1275         let image = tmp.join("clippy-image");
1276         drop(fs::remove_dir_all(&image));
1277         builder.create_dir(&image);
1278
1279         // Prepare the image directory
1280         // We expect clippy to build, because we've exited this step above if tool
1281         // state for clippy isn't testing.
1282         let clippy = builder.ensure(tool::Clippy {
1283             compiler,
1284             target,
1285             extra_features: Vec::new(),
1286         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1287         let cargoclippy = builder.ensure(tool::CargoClippy {
1288             compiler,
1289             target, extra_features: Vec::new()
1290         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1291
1292         builder.install(&clippy, &image.join("bin"), 0o755);
1293         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1294         let doc = image.join("share/doc/clippy");
1295         builder.install(&src.join("README.md"), &doc, 0o644);
1296         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1297         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1298
1299         // Prepare the overlay
1300         let overlay = tmp.join("clippy-overlay");
1301         drop(fs::remove_dir_all(&overlay));
1302         t!(fs::create_dir_all(&overlay));
1303         builder.install(&src.join("README.md"), &overlay, 0o644);
1304         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1305         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1306         builder.create(&overlay.join("version"), &version);
1307
1308         // Generate the installer tarball
1309         let mut cmd = rust_installer(builder);
1310         cmd.arg("generate")
1311            .arg("--product-name=Rust")
1312            .arg("--rel-manifest-dir=rustlib")
1313            .arg("--success-message=clippy-ready-to-serve.")
1314            .arg("--image-dir").arg(&image)
1315            .arg("--work-dir").arg(&tmpdir(builder))
1316            .arg("--output-dir").arg(&distdir(builder))
1317            .arg("--non-installed-overlay").arg(&overlay)
1318            .arg(format!("--package-name={}-{}", name, target))
1319            .arg("--legacy-manifest-dirs=rustlib,cargo")
1320            .arg("--component-name=clippy-preview");
1321
1322         builder.run(&mut cmd);
1323         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1324     }
1325 }
1326
1327 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1328 pub struct Miri {
1329     pub compiler: Compiler,
1330     pub target: Interned<String>,
1331 }
1332
1333 impl Step for Miri {
1334     type Output = Option<PathBuf>;
1335     const ONLY_HOSTS: bool = true;
1336
1337     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1338         run.path("miri")
1339     }
1340
1341     fn make_run(run: RunConfig<'_>) {
1342         run.builder.ensure(Miri {
1343             compiler: run.builder.compiler_for(
1344                 run.builder.top_stage,
1345                 run.builder.config.build,
1346                 run.target,
1347             ),
1348             target: run.target,
1349         });
1350     }
1351
1352     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1353         let compiler = self.compiler;
1354         let target = self.target;
1355         assert!(builder.config.extended);
1356
1357         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1358         let src = builder.src.join("src/tools/miri");
1359         let release_num = builder.release_num("miri");
1360         let name = pkgname(builder, "miri");
1361         let version = builder.miri_info.version(builder, &release_num);
1362
1363         let tmp = tmpdir(builder);
1364         let image = tmp.join("miri-image");
1365         drop(fs::remove_dir_all(&image));
1366         builder.create_dir(&image);
1367
1368         // Prepare the image directory
1369         // We expect miri to build, because we've exited this step above if tool
1370         // state for miri isn't testing.
1371         let miri = builder.ensure(tool::Miri {
1372             compiler,
1373             target,
1374             extra_features: Vec::new(),
1375         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1376         let cargomiri = builder.ensure(tool::CargoMiri {
1377             compiler,
1378             target,
1379             extra_features: Vec::new()
1380         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1381
1382         builder.install(&miri, &image.join("bin"), 0o755);
1383         builder.install(&cargomiri, &image.join("bin"), 0o755);
1384         let doc = image.join("share/doc/miri");
1385         builder.install(&src.join("README.md"), &doc, 0o644);
1386         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1387         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1388
1389         // Prepare the overlay
1390         let overlay = tmp.join("miri-overlay");
1391         drop(fs::remove_dir_all(&overlay));
1392         t!(fs::create_dir_all(&overlay));
1393         builder.install(&src.join("README.md"), &overlay, 0o644);
1394         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1395         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1396         builder.create(&overlay.join("version"), &version);
1397
1398         // Generate the installer tarball
1399         let mut cmd = rust_installer(builder);
1400         cmd.arg("generate")
1401            .arg("--product-name=Rust")
1402            .arg("--rel-manifest-dir=rustlib")
1403            .arg("--success-message=miri-ready-to-serve.")
1404            .arg("--image-dir").arg(&image)
1405            .arg("--work-dir").arg(&tmpdir(builder))
1406            .arg("--output-dir").arg(&distdir(builder))
1407            .arg("--non-installed-overlay").arg(&overlay)
1408            .arg(format!("--package-name={}-{}", name, target))
1409            .arg("--legacy-manifest-dirs=rustlib,cargo")
1410            .arg("--component-name=miri-preview");
1411
1412         builder.run(&mut cmd);
1413         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1414     }
1415 }
1416
1417 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1418 pub struct Rustfmt {
1419     pub compiler: Compiler,
1420     pub target: Interned<String>,
1421 }
1422
1423 impl Step for Rustfmt {
1424     type Output = Option<PathBuf>;
1425     const ONLY_HOSTS: bool = true;
1426
1427     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1428         run.path("rustfmt")
1429     }
1430
1431     fn make_run(run: RunConfig<'_>) {
1432         run.builder.ensure(Rustfmt {
1433             compiler: run.builder.compiler_for(
1434                 run.builder.top_stage,
1435                 run.builder.config.build,
1436                 run.target,
1437             ),
1438             target: run.target,
1439         });
1440     }
1441
1442     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1443         let compiler = self.compiler;
1444         let target = self.target;
1445
1446         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1447         let src = builder.src.join("src/tools/rustfmt");
1448         let release_num = builder.release_num("rustfmt");
1449         let name = pkgname(builder, "rustfmt");
1450         let version = builder.rustfmt_info.version(builder, &release_num);
1451
1452         let tmp = tmpdir(builder);
1453         let image = tmp.join("rustfmt-image");
1454         drop(fs::remove_dir_all(&image));
1455         builder.create_dir(&image);
1456
1457         // Prepare the image directory
1458         let rustfmt = builder.ensure(tool::Rustfmt {
1459             compiler,
1460             target,
1461             extra_features: Vec::new(),
1462         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1463         let cargofmt = builder.ensure(tool::Cargofmt {
1464             compiler,
1465             target,
1466             extra_features: Vec::new(),
1467         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1468
1469         builder.install(&rustfmt, &image.join("bin"), 0o755);
1470         builder.install(&cargofmt, &image.join("bin"), 0o755);
1471         let doc = image.join("share/doc/rustfmt");
1472         builder.install(&src.join("README.md"), &doc, 0o644);
1473         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1474         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1475
1476         // Prepare the overlay
1477         let overlay = tmp.join("rustfmt-overlay");
1478         drop(fs::remove_dir_all(&overlay));
1479         builder.create_dir(&overlay);
1480         builder.install(&src.join("README.md"), &overlay, 0o644);
1481         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1482         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1483         builder.create(&overlay.join("version"), &version);
1484
1485         // Generate the installer tarball
1486         let mut cmd = rust_installer(builder);
1487         cmd.arg("generate")
1488            .arg("--product-name=Rust")
1489            .arg("--rel-manifest-dir=rustlib")
1490            .arg("--success-message=rustfmt-ready-to-fmt.")
1491            .arg("--image-dir").arg(&image)
1492            .arg("--work-dir").arg(&tmpdir(builder))
1493            .arg("--output-dir").arg(&distdir(builder))
1494            .arg("--non-installed-overlay").arg(&overlay)
1495            .arg(format!("--package-name={}-{}", name, target))
1496            .arg("--legacy-manifest-dirs=rustlib,cargo")
1497            .arg("--component-name=rustfmt-preview");
1498
1499         builder.run(&mut cmd);
1500         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1501     }
1502 }
1503
1504 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1505 pub struct Extended {
1506     stage: u32,
1507     host: Interned<String>,
1508     target: Interned<String>,
1509 }
1510
1511 impl Step for Extended {
1512     type Output = ();
1513     const DEFAULT: bool = true;
1514     const ONLY_HOSTS: bool = true;
1515
1516     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1517         let builder = run.builder;
1518         run.path("extended").default_condition(builder.config.extended)
1519     }
1520
1521     fn make_run(run: RunConfig<'_>) {
1522         run.builder.ensure(Extended {
1523             stage: run.builder.top_stage,
1524             host: run.builder.config.build,
1525             target: run.target,
1526         });
1527     }
1528
1529     /// Creates a combined installer for the specified target in the provided stage.
1530     fn run(self, builder: &Builder<'_>) {
1531         let target = self.target;
1532         let stage = self.stage;
1533         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1534
1535         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1536
1537         let rustc_installer = builder.ensure(Rustc {
1538             compiler: builder.compiler(stage, target),
1539         });
1540         let cargo_installer = builder.ensure(Cargo { compiler, target });
1541         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1542         let rls_installer = builder.ensure(Rls { compiler, target });
1543         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1544         let clippy_installer = builder.ensure(Clippy { compiler, target });
1545         let miri_installer = builder.ensure(Miri { compiler, target });
1546         let lldb_installer = builder.ensure(Lldb { target });
1547         let mingw_installer = builder.ensure(Mingw { host: target });
1548         let analysis_installer = builder.ensure(Analysis { compiler, target });
1549
1550         let docs_installer = builder.ensure(Docs { host: target, });
1551         let std_installer = builder.ensure(Std {
1552             compiler: builder.compiler(stage, target),
1553             target,
1554         });
1555
1556         let tmp = tmpdir(builder);
1557         let overlay = tmp.join("extended-overlay");
1558         let etc = builder.src.join("src/etc/installer");
1559         let work = tmp.join("work");
1560
1561         let _ = fs::remove_dir_all(&overlay);
1562         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1563         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1564         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1565         let version = builder.rust_version();
1566         builder.create(&overlay.join("version"), &version);
1567         if let Some(sha) = builder.rust_sha() {
1568             builder.create(&overlay.join("git-commit-hash"), &sha);
1569         }
1570         builder.install(&etc.join("README.md"), &overlay, 0o644);
1571
1572         // When rust-std package split from rustc, we needed to ensure that during
1573         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1574         // the std files during uninstall. To do this ensure that rustc comes
1575         // before rust-std in the list below.
1576         let mut tarballs = Vec::new();
1577         tarballs.push(rustc_installer);
1578         tarballs.push(cargo_installer);
1579         tarballs.extend(rls_installer.clone());
1580         tarballs.extend(clippy_installer.clone());
1581         tarballs.extend(miri_installer.clone());
1582         tarballs.extend(rustfmt_installer.clone());
1583         tarballs.extend(llvm_tools_installer);
1584         tarballs.extend(lldb_installer);
1585         tarballs.push(analysis_installer);
1586         tarballs.push(std_installer);
1587         if builder.config.docs {
1588             tarballs.push(docs_installer);
1589         }
1590         if target.contains("pc-windows-gnu") {
1591             tarballs.push(mingw_installer.unwrap());
1592         }
1593         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1594         for tarball in &tarballs[1..] {
1595             input_tarballs.push(",");
1596             input_tarballs.push(tarball);
1597         }
1598
1599         let mut cmd = rust_installer(builder);
1600         cmd.arg("combine")
1601             .arg("--product-name=Rust")
1602             .arg("--rel-manifest-dir=rustlib")
1603             .arg("--success-message=Rust-is-ready-to-roll.")
1604             .arg("--work-dir").arg(&work)
1605             .arg("--output-dir").arg(&distdir(builder))
1606             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1607             .arg("--legacy-manifest-dirs=rustlib,cargo")
1608             .arg("--input-tarballs").arg(input_tarballs)
1609             .arg("--non-installed-overlay").arg(&overlay);
1610         builder.run(&mut cmd);
1611
1612         let mut license = String::new();
1613         license += &builder.read(&builder.src.join("COPYRIGHT"));
1614         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1615         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1616         license.push_str("\n");
1617         license.push_str("\n");
1618
1619         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1620         let mut rtf = rtf.to_string();
1621         rtf.push_str("\n");
1622         for line in license.lines() {
1623             rtf.push_str(line);
1624             rtf.push_str("\\line ");
1625         }
1626         rtf.push_str("}");
1627
1628         fn filter(contents: &str, marker: &str) -> String {
1629             let start = format!("tool-{}-start", marker);
1630             let end = format!("tool-{}-end", marker);
1631             let mut lines = Vec::new();
1632             let mut omitted = false;
1633             for line in contents.lines() {
1634                 if line.contains(&start) {
1635                     omitted = true;
1636                 } else if line.contains(&end) {
1637                     omitted = false;
1638                 } else if !omitted {
1639                     lines.push(line);
1640                 }
1641             }
1642
1643             lines.join("\n")
1644         }
1645
1646         let xform = |p: &Path| {
1647             let mut contents = t!(fs::read_to_string(p));
1648             if rls_installer.is_none() {
1649                 contents = filter(&contents, "rls");
1650             }
1651             if clippy_installer.is_none() {
1652                 contents = filter(&contents, "clippy");
1653             }
1654             if miri_installer.is_none() {
1655                 contents = filter(&contents, "miri");
1656             }
1657             if rustfmt_installer.is_none() {
1658                 contents = filter(&contents, "rustfmt");
1659             }
1660             let ret = tmp.join(p.file_name().unwrap());
1661             t!(fs::write(&ret, &contents));
1662             ret
1663         };
1664
1665         if target.contains("apple-darwin") {
1666             let pkg = tmp.join("pkg");
1667             let _ = fs::remove_dir_all(&pkg);
1668
1669             let pkgbuild = |component: &str| {
1670                 let mut cmd = Command::new("pkgbuild");
1671                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1672                     .arg("--scripts").arg(pkg.join(component))
1673                     .arg("--nopayload")
1674                     .arg(pkg.join(component).with_extension("pkg"));
1675                 builder.run(&mut cmd);
1676             };
1677
1678             let prepare = |name: &str| {
1679                 builder.create_dir(&pkg.join(name));
1680                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1681                         &pkg.join(name));
1682                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1683                 pkgbuild(name);
1684             };
1685             prepare("rustc");
1686             prepare("cargo");
1687             prepare("rust-docs");
1688             prepare("rust-std");
1689             prepare("rust-analysis");
1690
1691             if rls_installer.is_some() {
1692                 prepare("rls");
1693             }
1694             if clippy_installer.is_some() {
1695                 prepare("clippy");
1696             }
1697             if miri_installer.is_some() {
1698                 prepare("miri");
1699             }
1700
1701             // create an 'uninstall' package
1702             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1703             pkgbuild("uninstall");
1704
1705             builder.create_dir(&pkg.join("res"));
1706             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1707             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1708             let mut cmd = Command::new("productbuild");
1709             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1710                 .arg("--resources").arg(pkg.join("res"))
1711                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1712                                                     pkgname(builder, "rust"),
1713                                                     target)))
1714                 .arg("--package-path").arg(&pkg);
1715             builder.run(&mut cmd);
1716         }
1717
1718         if target.contains("windows") {
1719             let exe = tmp.join("exe");
1720             let _ = fs::remove_dir_all(&exe);
1721
1722             let prepare = |name: &str| {
1723                 builder.create_dir(&exe.join(name));
1724                 let dir = if name == "rust-std" || name == "rust-analysis" {
1725                     format!("{}-{}", name, target)
1726                 } else if name == "rls" {
1727                     "rls-preview".to_string()
1728                 } else if name == "clippy" {
1729                     "clippy-preview".to_string()
1730                 } else if name == "miri" {
1731                     "miri-preview".to_string()
1732                 } else {
1733                     name.to_string()
1734                 };
1735                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1736                             .join(dir),
1737                         &exe.join(name));
1738                 builder.remove(&exe.join(name).join("manifest.in"));
1739             };
1740             prepare("rustc");
1741             prepare("cargo");
1742             prepare("rust-analysis");
1743             prepare("rust-docs");
1744             prepare("rust-std");
1745             if rls_installer.is_some() {
1746                 prepare("rls");
1747             }
1748             if clippy_installer.is_some() {
1749                 prepare("clippy");
1750             }
1751             if miri_installer.is_some() {
1752                 prepare("miri");
1753             }
1754             if target.contains("windows-gnu") {
1755                 prepare("rust-mingw");
1756             }
1757
1758             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1759             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1760             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1761             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1762             builder.create(&exe.join("LICENSE.txt"), &license);
1763
1764             // Generate exe installer
1765             let mut cmd = Command::new("iscc");
1766             cmd.arg("rust.iss")
1767                 .current_dir(&exe);
1768             if target.contains("windows-gnu") {
1769                 cmd.arg("/dMINGW");
1770             }
1771             add_env(builder, &mut cmd, target);
1772             builder.run(&mut cmd);
1773             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1774                     &distdir(builder),
1775                     0o755);
1776
1777             // Generate msi installer
1778             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1779             let heat = wix.join("bin/heat.exe");
1780             let candle = wix.join("bin/candle.exe");
1781             let light = wix.join("bin/light.exe");
1782
1783             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1784             builder.run(Command::new(&heat)
1785                             .current_dir(&exe)
1786                             .arg("dir")
1787                             .arg("rustc")
1788                             .args(&heat_flags)
1789                             .arg("-cg").arg("RustcGroup")
1790                             .arg("-dr").arg("Rustc")
1791                             .arg("-var").arg("var.RustcDir")
1792                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1793             builder.run(Command::new(&heat)
1794                             .current_dir(&exe)
1795                             .arg("dir")
1796                             .arg("rust-docs")
1797                             .args(&heat_flags)
1798                             .arg("-cg").arg("DocsGroup")
1799                             .arg("-dr").arg("Docs")
1800                             .arg("-var").arg("var.DocsDir")
1801                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1802                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1803             builder.run(Command::new(&heat)
1804                             .current_dir(&exe)
1805                             .arg("dir")
1806                             .arg("cargo")
1807                             .args(&heat_flags)
1808                             .arg("-cg").arg("CargoGroup")
1809                             .arg("-dr").arg("Cargo")
1810                             .arg("-var").arg("var.CargoDir")
1811                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1812                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1813             builder.run(Command::new(&heat)
1814                             .current_dir(&exe)
1815                             .arg("dir")
1816                             .arg("rust-std")
1817                             .args(&heat_flags)
1818                             .arg("-cg").arg("StdGroup")
1819                             .arg("-dr").arg("Std")
1820                             .arg("-var").arg("var.StdDir")
1821                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1822             if rls_installer.is_some() {
1823                 builder.run(Command::new(&heat)
1824                                 .current_dir(&exe)
1825                                 .arg("dir")
1826                                 .arg("rls")
1827                                 .args(&heat_flags)
1828                                 .arg("-cg").arg("RlsGroup")
1829                                 .arg("-dr").arg("Rls")
1830                                 .arg("-var").arg("var.RlsDir")
1831                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1832                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1833             }
1834             if clippy_installer.is_some() {
1835                 builder.run(Command::new(&heat)
1836                                 .current_dir(&exe)
1837                                 .arg("dir")
1838                                 .arg("clippy")
1839                                 .args(&heat_flags)
1840                                 .arg("-cg").arg("ClippyGroup")
1841                                 .arg("-dr").arg("Clippy")
1842                                 .arg("-var").arg("var.ClippyDir")
1843                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1844                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1845             }
1846             if miri_installer.is_some() {
1847                 builder.run(Command::new(&heat)
1848                                 .current_dir(&exe)
1849                                 .arg("dir")
1850                                 .arg("miri")
1851                                 .args(&heat_flags)
1852                                 .arg("-cg").arg("MiriGroup")
1853                                 .arg("-dr").arg("Miri")
1854                                 .arg("-var").arg("var.MiriDir")
1855                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1856                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1857             }
1858             builder.run(Command::new(&heat)
1859                             .current_dir(&exe)
1860                             .arg("dir")
1861                             .arg("rust-analysis")
1862                             .args(&heat_flags)
1863                             .arg("-cg").arg("AnalysisGroup")
1864                             .arg("-dr").arg("Analysis")
1865                             .arg("-var").arg("var.AnalysisDir")
1866                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1867                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1868             if target.contains("windows-gnu") {
1869                 builder.run(Command::new(&heat)
1870                                 .current_dir(&exe)
1871                                 .arg("dir")
1872                                 .arg("rust-mingw")
1873                                 .args(&heat_flags)
1874                                 .arg("-cg").arg("GccGroup")
1875                                 .arg("-dr").arg("Gcc")
1876                                 .arg("-var").arg("var.GccDir")
1877                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1878             }
1879
1880             let candle = |input: &Path| {
1881                 let output = exe.join(input.file_stem().unwrap())
1882                                 .with_extension("wixobj");
1883                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1884                 let mut cmd = Command::new(&candle);
1885                 cmd.current_dir(&exe)
1886                     .arg("-nologo")
1887                     .arg("-dRustcDir=rustc")
1888                     .arg("-dDocsDir=rust-docs")
1889                     .arg("-dCargoDir=cargo")
1890                     .arg("-dStdDir=rust-std")
1891                     .arg("-dAnalysisDir=rust-analysis")
1892                     .arg("-arch").arg(&arch)
1893                     .arg("-out").arg(&output)
1894                     .arg(&input);
1895                 add_env(builder, &mut cmd, target);
1896
1897                 if rls_installer.is_some() {
1898                     cmd.arg("-dRlsDir=rls");
1899                 }
1900                 if clippy_installer.is_some() {
1901                     cmd.arg("-dClippyDir=clippy");
1902                 }
1903                 if miri_installer.is_some() {
1904                     cmd.arg("-dMiriDir=miri");
1905                 }
1906                 if target.contains("windows-gnu") {
1907                     cmd.arg("-dGccDir=rust-mingw");
1908                 }
1909                 builder.run(&mut cmd);
1910             };
1911             candle(&xform(&etc.join("msi/rust.wxs")));
1912             candle(&etc.join("msi/ui.wxs"));
1913             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1914             candle("RustcGroup.wxs".as_ref());
1915             candle("DocsGroup.wxs".as_ref());
1916             candle("CargoGroup.wxs".as_ref());
1917             candle("StdGroup.wxs".as_ref());
1918             if rls_installer.is_some() {
1919                 candle("RlsGroup.wxs".as_ref());
1920             }
1921             if clippy_installer.is_some() {
1922                 candle("ClippyGroup.wxs".as_ref());
1923             }
1924             if miri_installer.is_some() {
1925                 candle("MiriGroup.wxs".as_ref());
1926             }
1927             candle("AnalysisGroup.wxs".as_ref());
1928
1929             if target.contains("windows-gnu") {
1930                 candle("GccGroup.wxs".as_ref());
1931             }
1932
1933             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1934             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1935             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1936
1937             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1938             let mut cmd = Command::new(&light);
1939             cmd.arg("-nologo")
1940                 .arg("-ext").arg("WixUIExtension")
1941                 .arg("-ext").arg("WixUtilExtension")
1942                 .arg("-out").arg(exe.join(&filename))
1943                 .arg("rust.wixobj")
1944                 .arg("ui.wixobj")
1945                 .arg("rustwelcomedlg.wixobj")
1946                 .arg("RustcGroup.wixobj")
1947                 .arg("DocsGroup.wixobj")
1948                 .arg("CargoGroup.wixobj")
1949                 .arg("StdGroup.wixobj")
1950                 .arg("AnalysisGroup.wixobj")
1951                 .current_dir(&exe);
1952
1953             if rls_installer.is_some() {
1954                 cmd.arg("RlsGroup.wixobj");
1955             }
1956             if clippy_installer.is_some() {
1957                 cmd.arg("ClippyGroup.wixobj");
1958             }
1959             if miri_installer.is_some() {
1960                 cmd.arg("MiriGroup.wixobj");
1961             }
1962
1963             if target.contains("windows-gnu") {
1964                 cmd.arg("GccGroup.wixobj");
1965             }
1966             // ICE57 wrongly complains about the shortcuts
1967             cmd.arg("-sice:ICE57");
1968
1969             builder.run(&mut cmd);
1970
1971             if !builder.config.dry_run {
1972                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1973             }
1974         }
1975     }
1976 }
1977
1978 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1979     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1980     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1981        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1982        .env("CFG_RELEASE", builder.rust_release())
1983        .env("CFG_VER_MAJOR", parts.next().unwrap())
1984        .env("CFG_VER_MINOR", parts.next().unwrap())
1985        .env("CFG_VER_PATCH", parts.next().unwrap())
1986        .env("CFG_VER_BUILD", "0") // just needed to build
1987        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1988        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1989        .env("CFG_BUILD", target)
1990        .env("CFG_CHANNEL", &builder.config.channel);
1991
1992     if target.contains("windows-gnu") {
1993        cmd.env("CFG_MINGW", "1")
1994           .env("CFG_ABI", "GNU");
1995     } else {
1996        cmd.env("CFG_MINGW", "0")
1997           .env("CFG_ABI", "MSVC");
1998     }
1999
2000     if target.contains("x86_64") {
2001        cmd.env("CFG_PLATFORM", "x64");
2002     } else {
2003        cmd.env("CFG_PLATFORM", "x86");
2004     }
2005 }
2006
2007 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
2008 pub struct HashSign;
2009
2010 impl Step for HashSign {
2011     type Output = ();
2012     const ONLY_HOSTS: bool = true;
2013
2014     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2015         run.path("hash-and-sign")
2016     }
2017
2018     fn make_run(run: RunConfig<'_>) {
2019         run.builder.ensure(HashSign);
2020     }
2021
2022     fn run(self, builder: &Builder<'_>) {
2023         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2024         if builder.config.dry_run {
2025             return;
2026         }
2027         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2028             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2029         });
2030         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2031             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2032         });
2033         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2034             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2035         });
2036         let pass = t!(fs::read_to_string(&file));
2037
2038         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2039
2040         cmd.arg(sign);
2041         cmd.arg(distdir(builder));
2042         cmd.arg(today.trim());
2043         cmd.arg(builder.rust_package_vers());
2044         cmd.arg(addr);
2045         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2046         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2047         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2048         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2049         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2050         cmd.arg(builder.llvm_tools_package_vers());
2051         cmd.arg(builder.lldb_package_vers());
2052
2053         builder.create_dir(&distdir(builder));
2054
2055         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2056         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2057         let status = t!(child.wait());
2058         assert!(status.success());
2059     }
2060 }
2061
2062 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2063 // LLVM tools are linked dynamically.
2064 // Note: This function does no yet support Windows but we also don't support
2065 //       linking LLVM tools dynamically on Windows yet.
2066 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2067                                 target: Interned<String>,
2068                                 sysroot: &Path) {
2069     let src_libdir = builder
2070         .llvm_out(target)
2071         .join("lib");
2072     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2073     t!(fs::create_dir_all(&dst_libdir));
2074
2075     if target.contains("apple-darwin") {
2076         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2077         if llvm_dylib_path.exists() {
2078             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2079         }
2080         return
2081     }
2082
2083     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2084     // Since tools link to the latter rather than the former, we have to
2085     // follow the symlink to find out what to distribute.
2086     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2087     if llvm_dylib_path.exists() {
2088         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2089             panic!("dist: Error calling canonicalize path `{}`: {}",
2090                    llvm_dylib_path.display(), e);
2091         });
2092
2093
2094         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2095     }
2096 }
2097
2098 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2099 pub struct LlvmTools {
2100     pub target: Interned<String>,
2101 }
2102
2103 impl Step for LlvmTools {
2104     type Output = Option<PathBuf>;
2105     const ONLY_HOSTS: bool = true;
2106
2107     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2108         run.path("llvm-tools")
2109     }
2110
2111     fn make_run(run: RunConfig<'_>) {
2112         run.builder.ensure(LlvmTools {
2113             target: run.target,
2114         });
2115     }
2116
2117     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2118         let target = self.target;
2119         assert!(builder.config.extended);
2120
2121         /* run only if llvm-config isn't used */
2122         if let Some(config) = builder.config.target_config.get(&target) {
2123             if let Some(ref _s) = config.llvm_config {
2124                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2125                     target));
2126                 return None;
2127             }
2128         }
2129
2130         builder.info(&format!("Dist LlvmTools ({})", target));
2131         let src = builder.src.join("src/llvm-project/llvm");
2132         let name = pkgname(builder, "llvm-tools");
2133
2134         let tmp = tmpdir(builder);
2135         let image = tmp.join("llvm-tools-image");
2136         drop(fs::remove_dir_all(&image));
2137
2138         // Prepare the image directory
2139         let src_bindir = builder
2140             .llvm_out(target)
2141             .join("bin");
2142         let dst_bindir = image.join("lib/rustlib")
2143             .join(&*target)
2144             .join("bin");
2145         t!(fs::create_dir_all(&dst_bindir));
2146         for tool in LLVM_TOOLS {
2147             let exe = src_bindir.join(exe(tool, &target));
2148             builder.install(&exe, &dst_bindir, 0o755);
2149         }
2150
2151         // Prepare the overlay
2152         let overlay = tmp.join("llvm-tools-overlay");
2153         drop(fs::remove_dir_all(&overlay));
2154         builder.create_dir(&overlay);
2155         builder.install(&src.join("README.txt"), &overlay, 0o644);
2156         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2157         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2158
2159         // Generate the installer tarball
2160         let mut cmd = rust_installer(builder);
2161         cmd.arg("generate")
2162             .arg("--product-name=Rust")
2163             .arg("--rel-manifest-dir=rustlib")
2164             .arg("--success-message=llvm-tools-installed.")
2165             .arg("--image-dir").arg(&image)
2166             .arg("--work-dir").arg(&tmpdir(builder))
2167             .arg("--output-dir").arg(&distdir(builder))
2168             .arg("--non-installed-overlay").arg(&overlay)
2169             .arg(format!("--package-name={}-{}", name, target))
2170             .arg("--legacy-manifest-dirs=rustlib,cargo")
2171             .arg("--component-name=llvm-tools-preview");
2172
2173
2174         builder.run(&mut cmd);
2175         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2176     }
2177 }
2178
2179 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2180 pub struct Lldb {
2181     pub target: Interned<String>,
2182 }
2183
2184 impl Step for Lldb {
2185     type Output = Option<PathBuf>;
2186     const ONLY_HOSTS: bool = true;
2187     const DEFAULT: bool = true;
2188
2189     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2190         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2191     }
2192
2193     fn make_run(run: RunConfig<'_>) {
2194         run.builder.ensure(Lldb {
2195             target: run.target,
2196         });
2197     }
2198
2199     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2200         let target = self.target;
2201
2202         if builder.config.dry_run {
2203             return None;
2204         }
2205
2206         let bindir = builder
2207             .llvm_out(target)
2208             .join("bin");
2209         let lldb_exe = bindir.join(exe("lldb", &target));
2210         if !lldb_exe.exists() {
2211             return None;
2212         }
2213
2214         builder.info(&format!("Dist Lldb ({})", target));
2215         let src = builder.src.join("src/llvm-project/lldb");
2216         let name = pkgname(builder, "lldb");
2217
2218         let tmp = tmpdir(builder);
2219         let image = tmp.join("lldb-image");
2220         drop(fs::remove_dir_all(&image));
2221
2222         // Prepare the image directory
2223         let root = image.join("lib/rustlib").join(&*target);
2224         let dst = root.join("bin");
2225         t!(fs::create_dir_all(&dst));
2226         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2227             let exe = bindir.join(exe(program, &target));
2228             builder.install(&exe, &dst, 0o755);
2229         }
2230
2231         // The libraries.
2232         let libdir = builder.llvm_out(target).join("lib");
2233         let dst = root.join("lib");
2234         t!(fs::create_dir_all(&dst));
2235         for entry in t!(fs::read_dir(&libdir)) {
2236             let entry = entry.unwrap();
2237             if let Ok(name) = entry.file_name().into_string() {
2238                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2239                     if t!(entry.file_type()).is_symlink() {
2240                         builder.copy_to_folder(&entry.path(), &dst);
2241                     } else {
2242                        builder.install(&entry.path(), &dst, 0o755);
2243                     }
2244                 }
2245             }
2246         }
2247
2248         // The lldb scripts might be installed in lib/python$version
2249         // or in lib64/python$version.  If lib64 exists, use it;
2250         // otherwise lib.
2251         let libdir = builder.llvm_out(target).join("lib64");
2252         let (libdir, libdir_name) = if libdir.exists() {
2253             (libdir, "lib64")
2254         } else {
2255             (builder.llvm_out(target).join("lib"), "lib")
2256         };
2257         for entry in t!(fs::read_dir(&libdir)) {
2258             let entry = t!(entry);
2259             if let Ok(name) = entry.file_name().into_string() {
2260                 if name.starts_with("python") {
2261                     let dst = root.join(libdir_name)
2262                         .join(entry.file_name());
2263                     t!(fs::create_dir_all(&dst));
2264                     builder.cp_r(&entry.path(), &dst);
2265                     break;
2266                 }
2267             }
2268         }
2269
2270         // Prepare the overlay
2271         let overlay = tmp.join("lldb-overlay");
2272         drop(fs::remove_dir_all(&overlay));
2273         builder.create_dir(&overlay);
2274         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2275         builder.create(&overlay.join("version"), &builder.lldb_vers());
2276
2277         // Generate the installer tarball
2278         let mut cmd = rust_installer(builder);
2279         cmd.arg("generate")
2280             .arg("--product-name=Rust")
2281             .arg("--rel-manifest-dir=rustlib")
2282             .arg("--success-message=lldb-installed.")
2283             .arg("--image-dir").arg(&image)
2284             .arg("--work-dir").arg(&tmpdir(builder))
2285             .arg("--output-dir").arg(&distdir(builder))
2286             .arg("--non-installed-overlay").arg(&overlay)
2287             .arg(format!("--package-name={}-{}", name, target))
2288             .arg("--legacy-manifest-dirs=rustlib,cargo")
2289             .arg("--component-name=lldb-preview");
2290
2291
2292         builder.run(&mut cmd);
2293         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2294     }
2295 }