src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe, timeit};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         if !builder.config.docs {
  95             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  96         }
  97
  98         builder.default_doc(None);
  99
 100         builder.info(&format!("Dist docs ({})", host));
 101         let _time = timeit(builder);
 102
 103         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 104         let _ = fs::remove_dir_all(&image);
 105
 106         let dst = image.join("share/doc/rust/html");
 107         t!(fs::create_dir_all(&dst));
 108         let src = builder.doc_out(host);
 109         builder.cp_r(&src, &dst);
 110
 111         let mut cmd = rust_installer(builder);
 112         cmd.arg("generate")
 113            .arg("--product-name=Rust-Documentation")
 114            .arg("--rel-manifest-dir=rustlib")
 115            .arg("--success-message=Rust-documentation-is-installed.")
 116            .arg("--image-dir").arg(&image)
 117            .arg("--work-dir").arg(&tmpdir(builder))
 118            .arg("--output-dir").arg(&distdir(builder))
 119            .arg(format!("--package-name={}-{}", name, host))
 120            .arg("--component-name=rust-docs")
 121            .arg("--legacy-manifest-dirs=rustlib,cargo")
 122            .arg("--bulk-dirs=share/doc/rust/html");
 123         builder.run(&mut cmd);
 124         builder.remove_dir(&image);
 125
 126         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 127     }
 128 }
 129
 130 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 131 pub struct RustcDocs {
 132     pub host: Interned<String>,
 133 }
 134
 135 impl Step for RustcDocs {
 136     type Output = PathBuf;
 137     const DEFAULT: bool = true;
 138
 139     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 140         run.path("src/librustc")
 141     }
 142
 143     fn make_run(run: RunConfig<'_>) {
 144         run.builder.ensure(RustcDocs {
 145             host: run.target,
 146         });
 147     }
 148
 149     /// Builds the `rustc-docs` installer component.
 150     fn run(self, builder: &Builder<'_>) -> PathBuf {
 151         let host = self.host;
 152
 153         let name = pkgname(builder, "rustc-docs");
 154
 155         if !builder.config.compiler_docs {
 156             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 157         }
 158
 159         builder.default_doc(None);
 160
 161         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 162         let _ = fs::remove_dir_all(&image);
 163
 164         let dst = image.join("share/doc/rust/html");
 165         t!(fs::create_dir_all(&dst));
 166         let src = builder.compiler_doc_out(host);
 167         builder.cp_r(&src, &dst);
 168
 169         let mut cmd = rust_installer(builder);
 170         cmd.arg("generate")
 171            .arg("--product-name=Rustc-Documentation")
 172            .arg("--rel-manifest-dir=rustlib")
 173            .arg("--success-message=Rustc-documentation-is-installed.")
 174            .arg("--image-dir").arg(&image)
 175            .arg("--work-dir").arg(&tmpdir(builder))
 176            .arg("--output-dir").arg(&distdir(builder))
 177            .arg(format!("--package-name={}-{}", name, host))
 178            .arg("--component-name=rustc-docs")
 179            .arg("--legacy-manifest-dirs=rustlib,cargo")
 180            .arg("--bulk-dirs=share/doc/rust/html");
 181
 182         builder.info(&format!("Dist compiler docs ({})", host));
 183         let _time = timeit(builder);
 184         builder.run(&mut cmd);
 185         builder.remove_dir(&image);
 186
 187         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 188     }
 189 }
 190
 191 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 192     let mut found = Vec::with_capacity(files.len());
 193
 194     for file in files {
 195         let file_path =
 196             path.iter()
 197                 .map(|dir| dir.join(file))
 198                 .find(|p| p.exists());
 199
 200         if let Some(file_path) = file_path {
 201             found.push(file_path);
 202         } else {
 203             panic!("Could not find '{}' in {:?}", file, path);
 204         }
 205     }
 206
 207     found
 208 }
 209
 210 fn make_win_dist(
 211     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 212 ) {
 213     //Ask gcc where it keeps its stuff
 214     let mut cmd = Command::new(builder.cc(target_triple));
 215     cmd.arg("-print-search-dirs");
 216     let gcc_out = output(&mut cmd);
 217
 218     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 219     let mut lib_path = Vec::new();
 220
 221     for line in gcc_out.lines() {
 222         let idx = line.find(':').unwrap();
 223         let key = &line[..idx];
 224         let trim_chars: &[_] = &[' ', '='];
 225         let value =
 226             line[(idx + 1)..]
 227                 .trim_start_matches(trim_chars)
 228                 .split(';')
 229                 .map(PathBuf::from);
 230
 231         if key == "programs" {
 232             bin_path.extend(value);
 233         } else if key == "libraries" {
 234             lib_path.extend(value);
 235         }
 236     }
 237
 238     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 239     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 240     if target_triple.starts_with("i686-") {
 241         rustc_dlls.push("libgcc_s_dw2-1.dll");
 242     } else {
 243         rustc_dlls.push("libgcc_s_seh-1.dll");
 244     }
 245
 246     let target_libs = [ //MinGW libs
 247         "libgcc.a",
 248         "libgcc_eh.a",
 249         "libgcc_s.a",
 250         "libm.a",
 251         "libmingw32.a",
 252         "libmingwex.a",
 253         "libstdc++.a",
 254         "libiconv.a",
 255         "libmoldname.a",
 256         "libpthread.a",
 257         //Windows import libs
 258         "libadvapi32.a",
 259         "libbcrypt.a",
 260         "libcomctl32.a",
 261         "libcomdlg32.a",
 262         "libcredui.a",
 263         "libcrypt32.a",
 264         "libdbghelp.a",
 265         "libgdi32.a",
 266         "libimagehlp.a",
 267         "libiphlpapi.a",
 268         "libkernel32.a",
 269         "libmsimg32.a",
 270         "libmsvcrt.a",
 271         "libodbc32.a",
 272         "libole32.a",
 273         "liboleaut32.a",
 274         "libopengl32.a",
 275         "libpsapi.a",
 276         "librpcrt4.a",
 277         "libsecur32.a",
 278         "libsetupapi.a",
 279         "libshell32.a",
 280         "libsynchronization.a",
 281         "libuser32.a",
 282         "libuserenv.a",
 283         "libuuid.a",
 284         "libwinhttp.a",
 285         "libwinmm.a",
 286         "libwinspool.a",
 287         "libws2_32.a",
 288         "libwsock32.a",
 289     ];
 290
 291     //Find mingw artifacts we want to bundle
 292     let target_tools = find_files(&target_tools, &bin_path);
 293     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 294     let target_libs = find_files(&target_libs, &lib_path);
 295
 296     // Copy runtime dlls next to rustc.exe
 297     let dist_bin_dir = rust_root.join("bin/");
 298     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 299     for src in rustc_dlls {
 300         builder.copy_to_folder(&src, &dist_bin_dir);
 301     }
 302
 303     //Copy platform tools to platform-specific bin directory
 304     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 305     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 306     for src in target_tools {
 307         builder.copy_to_folder(&src, &target_bin_dir);
 308     }
 309
 310     // Warn windows-gnu users that the bundled GCC cannot compile C files
 311     builder.create(
 312         &target_bin_dir.join("GCC-WARNING.txt"),
 313         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 314          used as a linker. In order to be able to compile projects containing C code use\
 315          the GCC provided by MinGW or Cygwin."
 316     );
 317
 318     //Copy platform libs to platform-specific lib directory
 319     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 320     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 321     for src in target_libs {
 322         builder.copy_to_folder(&src, &target_lib_dir);
 323     }
 324 }
 325
 326 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 327 pub struct Mingw {
 328     pub host: Interned<String>,
 329 }
 330
 331 impl Step for Mingw {
 332     type Output = Option<PathBuf>;
 333     const DEFAULT: bool = true;
 334
 335     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 336         run.never()
 337     }
 338
 339     fn make_run(run: RunConfig<'_>) {
 340         run.builder.ensure(Mingw { host: run.target });
 341     }
 342
 343     /// Builds the `rust-mingw` installer component.
 344     ///
 345     /// This contains all the bits and pieces to run the MinGW Windows targets
 346     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 347     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 348         let host = self.host;
 349
 350         if !host.contains("pc-windows-gnu") {
 351             return None;
 352         }
 353
 354         builder.info(&format!("Dist mingw ({})", host));
 355         let _time = timeit(builder);
 356         let name = pkgname(builder, "rust-mingw");
 357         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 358         let _ = fs::remove_dir_all(&image);
 359         t!(fs::create_dir_all(&image));
 360
 361         // The first argument is a "temporary directory" which is just
 362         // thrown away (this contains the runtime DLLs included in the rustc package
 363         // above) and the second argument is where to place all the MinGW components
 364         // (which is what we want).
 365         make_win_dist(&tmpdir(builder), &image, host, &builder);
 366
 367         let mut cmd = rust_installer(builder);
 368         cmd.arg("generate")
 369            .arg("--product-name=Rust-MinGW")
 370            .arg("--rel-manifest-dir=rustlib")
 371            .arg("--success-message=Rust-MinGW-is-installed.")
 372            .arg("--image-dir").arg(&image)
 373            .arg("--work-dir").arg(&tmpdir(builder))
 374            .arg("--output-dir").arg(&distdir(builder))
 375            .arg(format!("--package-name={}-{}", name, host))
 376            .arg("--component-name=rust-mingw")
 377            .arg("--legacy-manifest-dirs=rustlib,cargo");
 378         builder.run(&mut cmd);
 379         t!(fs::remove_dir_all(&image));
 380         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 381     }
 382 }
 383
 384 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 385 pub struct Rustc {
 386     pub compiler: Compiler,
 387 }
 388
 389 impl Step for Rustc {
 390     type Output = PathBuf;
 391     const DEFAULT: bool = true;
 392     const ONLY_HOSTS: bool = true;
 393
 394     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 395         run.path("src/librustc")
 396     }
 397
 398     fn make_run(run: RunConfig<'_>) {
 399         run.builder.ensure(Rustc {
 400             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 401         });
 402     }
 403
 404     /// Creates the `rustc` installer component.
 405     fn run(self, builder: &Builder<'_>) -> PathBuf {
 406         let compiler = self.compiler;
 407         let host = self.compiler.host;
 408
 409         let name = pkgname(builder, "rustc");
 410         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 411         let _ = fs::remove_dir_all(&image);
 412         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 413         let _ = fs::remove_dir_all(&overlay);
 414
 415         // Prepare the rustc "image", what will actually end up getting installed
 416         prepare_image(builder, compiler, &image);
 417
 418         // Prepare the overlay which is part of the tarball but won't actually be
 419         // installed
 420         let cp = |file: &str| {
 421             builder.install(&builder.src.join(file), &overlay, 0o644);
 422         };
 423         cp("COPYRIGHT");
 424         cp("LICENSE-APACHE");
 425         cp("LICENSE-MIT");
 426         cp("README.md");
 427         // tiny morsel of metadata is used by rust-packaging
 428         let version = builder.rust_version();
 429         builder.create(&overlay.join("version"), &version);
 430         if let Some(sha) = builder.rust_sha() {
 431             builder.create(&overlay.join("git-commit-hash"), &sha);
 432         }
 433
 434         // On MinGW we've got a few runtime DLL dependencies that we need to
 435         // include. The first argument to this script is where to put these DLLs
 436         // (the image we're creating), and the second argument is a junk directory
 437         // to ignore all other MinGW stuff the script creates.
 438         //
 439         // On 32-bit MinGW we're always including a DLL which needs some extra
 440         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 441         // anything requiring us to distribute a license, but it's likely the
 442         // install will *also* include the rust-mingw package, which also needs
 443         // licenses, so to be safe we just include it here in all MinGW packages.
 444         if host.contains("pc-windows-gnu") {
 445             make_win_dist(&image, &tmpdir(builder), host, builder);
 446
 447             let dst = image.join("share/doc");
 448             t!(fs::create_dir_all(&dst));
 449             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 450         }
 451
 452         // Finally, wrap everything up in a nice tarball!
 453         let mut cmd = rust_installer(builder);
 454         cmd.arg("generate")
 455            .arg("--product-name=Rust")
 456            .arg("--rel-manifest-dir=rustlib")
 457            .arg("--success-message=Rust-is-ready-to-roll.")
 458            .arg("--image-dir").arg(&image)
 459            .arg("--work-dir").arg(&tmpdir(builder))
 460            .arg("--output-dir").arg(&distdir(builder))
 461            .arg("--non-installed-overlay").arg(&overlay)
 462            .arg(format!("--package-name={}-{}", name, host))
 463            .arg("--component-name=rustc")
 464            .arg("--legacy-manifest-dirs=rustlib,cargo");
 465
 466         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 467         let _time = timeit(builder);
 468         builder.run(&mut cmd);
 469         builder.remove_dir(&image);
 470         builder.remove_dir(&overlay);
 471
 472         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 473
 474         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 475             let host = compiler.host;
 476             let src = builder.sysroot(compiler);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 483
 484             let libdir_relative = builder.libdir_relative(compiler);
 485
 486             // Copy runtime DLLs needed by the compiler
 487             if libdir_relative.to_str() != Some("bin") {
 488                 let libdir = builder.rustc_libdir(compiler);
 489                 for entry in builder.read_dir(&libdir) {
 490                     let name = entry.file_name();
 491                     if let Some(s) = name.to_str() {
 492                         if is_dylib(s) {
 493                             // Don't use custom libdir here because ^lib/ will be resolved again
 494                             // with installer
 495                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 496                         }
 497                     }
 498                 }
 499             }
 500
 501             // Copy libLLVM.so to the lib dir as well, if needed. While not
 502             // technically needed by rustc itself it's needed by lots of other
 503             // components like the llvm tools and LLD. LLD is included below and
 504             // tools/LLDB come later, so let's just throw it in the rustc
 505             // component for now.
 506             maybe_install_llvm_dylib(builder, host, image);
 507
 508             // Copy over lld if it's there
 509             if builder.config.lld_enabled {
 510                 let exe = exe("rust-lld", &compiler.host);
 511                 let src = builder.sysroot_libdir(compiler, host)
 512                     .parent()
 513                     .unwrap()
 514                     .join("bin")
 515                     .join(&exe);
 516                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 517                 let dst = image.join("lib/rustlib")
 518                     .join(&*host)
 519                     .join("bin")
 520                     .join(&exe);
 521                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 522                 builder.copy(&src, &dst);
 523             }
 524
 525             // Man pages
 526             t!(fs::create_dir_all(image.join("share/man/man1")));
 527             let man_src = builder.src.join("src/doc/man");
 528             let man_dst = image.join("share/man/man1");
 529
 530             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 531             let time = env::var("SOURCE_DATE_EPOCH")
 532                 .map(|timestamp| {
 533                     let epoch = timestamp.parse().map_err(|err| {
 534                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 535                     }).unwrap();
 536
 537                     time::at(Timespec::new(epoch, 0))
 538                 })
 539                 .unwrap_or_else(|_| time::now());
 540
 541             let month_year = t!(time::strftime("%B %Y", &time));
 542             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 543             // to hardlink, and we don't want to edit the source templates
 544             for file_entry in builder.read_dir(&man_src) {
 545                 let page_src = file_entry.path();
 546                 let page_dst = man_dst.join(file_entry.file_name());
 547                 t!(fs::copy(&page_src, &page_dst));
 548                 // template in month/year and version number
 549                 builder.replace_in_file(&page_dst,
 550                                 &[("<INSERT DATE HERE>", &month_year),
 551                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 552             }
 553
 554             // Debugger scripts
 555             builder.ensure(DebuggerScripts {
 556                 sysroot: INTERNER.intern_path(image.to_owned()),
 557                 host,
 558             });
 559
 560             // Misc license info
 561             let cp = |file: &str| {
 562                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 563             };
 564             cp("COPYRIGHT");
 565             cp("LICENSE-APACHE");
 566             cp("LICENSE-MIT");
 567             cp("README.md");
 568         }
 569     }
 570 }
 571
 572 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 573 pub struct DebuggerScripts {
 574     pub sysroot: Interned<PathBuf>,
 575     pub host: Interned<String>,
 576 }
 577
 578 impl Step for DebuggerScripts {
 579     type Output = ();
 580
 581     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 582         run.path("src/lldb_batchmode.py")
 583     }
 584
 585     fn make_run(run: RunConfig<'_>) {
 586         run.builder.ensure(DebuggerScripts {
 587             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 588             host: run.target,
 589         });
 590     }
 591
 592     /// Copies debugger scripts for `target` into the `sysroot` specified.
 593     fn run(self, builder: &Builder<'_>) {
 594         let host = self.host;
 595         let sysroot = self.sysroot;
 596         let dst = sysroot.join("lib/rustlib/etc");
 597         t!(fs::create_dir_all(&dst));
 598         let cp_debugger_script = |file: &str| {
 599             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 600         };
 601         if host.contains("windows-msvc") {
 602             // windbg debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 604                 0o755);
 605
 606             cp_debugger_script("natvis/intrinsic.natvis");
 607             cp_debugger_script("natvis/liballoc.natvis");
 608             cp_debugger_script("natvis/libcore.natvis");
 609             cp_debugger_script("natvis/libstd.natvis");
 610         } else {
 611             cp_debugger_script("debugger_pretty_printers_common.py");
 612
 613             // gdb debugger scripts
 614             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 615                     0o755);
 616             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 617                     0o755);
 618
 619             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 620             cp_debugger_script("gdb_rust_pretty_printing.py");
 621
 622             // lldb debugger scripts
 623             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 624                     0o755);
 625
 626             cp_debugger_script("lldb_rust_formatters.py");
 627         }
 628     }
 629 }
 630
 631 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 632     // The only true set of target libraries came from the build triple, so
 633     // let's reduce redundant work by only producing archives from that host.
 634     if compiler.host != builder.config.build {
 635         builder.info("\tskipping, not a build host");
 636         true
 637     } else {
 638         false
 639     }
 640 }
 641
 642 /// Copy stamped files into an image's `target/lib` directory.
 643 fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
 644     let dst = image.join("lib/rustlib").join(target).join("lib");
 645     t!(fs::create_dir_all(&dst));
 646     for (path, host) in builder.read_stamp_file(stamp) {
 647         if !host || builder.config.build == target {
 648             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 649         }
 650     }
 651 }
 652
 653 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 654 pub struct Std {
 655     pub compiler: Compiler,
 656     pub target: Interned<String>,
 657 }
 658
 659 impl Step for Std {
 660     type Output = PathBuf;
 661     const DEFAULT: bool = true;
 662
 663     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 664         run.path("src/libstd")
 665     }
 666
 667     fn make_run(run: RunConfig<'_>) {
 668         run.builder.ensure(Std {
 669             compiler: run.builder.compiler_for(
 670                 run.builder.top_stage,
 671                 run.builder.config.build,
 672                 run.target,
 673             ),
 674             target: run.target,
 675         });
 676     }
 677
 678     fn run(self, builder: &Builder<'_>) -> PathBuf {
 679         let compiler = self.compiler;
 680         let target = self.target;
 681
 682         let name = pkgname(builder, "rust-std");
 683         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 684         if skip_host_target_lib(builder, compiler) {
 685             return archive;
 686         }
 687
 688         builder.ensure(compile::Std { compiler, target });
 689
 690         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 691         let _ = fs::remove_dir_all(&image);
 692
 693         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 694         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 695         copy_target_libs(builder, &target, &image, &stamp);
 696
 697         let mut cmd = rust_installer(builder);
 698         cmd.arg("generate")
 699            .arg("--product-name=Rust")
 700            .arg("--rel-manifest-dir=rustlib")
 701            .arg("--success-message=std-is-standing-at-the-ready.")
 702            .arg("--image-dir").arg(&image)
 703            .arg("--work-dir").arg(&tmpdir(builder))
 704            .arg("--output-dir").arg(&distdir(builder))
 705            .arg(format!("--package-name={}-{}", name, target))
 706            .arg(format!("--component-name=rust-std-{}", target))
 707            .arg("--legacy-manifest-dirs=rustlib,cargo");
 708
 709         builder.info(&format!("Dist std stage{} ({} -> {})",
 710             compiler.stage, &compiler.host, target));
 711         let _time = timeit(builder);
 712         builder.run(&mut cmd);
 713         builder.remove_dir(&image);
 714         archive
 715     }
 716 }
 717
 718 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 719 pub struct RustcDev {
 720     pub compiler: Compiler,
 721     pub target: Interned<String>,
 722 }
 723
 724 impl Step for RustcDev {
 725     type Output = PathBuf;
 726     const DEFAULT: bool = true;
 727     const ONLY_HOSTS: bool = true;
 728
 729     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 730         run.path("rustc-dev")
 731     }
 732
 733     fn make_run(run: RunConfig<'_>) {
 734         run.builder.ensure(RustcDev {
 735             compiler: run.builder.compiler_for(
 736                 run.builder.top_stage,
 737                 run.builder.config.build,
 738                 run.target,
 739             ),
 740             target: run.target,
 741         });
 742     }
 743
 744     fn run(self, builder: &Builder<'_>) -> PathBuf {
 745         let compiler = self.compiler;
 746         let target = self.target;
 747
 748         let name = pkgname(builder, "rustc-dev");
 749         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 750         if skip_host_target_lib(builder, compiler) {
 751             return archive;
 752         }
 753
 754         builder.ensure(compile::Rustc { compiler, target });
 755
 756         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 757         let _ = fs::remove_dir_all(&image);
 758
 759         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 760         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 761         copy_target_libs(builder, &target, &image, &stamp);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=Rust-is-ready-to-develop.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rustc-dev-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774
 775         builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
 776             compiler.stage, &compiler.host, target));
 777         let _time = timeit(builder);
 778         builder.run(&mut cmd);
 779         builder.remove_dir(&image);
 780         archive
 781     }
 782 }
 783
 784 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 785 pub struct Analysis {
 786     pub compiler: Compiler,
 787     pub target: Interned<String>,
 788 }
 789
 790 impl Step for Analysis {
 791     type Output = PathBuf;
 792     const DEFAULT: bool = true;
 793
 794     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 795         let builder = run.builder;
 796         run.path("analysis").default_condition(builder.config.extended)
 797     }
 798
 799     fn make_run(run: RunConfig<'_>) {
 800         run.builder.ensure(Analysis {
 801             // Find the actual compiler (handling the full bootstrap option) which
 802             // produced the save-analysis data because that data isn't copied
 803             // through the sysroot uplifting.
 804             compiler: run.builder.compiler_for(
 805                 run.builder.top_stage,
 806                 run.builder.config.build,
 807                 run.target,
 808             ),
 809             target: run.target,
 810         });
 811     }
 812
 813     /// Creates a tarball of save-analysis metadata, if available.
 814     fn run(self, builder: &Builder<'_>) -> PathBuf {
 815         let compiler = self.compiler;
 816         let target = self.target;
 817         assert!(builder.config.extended);
 818         let name = pkgname(builder, "rust-analysis");
 819
 820         if &compiler.host != builder.config.build {
 821             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 822         }
 823
 824         builder.ensure(compile::Std { compiler, target });
 825
 826         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 827
 828         let src = builder.stage_out(compiler, Mode::Std)
 829             .join(target).join(builder.cargo_dir()).join("deps");
 830
 831         let image_src = src.join("save-analysis");
 832         let dst = image.join("lib/rustlib").join(target).join("analysis");
 833         t!(fs::create_dir_all(&dst));
 834         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 835         builder.cp_r(&image_src, &dst);
 836
 837         let mut cmd = rust_installer(builder);
 838         cmd.arg("generate")
 839            .arg("--product-name=Rust")
 840            .arg("--rel-manifest-dir=rustlib")
 841            .arg("--success-message=save-analysis-saved.")
 842            .arg("--image-dir").arg(&image)
 843            .arg("--work-dir").arg(&tmpdir(builder))
 844            .arg("--output-dir").arg(&distdir(builder))
 845            .arg(format!("--package-name={}-{}", name, target))
 846            .arg(format!("--component-name=rust-analysis-{}", target))
 847            .arg("--legacy-manifest-dirs=rustlib,cargo");
 848
 849         builder.info("Dist analysis");
 850         let _time = timeit(builder);
 851         builder.run(&mut cmd);
 852         builder.remove_dir(&image);
 853         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 854     }
 855 }
 856
 857 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 858     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 859         let spath = match path.to_str() {
 860             Some(path) => path,
 861             None => return false,
 862         };
 863         if spath.ends_with("~") || spath.ends_with(".pyc") {
 864             return false
 865         }
 866
 867         const LLVM_PROJECTS: &[&str] = &[
 868             "llvm-project/clang", "llvm-project\\clang",
 869             "llvm-project/libunwind", "llvm-project\\libunwind",
 870             "llvm-project/lld", "llvm-project\\lld",
 871             "llvm-project/lldb", "llvm-project\\lldb",
 872             "llvm-project/llvm", "llvm-project\\llvm",
 873             "llvm-project/compiler-rt", "llvm-project\\compiler-rt",
 874         ];
 875         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 876             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 877         {
 878             return false;
 879         }
 880
 881         const LLVM_TEST: &[&str] = &[
 882             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 883         ];
 884         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 885             (spath.ends_with(".ll") ||
 886              spath.ends_with(".td") ||
 887              spath.ends_with(".s")) {
 888             return false
 889         }
 890
 891         let full_path = Path::new(dir).join(path);
 892         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 893             return false;
 894         }
 895
 896         let excludes = [
 897             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 898             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 899             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 900             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 901         ];
 902         !path.iter()
 903              .map(|s| s.to_str().unwrap())
 904              .any(|s| excludes.contains(&s))
 905     }
 906
 907     // Copy the directories using our filter
 908     for item in src_dirs {
 909         let dst = &dst_dir.join(item);
 910         t!(fs::create_dir_all(dst));
 911         builder.cp_filtered(
 912             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 913     }
 914 }
 915
 916 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 917 pub struct Src;
 918
 919 impl Step for Src {
 920     /// The output path of the src installer tarball
 921     type Output = PathBuf;
 922     const DEFAULT: bool = true;
 923     const ONLY_HOSTS: bool = true;
 924
 925     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 926         run.path("src")
 927     }
 928
 929     fn make_run(run: RunConfig<'_>) {
 930         run.builder.ensure(Src);
 931     }
 932
 933     /// Creates the `rust-src` installer component
 934     fn run(self, builder: &Builder<'_>) -> PathBuf {
 935         let name = pkgname(builder, "rust-src");
 936         let image = tmpdir(builder).join(format!("{}-image", name));
 937         let _ = fs::remove_dir_all(&image);
 938
 939         let dst = image.join("lib/rustlib/src");
 940         let dst_src = dst.join("rust");
 941         t!(fs::create_dir_all(&dst_src));
 942
 943         let src_files = [
 944             "Cargo.lock",
 945         ];
 946         // This is the reduced set of paths which will become the rust-src component
 947         // (essentially libstd and all of its path dependencies)
 948         let std_src_dirs = [
 949             "src/build_helper",
 950             "src/liballoc",
 951             "src/libcore",
 952             "src/libpanic_abort",
 953             "src/libpanic_unwind",
 954             "src/librustc_asan",
 955             "src/librustc_lsan",
 956             "src/librustc_msan",
 957             "src/librustc_tsan",
 958             "src/libstd",
 959             "src/libunwind",
 960             "src/libtest",
 961             "src/libterm",
 962             "src/libprofiler_builtins",
 963             "src/stdarch",
 964             "src/libproc_macro",
 965             "src/tools/rustc-std-workspace-core",
 966             "src/tools/rustc-std-workspace-alloc",
 967             "src/tools/rustc-std-workspace-std",
 968             "src/librustc",
 969             "src/libsyntax",
 970         ];
 971
 972         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 973         for file in src_files.iter() {
 974             builder.copy(&builder.src.join(file), &dst_src.join(file));
 975         }
 976
 977         // Create source tarball in rust-installer format
 978         let mut cmd = rust_installer(builder);
 979         cmd.arg("generate")
 980            .arg("--product-name=Rust")
 981            .arg("--rel-manifest-dir=rustlib")
 982            .arg("--success-message=Awesome-Source.")
 983            .arg("--image-dir").arg(&image)
 984            .arg("--work-dir").arg(&tmpdir(builder))
 985            .arg("--output-dir").arg(&distdir(builder))
 986            .arg(format!("--package-name={}", name))
 987            .arg("--component-name=rust-src")
 988            .arg("--legacy-manifest-dirs=rustlib,cargo");
 989
 990         builder.info("Dist src");
 991         let _time = timeit(builder);
 992         builder.run(&mut cmd);
 993
 994         builder.remove_dir(&image);
 995         distdir(builder).join(&format!("{}.tar.gz", name))
 996     }
 997 }
 998
 999 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1000 pub struct PlainSourceTarball;
1001
1002 impl Step for PlainSourceTarball {
1003     /// Produces the location of the tarball generated
1004     type Output = PathBuf;
1005     const DEFAULT: bool = true;
1006     const ONLY_HOSTS: bool = true;
1007
1008     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1009         let builder = run.builder;
1010         run.path("src").default_condition(builder.config.rust_dist_src)
1011     }
1012
1013     fn make_run(run: RunConfig<'_>) {
1014         run.builder.ensure(PlainSourceTarball);
1015     }
1016
1017     /// Creates the plain source tarball
1018     fn run(self, builder: &Builder<'_>) -> PathBuf {
1019         // Make sure that the root folder of tarball has the correct name
1020         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
1021         let plain_dst_src = tmpdir(builder).join(&plain_name);
1022         let _ = fs::remove_dir_all(&plain_dst_src);
1023         t!(fs::create_dir_all(&plain_dst_src));
1024
1025         // This is the set of root paths which will become part of the source package
1026         let src_files = [
1027             "COPYRIGHT",
1028             "LICENSE-APACHE",
1029             "LICENSE-MIT",
1030             "CONTRIBUTING.md",
1031             "README.md",
1032             "RELEASES.md",
1033             "configure",
1034             "x.py",
1035             "config.toml.example",
1036             "Cargo.toml",
1037             "Cargo.lock",
1038         ];
1039         let src_dirs = [
1040             "src",
1041         ];
1042
1043         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
1044
1045         // Copy the files normally
1046         for item in &src_files {
1047             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
1048         }
1049
1050         // Create the version file
1051         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
1052         if let Some(sha) = builder.rust_sha() {
1053             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
1054         }
1055
1056         // If we're building from git sources, we need to vendor a complete distribution.
1057         if builder.rust_info.is_git() {
1058             // Vendor all Cargo dependencies
1059             let mut cmd = Command::new(&builder.initial_cargo);
1060             cmd.arg("vendor")
1061                .current_dir(&plain_dst_src);
1062             builder.run(&mut cmd);
1063         }
1064
1065         // Create plain source tarball
1066         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1067         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1068         tarball.set_extension(""); // strip .gz
1069         tarball.set_extension(""); // strip .tar
1070         if let Some(dir) = tarball.parent() {
1071             builder.create_dir(&dir);
1072         }
1073         builder.info("running installer");
1074         let mut cmd = rust_installer(builder);
1075         cmd.arg("tarball")
1076            .arg("--input").arg(&plain_name)
1077            .arg("--output").arg(&tarball)
1078            .arg("--work-dir=.")
1079            .current_dir(tmpdir(builder));
1080
1081         builder.info("Create plain source tarball");
1082         let _time = timeit(builder);
1083         builder.run(&mut cmd);
1084         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1085     }
1086 }
1087
1088 // We have to run a few shell scripts, which choke quite a bit on both `\`
1089 // characters and on `C:\` paths, so normalize both of them away.
1090 pub fn sanitize_sh(path: &Path) -> String {
1091     let path = path.to_str().unwrap().replace("\\", "/");
1092     return change_drive(&path).unwrap_or(path);
1093
1094     fn change_drive(s: &str) -> Option<String> {
1095         let mut ch = s.chars();
1096         let drive = ch.next().unwrap_or('C');
1097         if ch.next() != Some(':') {
1098             return None
1099         }
1100         if ch.next() != Some('/') {
1101             return None
1102         }
1103         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1104     }
1105 }
1106
1107 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1108 pub struct Cargo {
1109     pub compiler: Compiler,
1110     pub target: Interned<String>,
1111 }
1112
1113 impl Step for Cargo {
1114     type Output = PathBuf;
1115     const ONLY_HOSTS: bool = true;
1116
1117     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1118         run.path("cargo")
1119     }
1120
1121     fn make_run(run: RunConfig<'_>) {
1122         run.builder.ensure(Cargo {
1123             compiler: run.builder.compiler_for(
1124                 run.builder.top_stage,
1125                 run.builder.config.build,
1126                 run.target,
1127             ),
1128             target: run.target,
1129         });
1130     }
1131
1132     fn run(self, builder: &Builder<'_>) -> PathBuf {
1133         let compiler = self.compiler;
1134         let target = self.target;
1135
1136         let src = builder.src.join("src/tools/cargo");
1137         let etc = src.join("src/etc");
1138         let release_num = builder.release_num("cargo");
1139         let name = pkgname(builder, "cargo");
1140         let version = builder.cargo_info.version(builder, &release_num);
1141
1142         let tmp = tmpdir(builder);
1143         let image = tmp.join("cargo-image");
1144         drop(fs::remove_dir_all(&image));
1145         builder.create_dir(&image);
1146
1147         // Prepare the image directory
1148         builder.create_dir(&image.join("share/zsh/site-functions"));
1149         builder.create_dir(&image.join("etc/bash_completion.d"));
1150         let cargo = builder.ensure(tool::Cargo { compiler, target });
1151         builder.install(&cargo, &image.join("bin"), 0o755);
1152         for man in t!(etc.join("man").read_dir()) {
1153             let man = t!(man);
1154             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1155         }
1156         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1157         builder.copy(&etc.join("cargo.bashcomp.sh"),
1158              &image.join("etc/bash_completion.d/cargo"));
1159         let doc = image.join("share/doc/cargo");
1160         builder.install(&src.join("README.md"), &doc, 0o644);
1161         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1162         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1163         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1164
1165         // Prepare the overlay
1166         let overlay = tmp.join("cargo-overlay");
1167         drop(fs::remove_dir_all(&overlay));
1168         builder.create_dir(&overlay);
1169         builder.install(&src.join("README.md"), &overlay, 0o644);
1170         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1171         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1172         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1173         builder.create(&overlay.join("version"), &version);
1174
1175         // Generate the installer tarball
1176         let mut cmd = rust_installer(builder);
1177         cmd.arg("generate")
1178            .arg("--product-name=Rust")
1179            .arg("--rel-manifest-dir=rustlib")
1180            .arg("--success-message=Rust-is-ready-to-roll.")
1181            .arg("--image-dir").arg(&image)
1182            .arg("--work-dir").arg(&tmpdir(builder))
1183            .arg("--output-dir").arg(&distdir(builder))
1184            .arg("--non-installed-overlay").arg(&overlay)
1185            .arg(format!("--package-name={}-{}", name, target))
1186            .arg("--component-name=cargo")
1187            .arg("--legacy-manifest-dirs=rustlib,cargo");
1188
1189         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1190         let _time = timeit(builder);
1191         builder.run(&mut cmd);
1192         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Rls {
1198     pub compiler: Compiler,
1199     pub target: Interned<String>,
1200 }
1201
1202 impl Step for Rls {
1203     type Output = Option<PathBuf>;
1204     const ONLY_HOSTS: bool = true;
1205
1206     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1207         run.path("rls")
1208     }
1209
1210     fn make_run(run: RunConfig<'_>) {
1211         run.builder.ensure(Rls {
1212             compiler: run.builder.compiler_for(
1213                 run.builder.top_stage,
1214                 run.builder.config.build,
1215                 run.target,
1216             ),
1217             target: run.target,
1218         });
1219     }
1220
1221     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1222         let compiler = self.compiler;
1223         let target = self.target;
1224         assert!(builder.config.extended);
1225
1226         let src = builder.src.join("src/tools/rls");
1227         let release_num = builder.release_num("rls");
1228         let name = pkgname(builder, "rls");
1229         let version = builder.rls_info.version(builder, &release_num);
1230
1231         let tmp = tmpdir(builder);
1232         let image = tmp.join("rls-image");
1233         drop(fs::remove_dir_all(&image));
1234         t!(fs::create_dir_all(&image));
1235
1236         // Prepare the image directory
1237         // We expect RLS to build, because we've exited this step above if tool
1238         // state for RLS isn't testing.
1239         let rls = builder.ensure(tool::Rls {
1240             compiler,
1241             target,
1242             extra_features: Vec::new(),
1243         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1244
1245         builder.install(&rls, &image.join("bin"), 0o755);
1246         let doc = image.join("share/doc/rls");
1247         builder.install(&src.join("README.md"), &doc, 0o644);
1248         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1249         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1250
1251         // Prepare the overlay
1252         let overlay = tmp.join("rls-overlay");
1253         drop(fs::remove_dir_all(&overlay));
1254         t!(fs::create_dir_all(&overlay));
1255         builder.install(&src.join("README.md"), &overlay, 0o644);
1256         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1257         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1258         builder.create(&overlay.join("version"), &version);
1259
1260         // Generate the installer tarball
1261         let mut cmd = rust_installer(builder);
1262         cmd.arg("generate")
1263            .arg("--product-name=Rust")
1264            .arg("--rel-manifest-dir=rustlib")
1265            .arg("--success-message=RLS-ready-to-serve.")
1266            .arg("--image-dir").arg(&image)
1267            .arg("--work-dir").arg(&tmpdir(builder))
1268            .arg("--output-dir").arg(&distdir(builder))
1269            .arg("--non-installed-overlay").arg(&overlay)
1270            .arg(format!("--package-name={}-{}", name, target))
1271            .arg("--legacy-manifest-dirs=rustlib,cargo")
1272            .arg("--component-name=rls-preview");
1273
1274         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1275         let _time = timeit(builder);
1276         builder.run(&mut cmd);
1277         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1278     }
1279 }
1280
1281 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1282 pub struct Clippy {
1283     pub compiler: Compiler,
1284     pub target: Interned<String>,
1285 }
1286
1287 impl Step for Clippy {
1288     type Output = Option<PathBuf>;
1289     const ONLY_HOSTS: bool = true;
1290
1291     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1292         run.path("clippy")
1293     }
1294
1295     fn make_run(run: RunConfig<'_>) {
1296         run.builder.ensure(Clippy {
1297             compiler: run.builder.compiler_for(
1298                 run.builder.top_stage,
1299                 run.builder.config.build,
1300                 run.target,
1301             ),
1302             target: run.target,
1303         });
1304     }
1305
1306     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1307         let compiler = self.compiler;
1308         let target = self.target;
1309         assert!(builder.config.extended);
1310
1311         let src = builder.src.join("src/tools/clippy");
1312         let release_num = builder.release_num("clippy");
1313         let name = pkgname(builder, "clippy");
1314         let version = builder.clippy_info.version(builder, &release_num);
1315
1316         let tmp = tmpdir(builder);
1317         let image = tmp.join("clippy-image");
1318         drop(fs::remove_dir_all(&image));
1319         builder.create_dir(&image);
1320
1321         // Prepare the image directory
1322         // We expect clippy to build, because we've exited this step above if tool
1323         // state for clippy isn't testing.
1324         let clippy = builder.ensure(tool::Clippy {
1325             compiler,
1326             target,
1327             extra_features: Vec::new(),
1328         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1329         let cargoclippy = builder.ensure(tool::CargoClippy {
1330             compiler,
1331             target, extra_features: Vec::new()
1332         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1333
1334         builder.install(&clippy, &image.join("bin"), 0o755);
1335         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1336         let doc = image.join("share/doc/clippy");
1337         builder.install(&src.join("README.md"), &doc, 0o644);
1338         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1339         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1340
1341         // Prepare the overlay
1342         let overlay = tmp.join("clippy-overlay");
1343         drop(fs::remove_dir_all(&overlay));
1344         t!(fs::create_dir_all(&overlay));
1345         builder.install(&src.join("README.md"), &overlay, 0o644);
1346         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1347         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1348         builder.create(&overlay.join("version"), &version);
1349
1350         // Generate the installer tarball
1351         let mut cmd = rust_installer(builder);
1352         cmd.arg("generate")
1353            .arg("--product-name=Rust")
1354            .arg("--rel-manifest-dir=rustlib")
1355            .arg("--success-message=clippy-ready-to-serve.")
1356            .arg("--image-dir").arg(&image)
1357            .arg("--work-dir").arg(&tmpdir(builder))
1358            .arg("--output-dir").arg(&distdir(builder))
1359            .arg("--non-installed-overlay").arg(&overlay)
1360            .arg(format!("--package-name={}-{}", name, target))
1361            .arg("--legacy-manifest-dirs=rustlib,cargo")
1362            .arg("--component-name=clippy-preview");
1363
1364         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1365         let _time = timeit(builder);
1366         builder.run(&mut cmd);
1367         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1368     }
1369 }
1370
1371 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1372 pub struct Miri {
1373     pub compiler: Compiler,
1374     pub target: Interned<String>,
1375 }
1376
1377 impl Step for Miri {
1378     type Output = Option<PathBuf>;
1379     const ONLY_HOSTS: bool = true;
1380
1381     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1382         run.path("miri")
1383     }
1384
1385     fn make_run(run: RunConfig<'_>) {
1386         run.builder.ensure(Miri {
1387             compiler: run.builder.compiler_for(
1388                 run.builder.top_stage,
1389                 run.builder.config.build,
1390                 run.target,
1391             ),
1392             target: run.target,
1393         });
1394     }
1395
1396     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1397         let compiler = self.compiler;
1398         let target = self.target;
1399         assert!(builder.config.extended);
1400
1401         let src = builder.src.join("src/tools/miri");
1402         let release_num = builder.release_num("miri");
1403         let name = pkgname(builder, "miri");
1404         let version = builder.miri_info.version(builder, &release_num);
1405
1406         let tmp = tmpdir(builder);
1407         let image = tmp.join("miri-image");
1408         drop(fs::remove_dir_all(&image));
1409         builder.create_dir(&image);
1410
1411         // Prepare the image directory
1412         // We expect miri to build, because we've exited this step above if tool
1413         // state for miri isn't testing.
1414         let miri = builder.ensure(tool::Miri {
1415             compiler,
1416             target,
1417             extra_features: Vec::new(),
1418         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1419         let cargomiri = builder.ensure(tool::CargoMiri {
1420             compiler,
1421             target,
1422             extra_features: Vec::new()
1423         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1424
1425         builder.install(&miri, &image.join("bin"), 0o755);
1426         builder.install(&cargomiri, &image.join("bin"), 0o755);
1427         let doc = image.join("share/doc/miri");
1428         builder.install(&src.join("README.md"), &doc, 0o644);
1429         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1430         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1431
1432         // Prepare the overlay
1433         let overlay = tmp.join("miri-overlay");
1434         drop(fs::remove_dir_all(&overlay));
1435         t!(fs::create_dir_all(&overlay));
1436         builder.install(&src.join("README.md"), &overlay, 0o644);
1437         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1438         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1439         builder.create(&overlay.join("version"), &version);
1440
1441         // Generate the installer tarball
1442         let mut cmd = rust_installer(builder);
1443         cmd.arg("generate")
1444            .arg("--product-name=Rust")
1445            .arg("--rel-manifest-dir=rustlib")
1446            .arg("--success-message=miri-ready-to-serve.")
1447            .arg("--image-dir").arg(&image)
1448            .arg("--work-dir").arg(&tmpdir(builder))
1449            .arg("--output-dir").arg(&distdir(builder))
1450            .arg("--non-installed-overlay").arg(&overlay)
1451            .arg(format!("--package-name={}-{}", name, target))
1452            .arg("--legacy-manifest-dirs=rustlib,cargo")
1453            .arg("--component-name=miri-preview");
1454
1455         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1456         let _time = timeit(builder);
1457         builder.run(&mut cmd);
1458         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1459     }
1460 }
1461
1462 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1463 pub struct Rustfmt {
1464     pub compiler: Compiler,
1465     pub target: Interned<String>,
1466 }
1467
1468 impl Step for Rustfmt {
1469     type Output = Option<PathBuf>;
1470     const ONLY_HOSTS: bool = true;
1471
1472     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1473         run.path("rustfmt")
1474     }
1475
1476     fn make_run(run: RunConfig<'_>) {
1477         run.builder.ensure(Rustfmt {
1478             compiler: run.builder.compiler_for(
1479                 run.builder.top_stage,
1480                 run.builder.config.build,
1481                 run.target,
1482             ),
1483             target: run.target,
1484         });
1485     }
1486
1487     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1488         let compiler = self.compiler;
1489         let target = self.target;
1490
1491         let src = builder.src.join("src/tools/rustfmt");
1492         let release_num = builder.release_num("rustfmt");
1493         let name = pkgname(builder, "rustfmt");
1494         let version = builder.rustfmt_info.version(builder, &release_num);
1495
1496         let tmp = tmpdir(builder);
1497         let image = tmp.join("rustfmt-image");
1498         drop(fs::remove_dir_all(&image));
1499         builder.create_dir(&image);
1500
1501         // Prepare the image directory
1502         let rustfmt = builder.ensure(tool::Rustfmt {
1503             compiler,
1504             target,
1505             extra_features: Vec::new(),
1506         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1507         let cargofmt = builder.ensure(tool::Cargofmt {
1508             compiler,
1509             target,
1510             extra_features: Vec::new(),
1511         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1512
1513         builder.install(&rustfmt, &image.join("bin"), 0o755);
1514         builder.install(&cargofmt, &image.join("bin"), 0o755);
1515         let doc = image.join("share/doc/rustfmt");
1516         builder.install(&src.join("README.md"), &doc, 0o644);
1517         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1518         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1519
1520         // Prepare the overlay
1521         let overlay = tmp.join("rustfmt-overlay");
1522         drop(fs::remove_dir_all(&overlay));
1523         builder.create_dir(&overlay);
1524         builder.install(&src.join("README.md"), &overlay, 0o644);
1525         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1526         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1527         builder.create(&overlay.join("version"), &version);
1528
1529         // Generate the installer tarball
1530         let mut cmd = rust_installer(builder);
1531         cmd.arg("generate")
1532            .arg("--product-name=Rust")
1533            .arg("--rel-manifest-dir=rustlib")
1534            .arg("--success-message=rustfmt-ready-to-fmt.")
1535            .arg("--image-dir").arg(&image)
1536            .arg("--work-dir").arg(&tmpdir(builder))
1537            .arg("--output-dir").arg(&distdir(builder))
1538            .arg("--non-installed-overlay").arg(&overlay)
1539            .arg(format!("--package-name={}-{}", name, target))
1540            .arg("--legacy-manifest-dirs=rustlib,cargo")
1541            .arg("--component-name=rustfmt-preview");
1542
1543         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1544         let _time = timeit(builder);
1545         builder.run(&mut cmd);
1546         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1547     }
1548 }
1549
1550 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1551 pub struct Extended {
1552     stage: u32,
1553     host: Interned<String>,
1554     target: Interned<String>,
1555 }
1556
1557 impl Step for Extended {
1558     type Output = ();
1559     const DEFAULT: bool = true;
1560     const ONLY_HOSTS: bool = true;
1561
1562     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1563         let builder = run.builder;
1564         run.path("extended").default_condition(builder.config.extended)
1565     }
1566
1567     fn make_run(run: RunConfig<'_>) {
1568         run.builder.ensure(Extended {
1569             stage: run.builder.top_stage,
1570             host: run.builder.config.build,
1571             target: run.target,
1572         });
1573     }
1574
1575     /// Creates a combined installer for the specified target in the provided stage.
1576     fn run(self, builder: &Builder<'_>) {
1577         let target = self.target;
1578         let stage = self.stage;
1579         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1580
1581         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1582
1583         let rustc_installer = builder.ensure(Rustc {
1584             compiler: builder.compiler(stage, target),
1585         });
1586         let cargo_installer = builder.ensure(Cargo { compiler, target });
1587         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1588         let rls_installer = builder.ensure(Rls { compiler, target });
1589         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1590         let clippy_installer = builder.ensure(Clippy { compiler, target });
1591         let miri_installer = builder.ensure(Miri { compiler, target });
1592         let lldb_installer = builder.ensure(Lldb { target });
1593         let mingw_installer = builder.ensure(Mingw { host: target });
1594         let analysis_installer = builder.ensure(Analysis { compiler, target });
1595
1596         let docs_installer = builder.ensure(Docs { host: target, });
1597         let std_installer = builder.ensure(Std {
1598             compiler: builder.compiler(stage, target),
1599             target,
1600         });
1601
1602         let tmp = tmpdir(builder);
1603         let overlay = tmp.join("extended-overlay");
1604         let etc = builder.src.join("src/etc/installer");
1605         let work = tmp.join("work");
1606
1607         let _ = fs::remove_dir_all(&overlay);
1608         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1609         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1610         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1611         let version = builder.rust_version();
1612         builder.create(&overlay.join("version"), &version);
1613         if let Some(sha) = builder.rust_sha() {
1614             builder.create(&overlay.join("git-commit-hash"), &sha);
1615         }
1616         builder.install(&etc.join("README.md"), &overlay, 0o644);
1617
1618         // When rust-std package split from rustc, we needed to ensure that during
1619         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1620         // the std files during uninstall. To do this ensure that rustc comes
1621         // before rust-std in the list below.
1622         let mut tarballs = Vec::new();
1623         tarballs.push(rustc_installer);
1624         tarballs.push(cargo_installer);
1625         tarballs.extend(rls_installer.clone());
1626         tarballs.extend(clippy_installer.clone());
1627         tarballs.extend(miri_installer.clone());
1628         tarballs.extend(rustfmt_installer.clone());
1629         tarballs.extend(llvm_tools_installer);
1630         tarballs.extend(lldb_installer);
1631         tarballs.push(analysis_installer);
1632         tarballs.push(std_installer);
1633         if builder.config.docs {
1634             tarballs.push(docs_installer);
1635         }
1636         if target.contains("pc-windows-gnu") {
1637             tarballs.push(mingw_installer.unwrap());
1638         }
1639         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1640         for tarball in &tarballs[1..] {
1641             input_tarballs.push(",");
1642             input_tarballs.push(tarball);
1643         }
1644
1645         builder.info("building combined installer");
1646         let mut cmd = rust_installer(builder);
1647         cmd.arg("combine")
1648             .arg("--product-name=Rust")
1649             .arg("--rel-manifest-dir=rustlib")
1650             .arg("--success-message=Rust-is-ready-to-roll.")
1651             .arg("--work-dir").arg(&work)
1652             .arg("--output-dir").arg(&distdir(builder))
1653             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1654             .arg("--legacy-manifest-dirs=rustlib,cargo")
1655             .arg("--input-tarballs").arg(input_tarballs)
1656             .arg("--non-installed-overlay").arg(&overlay);
1657         let time = timeit(&builder);
1658         builder.run(&mut cmd);
1659         drop(time);
1660
1661         let mut license = String::new();
1662         license += &builder.read(&builder.src.join("COPYRIGHT"));
1663         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1664         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1665         license.push_str("\n");
1666         license.push_str("\n");
1667
1668         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1669         let mut rtf = rtf.to_string();
1670         rtf.push_str("\n");
1671         for line in license.lines() {
1672             rtf.push_str(line);
1673             rtf.push_str("\\line ");
1674         }
1675         rtf.push_str("}");
1676
1677         fn filter(contents: &str, marker: &str) -> String {
1678             let start = format!("tool-{}-start", marker);
1679             let end = format!("tool-{}-end", marker);
1680             let mut lines = Vec::new();
1681             let mut omitted = false;
1682             for line in contents.lines() {
1683                 if line.contains(&start) {
1684                     omitted = true;
1685                 } else if line.contains(&end) {
1686                     omitted = false;
1687                 } else if !omitted {
1688                     lines.push(line);
1689                 }
1690             }
1691
1692             lines.join("\n")
1693         }
1694
1695         let xform = |p: &Path| {
1696             let mut contents = t!(fs::read_to_string(p));
1697             if rls_installer.is_none() {
1698                 contents = filter(&contents, "rls");
1699             }
1700             if clippy_installer.is_none() {
1701                 contents = filter(&contents, "clippy");
1702             }
1703             if miri_installer.is_none() {
1704                 contents = filter(&contents, "miri");
1705             }
1706             if rustfmt_installer.is_none() {
1707                 contents = filter(&contents, "rustfmt");
1708             }
1709             let ret = tmp.join(p.file_name().unwrap());
1710             t!(fs::write(&ret, &contents));
1711             ret
1712         };
1713
1714         if target.contains("apple-darwin") {
1715             builder.info("building pkg installer");
1716             let pkg = tmp.join("pkg");
1717             let _ = fs::remove_dir_all(&pkg);
1718
1719             let pkgbuild = |component: &str| {
1720                 let mut cmd = Command::new("pkgbuild");
1721                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1722                     .arg("--scripts").arg(pkg.join(component))
1723                     .arg("--nopayload")
1724                     .arg(pkg.join(component).with_extension("pkg"));
1725                 builder.run(&mut cmd);
1726             };
1727
1728             let prepare = |name: &str| {
1729                 builder.create_dir(&pkg.join(name));
1730                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1731                         &pkg.join(name));
1732                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1733                 pkgbuild(name);
1734             };
1735             prepare("rustc");
1736             prepare("cargo");
1737             prepare("rust-docs");
1738             prepare("rust-std");
1739             prepare("rust-analysis");
1740
1741             if rls_installer.is_some() {
1742                 prepare("rls");
1743             }
1744             if clippy_installer.is_some() {
1745                 prepare("clippy");
1746             }
1747             if miri_installer.is_some() {
1748                 prepare("miri");
1749             }
1750
1751             // create an 'uninstall' package
1752             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1753             pkgbuild("uninstall");
1754
1755             builder.create_dir(&pkg.join("res"));
1756             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1757             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1758             let mut cmd = Command::new("productbuild");
1759             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1760                 .arg("--resources").arg(pkg.join("res"))
1761                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1762                                                     pkgname(builder, "rust"),
1763                                                     target)))
1764                 .arg("--package-path").arg(&pkg);
1765             let _time = timeit(builder);
1766             builder.run(&mut cmd);
1767         }
1768
1769         if target.contains("windows") {
1770             let exe = tmp.join("exe");
1771             let _ = fs::remove_dir_all(&exe);
1772
1773             let prepare = |name: &str| {
1774                 builder.create_dir(&exe.join(name));
1775                 let dir = if name == "rust-std" || name == "rust-analysis" {
1776                     format!("{}-{}", name, target)
1777                 } else if name == "rls" {
1778                     "rls-preview".to_string()
1779                 } else if name == "clippy" {
1780                     "clippy-preview".to_string()
1781                 } else if name == "miri" {
1782                     "miri-preview".to_string()
1783                 } else {
1784                     name.to_string()
1785                 };
1786                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1787                             .join(dir),
1788                         &exe.join(name));
1789                 builder.remove(&exe.join(name).join("manifest.in"));
1790             };
1791             prepare("rustc");
1792             prepare("cargo");
1793             prepare("rust-analysis");
1794             prepare("rust-docs");
1795             prepare("rust-std");
1796             if rls_installer.is_some() {
1797                 prepare("rls");
1798             }
1799             if clippy_installer.is_some() {
1800                 prepare("clippy");
1801             }
1802             if miri_installer.is_some() {
1803                 prepare("miri");
1804             }
1805             if target.contains("windows-gnu") {
1806                 prepare("rust-mingw");
1807             }
1808
1809             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1810             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1811             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1812             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1813             builder.create(&exe.join("LICENSE.txt"), &license);
1814
1815             // Generate exe installer
1816             builder.info("building `exe` installer with `iscc`");
1817             let mut cmd = Command::new("iscc");
1818             cmd.arg("rust.iss")
1819                 .arg("/Q")
1820                 .current_dir(&exe);
1821             if target.contains("windows-gnu") {
1822                 cmd.arg("/dMINGW");
1823             }
1824             add_env(builder, &mut cmd, target);
1825             let time = timeit(builder);
1826             builder.run(&mut cmd);
1827             drop(time);
1828             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1829                     &distdir(builder),
1830                     0o755);
1831
1832             // Generate msi installer
1833             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1834             let heat = wix.join("bin/heat.exe");
1835             let candle = wix.join("bin/candle.exe");
1836             let light = wix.join("bin/light.exe");
1837
1838             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1839             builder.run(Command::new(&heat)
1840                             .current_dir(&exe)
1841                             .arg("dir")
1842                             .arg("rustc")
1843                             .args(&heat_flags)
1844                             .arg("-cg").arg("RustcGroup")
1845                             .arg("-dr").arg("Rustc")
1846                             .arg("-var").arg("var.RustcDir")
1847                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1848             builder.run(Command::new(&heat)
1849                             .current_dir(&exe)
1850                             .arg("dir")
1851                             .arg("rust-docs")
1852                             .args(&heat_flags)
1853                             .arg("-cg").arg("DocsGroup")
1854                             .arg("-dr").arg("Docs")
1855                             .arg("-var").arg("var.DocsDir")
1856                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1857                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1858             builder.run(Command::new(&heat)
1859                             .current_dir(&exe)
1860                             .arg("dir")
1861                             .arg("cargo")
1862                             .args(&heat_flags)
1863                             .arg("-cg").arg("CargoGroup")
1864                             .arg("-dr").arg("Cargo")
1865                             .arg("-var").arg("var.CargoDir")
1866                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1867                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1868             builder.run(Command::new(&heat)
1869                             .current_dir(&exe)
1870                             .arg("dir")
1871                             .arg("rust-std")
1872                             .args(&heat_flags)
1873                             .arg("-cg").arg("StdGroup")
1874                             .arg("-dr").arg("Std")
1875                             .arg("-var").arg("var.StdDir")
1876                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1877             if rls_installer.is_some() {
1878                 builder.run(Command::new(&heat)
1879                                 .current_dir(&exe)
1880                                 .arg("dir")
1881                                 .arg("rls")
1882                                 .args(&heat_flags)
1883                                 .arg("-cg").arg("RlsGroup")
1884                                 .arg("-dr").arg("Rls")
1885                                 .arg("-var").arg("var.RlsDir")
1886                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1887                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1888             }
1889             if clippy_installer.is_some() {
1890                 builder.run(Command::new(&heat)
1891                                 .current_dir(&exe)
1892                                 .arg("dir")
1893                                 .arg("clippy")
1894                                 .args(&heat_flags)
1895                                 .arg("-cg").arg("ClippyGroup")
1896                                 .arg("-dr").arg("Clippy")
1897                                 .arg("-var").arg("var.ClippyDir")
1898                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1899                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1900             }
1901             if miri_installer.is_some() {
1902                 builder.run(Command::new(&heat)
1903                                 .current_dir(&exe)
1904                                 .arg("dir")
1905                                 .arg("miri")
1906                                 .args(&heat_flags)
1907                                 .arg("-cg").arg("MiriGroup")
1908                                 .arg("-dr").arg("Miri")
1909                                 .arg("-var").arg("var.MiriDir")
1910                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1911                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1912             }
1913             builder.run(Command::new(&heat)
1914                             .current_dir(&exe)
1915                             .arg("dir")
1916                             .arg("rust-analysis")
1917                             .args(&heat_flags)
1918                             .arg("-cg").arg("AnalysisGroup")
1919                             .arg("-dr").arg("Analysis")
1920                             .arg("-var").arg("var.AnalysisDir")
1921                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1922                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1923             if target.contains("windows-gnu") {
1924                 builder.run(Command::new(&heat)
1925                                 .current_dir(&exe)
1926                                 .arg("dir")
1927                                 .arg("rust-mingw")
1928                                 .args(&heat_flags)
1929                                 .arg("-cg").arg("GccGroup")
1930                                 .arg("-dr").arg("Gcc")
1931                                 .arg("-var").arg("var.GccDir")
1932                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1933             }
1934
1935             let candle = |input: &Path| {
1936                 let output = exe.join(input.file_stem().unwrap())
1937                                 .with_extension("wixobj");
1938                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1939                 let mut cmd = Command::new(&candle);
1940                 cmd.current_dir(&exe)
1941                     .arg("-nologo")
1942                     .arg("-dRustcDir=rustc")
1943                     .arg("-dDocsDir=rust-docs")
1944                     .arg("-dCargoDir=cargo")
1945                     .arg("-dStdDir=rust-std")
1946                     .arg("-dAnalysisDir=rust-analysis")
1947                     .arg("-arch").arg(&arch)
1948                     .arg("-out").arg(&output)
1949                     .arg(&input);
1950                 add_env(builder, &mut cmd, target);
1951
1952                 if rls_installer.is_some() {
1953                     cmd.arg("-dRlsDir=rls");
1954                 }
1955                 if clippy_installer.is_some() {
1956                     cmd.arg("-dClippyDir=clippy");
1957                 }
1958                 if miri_installer.is_some() {
1959                     cmd.arg("-dMiriDir=miri");
1960                 }
1961                 if target.contains("windows-gnu") {
1962                     cmd.arg("-dGccDir=rust-mingw");
1963                 }
1964                 builder.run(&mut cmd);
1965             };
1966             candle(&xform(&etc.join("msi/rust.wxs")));
1967             candle(&etc.join("msi/ui.wxs"));
1968             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1969             candle("RustcGroup.wxs".as_ref());
1970             candle("DocsGroup.wxs".as_ref());
1971             candle("CargoGroup.wxs".as_ref());
1972             candle("StdGroup.wxs".as_ref());
1973             if rls_installer.is_some() {
1974                 candle("RlsGroup.wxs".as_ref());
1975             }
1976             if clippy_installer.is_some() {
1977                 candle("ClippyGroup.wxs".as_ref());
1978             }
1979             if miri_installer.is_some() {
1980                 candle("MiriGroup.wxs".as_ref());
1981             }
1982             candle("AnalysisGroup.wxs".as_ref());
1983
1984             if target.contains("windows-gnu") {
1985                 candle("GccGroup.wxs".as_ref());
1986             }
1987
1988             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1989             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1990             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1991
1992             builder.info(&format!("building `msi` installer with {:?}", light));
1993             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1994             let mut cmd = Command::new(&light);
1995             cmd.arg("-nologo")
1996                 .arg("-ext").arg("WixUIExtension")
1997                 .arg("-ext").arg("WixUtilExtension")
1998                 .arg("-out").arg(exe.join(&filename))
1999                 .arg("rust.wixobj")
2000                 .arg("ui.wixobj")
2001                 .arg("rustwelcomedlg.wixobj")
2002                 .arg("RustcGroup.wixobj")
2003                 .arg("DocsGroup.wixobj")
2004                 .arg("CargoGroup.wixobj")
2005                 .arg("StdGroup.wixobj")
2006                 .arg("AnalysisGroup.wixobj")
2007                 .current_dir(&exe);
2008
2009             if rls_installer.is_some() {
2010                 cmd.arg("RlsGroup.wixobj");
2011             }
2012             if clippy_installer.is_some() {
2013                 cmd.arg("ClippyGroup.wixobj");
2014             }
2015             if miri_installer.is_some() {
2016                 cmd.arg("MiriGroup.wixobj");
2017             }
2018
2019             if target.contains("windows-gnu") {
2020                 cmd.arg("GccGroup.wixobj");
2021             }
2022             // ICE57 wrongly complains about the shortcuts
2023             cmd.arg("-sice:ICE57");
2024
2025             let _time = timeit(builder);
2026             builder.run(&mut cmd);
2027
2028             if !builder.config.dry_run {
2029                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
2030             }
2031         }
2032     }
2033 }
2034
2035 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
2036     let mut parts = channel::CFG_RELEASE_NUM.split('.');
2037     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
2038        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
2039        .env("CFG_RELEASE", builder.rust_release())
2040        .env("CFG_VER_MAJOR", parts.next().unwrap())
2041        .env("CFG_VER_MINOR", parts.next().unwrap())
2042        .env("CFG_VER_PATCH", parts.next().unwrap())
2043        .env("CFG_VER_BUILD", "0") // just needed to build
2044        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
2045        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
2046        .env("CFG_BUILD", target)
2047        .env("CFG_CHANNEL", &builder.config.channel);
2048
2049     if target.contains("windows-gnu") {
2050        cmd.env("CFG_MINGW", "1")
2051           .env("CFG_ABI", "GNU");
2052     } else {
2053        cmd.env("CFG_MINGW", "0")
2054           .env("CFG_ABI", "MSVC");
2055     }
2056
2057     if target.contains("x86_64") {
2058        cmd.env("CFG_PLATFORM", "x64");
2059     } else {
2060        cmd.env("CFG_PLATFORM", "x86");
2061     }
2062 }
2063
2064 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
2065 pub struct HashSign;
2066
2067 impl Step for HashSign {
2068     type Output = ();
2069     const ONLY_HOSTS: bool = true;
2070
2071     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2072         run.path("hash-and-sign")
2073     }
2074
2075     fn make_run(run: RunConfig<'_>) {
2076         run.builder.ensure(HashSign);
2077     }
2078
2079     fn run(self, builder: &Builder<'_>) {
2080         // This gets called by `promote-release`
2081         // (https://github.com/rust-lang/rust-central-station/tree/master/promote-release).
2082         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2083         if builder.config.dry_run {
2084             return;
2085         }
2086         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2087             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2088         });
2089         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2090             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2091         });
2092         let pass = if env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err() {
2093             let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2094                 panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2095             });
2096             t!(fs::read_to_string(&file))
2097         } else {
2098             String::new()
2099         };
2100
2101         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2102
2103         cmd.arg(sign);
2104         cmd.arg(distdir(builder));
2105         cmd.arg(today.trim());
2106         cmd.arg(builder.rust_package_vers());
2107         cmd.arg(addr);
2108         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2109         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2110         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2111         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2112         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2113         cmd.arg(builder.llvm_tools_package_vers());
2114         cmd.arg(builder.lldb_package_vers());
2115
2116         builder.create_dir(&distdir(builder));
2117
2118         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2119         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2120         let status = t!(child.wait());
2121         assert!(status.success());
2122     }
2123 }
2124
2125 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2126 // LLVM tools are linked dynamically.
2127 //
2128 // We add this to both the libdir of the rustc binary itself (for it to load at
2129 // runtime) and also to the target directory so it can find it at link-time.
2130 //
2131 // Note: This function does no yet support Windows but we also don't support
2132 //       linking LLVM tools dynamically on Windows yet.
2133 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2134                                 target: Interned<String>,
2135                                 sysroot: &Path) {
2136     let src_libdir = builder
2137         .llvm_out(target)
2138         .join("lib");
2139     let dst_libdir1 = sysroot.join("lib/rustlib").join(&*target).join("lib");
2140     let dst_libdir2 = sysroot.join(builder.sysroot_libdir_relative(Compiler {
2141         stage: 1,
2142         host: target,
2143     }));
2144     t!(fs::create_dir_all(&dst_libdir1));
2145     t!(fs::create_dir_all(&dst_libdir2));
2146
2147     if target.contains("apple-darwin") {
2148         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2149         if llvm_dylib_path.exists() {
2150             builder.install(&llvm_dylib_path, &dst_libdir1, 0o644);
2151             builder.install(&llvm_dylib_path, &dst_libdir2, 0o644);
2152         }
2153         return
2154     }
2155
2156     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2157     // Since tools link to the latter rather than the former, we have to
2158     // follow the symlink to find out what to distribute.
2159     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2160     if llvm_dylib_path.exists() {
2161         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2162             panic!("dist: Error calling canonicalize path `{}`: {}",
2163                    llvm_dylib_path.display(), e);
2164         });
2165
2166
2167         builder.install(&llvm_dylib_path, &dst_libdir1, 0o644);
2168         builder.install(&llvm_dylib_path, &dst_libdir2, 0o644);
2169     }
2170 }
2171
2172 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2173 pub struct LlvmTools {
2174     pub target: Interned<String>,
2175 }
2176
2177 impl Step for LlvmTools {
2178     type Output = Option<PathBuf>;
2179     const ONLY_HOSTS: bool = true;
2180
2181     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2182         run.path("llvm-tools")
2183     }
2184
2185     fn make_run(run: RunConfig<'_>) {
2186         run.builder.ensure(LlvmTools {
2187             target: run.target,
2188         });
2189     }
2190
2191     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2192         let target = self.target;
2193         assert!(builder.config.extended);
2194
2195         /* run only if llvm-config isn't used */
2196         if let Some(config) = builder.config.target_config.get(&target) {
2197             if let Some(ref _s) = config.llvm_config {
2198                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2199                     target));
2200                 return None;
2201             }
2202         }
2203
2204         builder.info(&format!("Dist LlvmTools ({})", target));
2205         let _time = timeit(builder);
2206         let src = builder.src.join("src/llvm-project/llvm");
2207         let name = pkgname(builder, "llvm-tools");
2208
2209         let tmp = tmpdir(builder);
2210         let image = tmp.join("llvm-tools-image");
2211         drop(fs::remove_dir_all(&image));
2212
2213         // Prepare the image directory
2214         let src_bindir = builder
2215             .llvm_out(target)
2216             .join("bin");
2217         let dst_bindir = image.join("lib/rustlib")
2218             .join(&*target)
2219             .join("bin");
2220         t!(fs::create_dir_all(&dst_bindir));
2221         for tool in LLVM_TOOLS {
2222             let exe = src_bindir.join(exe(tool, &target));
2223             builder.install(&exe, &dst_bindir, 0o755);
2224         }
2225
2226         // Prepare the overlay
2227         let overlay = tmp.join("llvm-tools-overlay");
2228         drop(fs::remove_dir_all(&overlay));
2229         builder.create_dir(&overlay);
2230         builder.install(&src.join("README.txt"), &overlay, 0o644);
2231         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2232         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2233
2234         // Generate the installer tarball
2235         let mut cmd = rust_installer(builder);
2236         cmd.arg("generate")
2237             .arg("--product-name=Rust")
2238             .arg("--rel-manifest-dir=rustlib")
2239             .arg("--success-message=llvm-tools-installed.")
2240             .arg("--image-dir").arg(&image)
2241             .arg("--work-dir").arg(&tmpdir(builder))
2242             .arg("--output-dir").arg(&distdir(builder))
2243             .arg("--non-installed-overlay").arg(&overlay)
2244             .arg(format!("--package-name={}-{}", name, target))
2245             .arg("--legacy-manifest-dirs=rustlib,cargo")
2246             .arg("--component-name=llvm-tools-preview");
2247
2248
2249         builder.run(&mut cmd);
2250         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2251     }
2252 }
2253
2254 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2255 pub struct Lldb {
2256     pub target: Interned<String>,
2257 }
2258
2259 impl Step for Lldb {
2260     type Output = Option<PathBuf>;
2261     const ONLY_HOSTS: bool = true;
2262     const DEFAULT: bool = true;
2263
2264     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2265         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2266     }
2267
2268     fn make_run(run: RunConfig<'_>) {
2269         run.builder.ensure(Lldb {
2270             target: run.target,
2271         });
2272     }
2273
2274     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2275         let target = self.target;
2276
2277         if builder.config.dry_run {
2278             return None;
2279         }
2280
2281         let bindir = builder
2282             .llvm_out(target)
2283             .join("bin");
2284         let lldb_exe = bindir.join(exe("lldb", &target));
2285         if !lldb_exe.exists() {
2286             return None;
2287         }
2288
2289         builder.info(&format!("Dist Lldb ({})", target));
2290         let src = builder.src.join("src/llvm-project/lldb");
2291         let name = pkgname(builder, "lldb");
2292
2293         let tmp = tmpdir(builder);
2294         let image = tmp.join("lldb-image");
2295         drop(fs::remove_dir_all(&image));
2296
2297         // Prepare the image directory
2298         let root = image.join("lib/rustlib").join(&*target);
2299         let dst = root.join("bin");
2300         t!(fs::create_dir_all(&dst));
2301         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2302             let exe = bindir.join(exe(program, &target));
2303             builder.install(&exe, &dst, 0o755);
2304         }
2305
2306         // The libraries.
2307         let libdir = builder.llvm_out(target).join("lib");
2308         let dst = root.join("lib");
2309         t!(fs::create_dir_all(&dst));
2310         for entry in t!(fs::read_dir(&libdir)) {
2311             let entry = entry.unwrap();
2312             if let Ok(name) = entry.file_name().into_string() {
2313                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2314                     if t!(entry.file_type()).is_symlink() {
2315                         builder.copy_to_folder(&entry.path(), &dst);
2316                     } else {
2317                        builder.install(&entry.path(), &dst, 0o755);
2318                     }
2319                 }
2320             }
2321         }
2322
2323         // The lldb scripts might be installed in lib/python$version
2324         // or in lib64/python$version.  If lib64 exists, use it;
2325         // otherwise lib.
2326         let libdir = builder.llvm_out(target).join("lib64");
2327         let (libdir, libdir_name) = if libdir.exists() {
2328             (libdir, "lib64")
2329         } else {
2330             (builder.llvm_out(target).join("lib"), "lib")
2331         };
2332         for entry in t!(fs::read_dir(&libdir)) {
2333             let entry = t!(entry);
2334             if let Ok(name) = entry.file_name().into_string() {
2335                 if name.starts_with("python") {
2336                     let dst = root.join(libdir_name)
2337                         .join(entry.file_name());
2338                     t!(fs::create_dir_all(&dst));
2339                     builder.cp_r(&entry.path(), &dst);
2340                     break;
2341                 }
2342             }
2343         }
2344
2345         // Prepare the overlay
2346         let overlay = tmp.join("lldb-overlay");
2347         drop(fs::remove_dir_all(&overlay));
2348         builder.create_dir(&overlay);
2349         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2350         builder.create(&overlay.join("version"), &builder.lldb_vers());
2351
2352         // Generate the installer tarball
2353         let mut cmd = rust_installer(builder);
2354         cmd.arg("generate")
2355             .arg("--product-name=Rust")
2356             .arg("--rel-manifest-dir=rustlib")
2357             .arg("--success-message=lldb-installed.")
2358             .arg("--image-dir").arg(&image)
2359             .arg("--work-dir").arg(&tmpdir(builder))
2360             .arg("--output-dir").arg(&distdir(builder))
2361             .arg("--non-installed-overlay").arg(&overlay)
2362             .arg(format!("--package-name={}-{}", name, target))
2363             .arg("--legacy-manifest-dirs=rustlib,cargo")
2364             .arg("--component-name=lldb-preview");
2365
2366
2367         builder.run(&mut cmd);
2368         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2369     }
2370 }