src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe, timeit};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         if !builder.config.docs {
  95             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  96         }
  97
  98         builder.default_doc(None);
  99
 100         builder.info(&format!("Dist docs ({})", host));
 101         let _time = timeit(builder);
 102
 103         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 104         let _ = fs::remove_dir_all(&image);
 105
 106         let dst = image.join("share/doc/rust/html");
 107         t!(fs::create_dir_all(&dst));
 108         let src = builder.doc_out(host);
 109         builder.cp_r(&src, &dst);
 110
 111         let mut cmd = rust_installer(builder);
 112         cmd.arg("generate")
 113            .arg("--product-name=Rust-Documentation")
 114            .arg("--rel-manifest-dir=rustlib")
 115            .arg("--success-message=Rust-documentation-is-installed.")
 116            .arg("--image-dir").arg(&image)
 117            .arg("--work-dir").arg(&tmpdir(builder))
 118            .arg("--output-dir").arg(&distdir(builder))
 119            .arg(format!("--package-name={}-{}", name, host))
 120            .arg("--component-name=rust-docs")
 121            .arg("--legacy-manifest-dirs=rustlib,cargo")
 122            .arg("--bulk-dirs=share/doc/rust/html");
 123         builder.run(&mut cmd);
 124         builder.remove_dir(&image);
 125
 126         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 127     }
 128 }
 129
 130 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 131 pub struct RustcDocs {
 132     pub host: Interned<String>,
 133 }
 134
 135 impl Step for RustcDocs {
 136     type Output = PathBuf;
 137     const DEFAULT: bool = true;
 138
 139     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 140         run.path("src/librustc")
 141     }
 142
 143     fn make_run(run: RunConfig<'_>) {
 144         run.builder.ensure(RustcDocs {
 145             host: run.target,
 146         });
 147     }
 148
 149     /// Builds the `rustc-docs` installer component.
 150     fn run(self, builder: &Builder<'_>) -> PathBuf {
 151         let host = self.host;
 152
 153         let name = pkgname(builder, "rustc-docs");
 154
 155         if !builder.config.compiler_docs {
 156             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 157         }
 158
 159         builder.default_doc(None);
 160
 161         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 162         let _ = fs::remove_dir_all(&image);
 163
 164         let dst = image.join("share/doc/rust/html");
 165         t!(fs::create_dir_all(&dst));
 166         let src = builder.compiler_doc_out(host);
 167         builder.cp_r(&src, &dst);
 168
 169         let mut cmd = rust_installer(builder);
 170         cmd.arg("generate")
 171            .arg("--product-name=Rustc-Documentation")
 172            .arg("--rel-manifest-dir=rustlib")
 173            .arg("--success-message=Rustc-documentation-is-installed.")
 174            .arg("--image-dir").arg(&image)
 175            .arg("--work-dir").arg(&tmpdir(builder))
 176            .arg("--output-dir").arg(&distdir(builder))
 177            .arg(format!("--package-name={}-{}", name, host))
 178            .arg("--component-name=rustc-docs")
 179            .arg("--legacy-manifest-dirs=rustlib,cargo")
 180            .arg("--bulk-dirs=share/doc/rust/html");
 181
 182         builder.info(&format!("Dist compiler docs ({})", host));
 183         let _time = timeit(builder);
 184         builder.run(&mut cmd);
 185         builder.remove_dir(&image);
 186
 187         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 188     }
 189 }
 190
 191 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 192     let mut found = Vec::with_capacity(files.len());
 193
 194     for file in files {
 195         let file_path =
 196             path.iter()
 197                 .map(|dir| dir.join(file))
 198                 .find(|p| p.exists());
 199
 200         if let Some(file_path) = file_path {
 201             found.push(file_path);
 202         } else {
 203             panic!("Could not find '{}' in {:?}", file, path);
 204         }
 205     }
 206
 207     found
 208 }
 209
 210 fn make_win_dist(
 211     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 212 ) {
 213     //Ask gcc where it keeps its stuff
 214     let mut cmd = Command::new(builder.cc(target_triple));
 215     cmd.arg("-print-search-dirs");
 216     let gcc_out = output(&mut cmd);
 217
 218     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 219     let mut lib_path = Vec::new();
 220
 221     for line in gcc_out.lines() {
 222         let idx = line.find(':').unwrap();
 223         let key = &line[..idx];
 224         let trim_chars: &[_] = &[' ', '='];
 225         let value =
 226             line[(idx + 1)..]
 227                 .trim_start_matches(trim_chars)
 228                 .split(';')
 229                 .map(PathBuf::from);
 230
 231         if key == "programs" {
 232             bin_path.extend(value);
 233         } else if key == "libraries" {
 234             lib_path.extend(value);
 235         }
 236     }
 237
 238     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 239     let mut rustc_dlls = vec!["libwinpthread-1.dll"];
 240     if target_triple.starts_with("i686-") {
 241         rustc_dlls.push("libgcc_s_dw2-1.dll");
 242     } else {
 243         rustc_dlls.push("libgcc_s_seh-1.dll");
 244     }
 245
 246     let target_libs = [ //MinGW libs
 247         "libgcc.a",
 248         "libgcc_eh.a",
 249         "libgcc_s.a",
 250         "libm.a",
 251         "libmingw32.a",
 252         "libmingwex.a",
 253         "libstdc++.a",
 254         "libiconv.a",
 255         "libmoldname.a",
 256         "libpthread.a",
 257         //Windows import libs
 258         "libadvapi32.a",
 259         "libbcrypt.a",
 260         "libcomctl32.a",
 261         "libcomdlg32.a",
 262         "libcredui.a",
 263         "libcrypt32.a",
 264         "libdbghelp.a",
 265         "libgdi32.a",
 266         "libimagehlp.a",
 267         "libiphlpapi.a",
 268         "libkernel32.a",
 269         "libmsimg32.a",
 270         "libmsvcrt.a",
 271         "libodbc32.a",
 272         "libole32.a",
 273         "liboleaut32.a",
 274         "libopengl32.a",
 275         "libpsapi.a",
 276         "librpcrt4.a",
 277         "libsecur32.a",
 278         "libsetupapi.a",
 279         "libshell32.a",
 280         "libsynchronization.a",
 281         "libuser32.a",
 282         "libuserenv.a",
 283         "libuuid.a",
 284         "libwinhttp.a",
 285         "libwinmm.a",
 286         "libwinspool.a",
 287         "libws2_32.a",
 288         "libwsock32.a",
 289     ];
 290
 291     //Find mingw artifacts we want to bundle
 292     let target_tools = find_files(&target_tools, &bin_path);
 293     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 294     let target_libs = find_files(&target_libs, &lib_path);
 295
 296     // Copy runtime dlls next to rustc.exe
 297     let dist_bin_dir = rust_root.join("bin/");
 298     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 299     for src in rustc_dlls {
 300         builder.copy_to_folder(&src, &dist_bin_dir);
 301     }
 302
 303     //Copy platform tools to platform-specific bin directory
 304     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 305     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 306     for src in target_tools {
 307         builder.copy_to_folder(&src, &target_bin_dir);
 308     }
 309
 310     // Warn windows-gnu users that the bundled GCC cannot compile C files
 311     builder.create(
 312         &target_bin_dir.join("GCC-WARNING.txt"),
 313         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 314          used as a linker. In order to be able to compile projects containing C code use\
 315          the GCC provided by MinGW or Cygwin."
 316     );
 317
 318     //Copy platform libs to platform-specific lib directory
 319     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 320     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 321     for src in target_libs {
 322         builder.copy_to_folder(&src, &target_lib_dir);
 323     }
 324 }
 325
 326 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 327 pub struct Mingw {
 328     pub host: Interned<String>,
 329 }
 330
 331 impl Step for Mingw {
 332     type Output = Option<PathBuf>;
 333     const DEFAULT: bool = true;
 334
 335     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 336         run.never()
 337     }
 338
 339     fn make_run(run: RunConfig<'_>) {
 340         run.builder.ensure(Mingw { host: run.target });
 341     }
 342
 343     /// Builds the `rust-mingw` installer component.
 344     ///
 345     /// This contains all the bits and pieces to run the MinGW Windows targets
 346     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 347     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 348         let host = self.host;
 349
 350         if !host.contains("pc-windows-gnu") {
 351             return None;
 352         }
 353
 354         builder.info(&format!("Dist mingw ({})", host));
 355         let _time = timeit(builder);
 356         let name = pkgname(builder, "rust-mingw");
 357         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 358         let _ = fs::remove_dir_all(&image);
 359         t!(fs::create_dir_all(&image));
 360
 361         // The first argument is a "temporary directory" which is just
 362         // thrown away (this contains the runtime DLLs included in the rustc package
 363         // above) and the second argument is where to place all the MinGW components
 364         // (which is what we want).
 365         make_win_dist(&tmpdir(builder), &image, host, &builder);
 366
 367         let mut cmd = rust_installer(builder);
 368         cmd.arg("generate")
 369            .arg("--product-name=Rust-MinGW")
 370            .arg("--rel-manifest-dir=rustlib")
 371            .arg("--success-message=Rust-MinGW-is-installed.")
 372            .arg("--image-dir").arg(&image)
 373            .arg("--work-dir").arg(&tmpdir(builder))
 374            .arg("--output-dir").arg(&distdir(builder))
 375            .arg(format!("--package-name={}-{}", name, host))
 376            .arg("--component-name=rust-mingw")
 377            .arg("--legacy-manifest-dirs=rustlib,cargo");
 378         builder.run(&mut cmd);
 379         t!(fs::remove_dir_all(&image));
 380         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 381     }
 382 }
 383
 384 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 385 pub struct Rustc {
 386     pub compiler: Compiler,
 387 }
 388
 389 impl Step for Rustc {
 390     type Output = PathBuf;
 391     const DEFAULT: bool = true;
 392     const ONLY_HOSTS: bool = true;
 393
 394     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 395         run.path("src/librustc")
 396     }
 397
 398     fn make_run(run: RunConfig<'_>) {
 399         run.builder.ensure(Rustc {
 400             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 401         });
 402     }
 403
 404     /// Creates the `rustc` installer component.
 405     fn run(self, builder: &Builder<'_>) -> PathBuf {
 406         let compiler = self.compiler;
 407         let host = self.compiler.host;
 408
 409         let name = pkgname(builder, "rustc");
 410         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 411         let _ = fs::remove_dir_all(&image);
 412         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 413         let _ = fs::remove_dir_all(&overlay);
 414
 415         // Prepare the rustc "image", what will actually end up getting installed
 416         prepare_image(builder, compiler, &image);
 417
 418         // Prepare the overlay which is part of the tarball but won't actually be
 419         // installed
 420         let cp = |file: &str| {
 421             builder.install(&builder.src.join(file), &overlay, 0o644);
 422         };
 423         cp("COPYRIGHT");
 424         cp("LICENSE-APACHE");
 425         cp("LICENSE-MIT");
 426         cp("README.md");
 427         // tiny morsel of metadata is used by rust-packaging
 428         let version = builder.rust_version();
 429         builder.create(&overlay.join("version"), &version);
 430         if let Some(sha) = builder.rust_sha() {
 431             builder.create(&overlay.join("git-commit-hash"), &sha);
 432         }
 433
 434         // On MinGW we've got a few runtime DLL dependencies that we need to
 435         // include. The first argument to this script is where to put these DLLs
 436         // (the image we're creating), and the second argument is a junk directory
 437         // to ignore all other MinGW stuff the script creates.
 438         //
 439         // On 32-bit MinGW we're always including a DLL which needs some extra
 440         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 441         // anything requiring us to distribute a license, but it's likely the
 442         // install will *also* include the rust-mingw package, which also needs
 443         // licenses, so to be safe we just include it here in all MinGW packages.
 444         if host.contains("pc-windows-gnu") {
 445             make_win_dist(&image, &tmpdir(builder), host, builder);
 446
 447             let dst = image.join("share/doc");
 448             t!(fs::create_dir_all(&dst));
 449             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 450         }
 451
 452         // Finally, wrap everything up in a nice tarball!
 453         let mut cmd = rust_installer(builder);
 454         cmd.arg("generate")
 455            .arg("--product-name=Rust")
 456            .arg("--rel-manifest-dir=rustlib")
 457            .arg("--success-message=Rust-is-ready-to-roll.")
 458            .arg("--image-dir").arg(&image)
 459            .arg("--work-dir").arg(&tmpdir(builder))
 460            .arg("--output-dir").arg(&distdir(builder))
 461            .arg("--non-installed-overlay").arg(&overlay)
 462            .arg(format!("--package-name={}-{}", name, host))
 463            .arg("--component-name=rustc")
 464            .arg("--legacy-manifest-dirs=rustlib,cargo");
 465
 466         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 467         let _time = timeit(builder);
 468         builder.run(&mut cmd);
 469         builder.remove_dir(&image);
 470         builder.remove_dir(&overlay);
 471
 472         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 473
 474         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 475             let host = compiler.host;
 476             let src = builder.sysroot(compiler);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 483
 484             let libdir_relative = builder.libdir_relative(compiler);
 485
 486             // Copy runtime DLLs needed by the compiler
 487             if libdir_relative.to_str() != Some("bin") {
 488                 let libdir = builder.rustc_libdir(compiler);
 489                 for entry in builder.read_dir(&libdir) {
 490                     let name = entry.file_name();
 491                     if let Some(s) = name.to_str() {
 492                         if is_dylib(s) {
 493                             // Don't use custom libdir here because ^lib/ will be resolved again
 494                             // with installer
 495                             builder.install(&entry.path(), &image.join("lib"), 0o644);
 496                         }
 497                     }
 498                 }
 499             }
 500
 501             // Copy over the codegen backends
 502             let backends_src = builder.sysroot_codegen_backends(compiler);
 503             let backends_rel = backends_src.strip_prefix(&src).unwrap()
 504                 .strip_prefix(builder.sysroot_libdir_relative(compiler)).unwrap();
 505             // Don't use custom libdir here because ^lib/ will be resolved again with installer
 506             let backends_dst = image.join("lib").join(&backends_rel);
 507
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539
 540             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 541             let time = env::var("SOURCE_DATE_EPOCH")
 542                 .map(|timestamp| {
 543                     let epoch = timestamp.parse().map_err(|err| {
 544                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 545                     }).unwrap();
 546
 547                     time::at(Timespec::new(epoch, 0))
 548                 })
 549                 .unwrap_or_else(|_| time::now());
 550
 551             let month_year = t!(time::strftime("%B %Y", &time));
 552             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 553             // to hardlink, and we don't want to edit the source templates
 554             for file_entry in builder.read_dir(&man_src) {
 555                 let page_src = file_entry.path();
 556                 let page_dst = man_dst.join(file_entry.file_name());
 557                 t!(fs::copy(&page_src, &page_dst));
 558                 // template in month/year and version number
 559                 builder.replace_in_file(&page_dst,
 560                                 &[("<INSERT DATE HERE>", &month_year),
 561                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 562             }
 563
 564             // Debugger scripts
 565             builder.ensure(DebuggerScripts {
 566                 sysroot: INTERNER.intern_path(image.to_owned()),
 567                 host,
 568             });
 569
 570             // Misc license info
 571             let cp = |file: &str| {
 572                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 573             };
 574             cp("COPYRIGHT");
 575             cp("LICENSE-APACHE");
 576             cp("LICENSE-MIT");
 577             cp("README.md");
 578         }
 579     }
 580 }
 581
 582 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 583 pub struct DebuggerScripts {
 584     pub sysroot: Interned<PathBuf>,
 585     pub host: Interned<String>,
 586 }
 587
 588 impl Step for DebuggerScripts {
 589     type Output = ();
 590
 591     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 592         run.path("src/lldb_batchmode.py")
 593     }
 594
 595     fn make_run(run: RunConfig<'_>) {
 596         run.builder.ensure(DebuggerScripts {
 597             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 598             host: run.target,
 599         });
 600     }
 601
 602     /// Copies debugger scripts for `target` into the `sysroot` specified.
 603     fn run(self, builder: &Builder<'_>) {
 604         let host = self.host;
 605         let sysroot = self.sysroot;
 606         let dst = sysroot.join("lib/rustlib/etc");
 607         t!(fs::create_dir_all(&dst));
 608         let cp_debugger_script = |file: &str| {
 609             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 610         };
 611         if host.contains("windows-msvc") {
 612             // windbg debugger scripts
 613             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 614                 0o755);
 615
 616             cp_debugger_script("natvis/intrinsic.natvis");
 617             cp_debugger_script("natvis/liballoc.natvis");
 618             cp_debugger_script("natvis/libcore.natvis");
 619         } else {
 620             cp_debugger_script("debugger_pretty_printers_common.py");
 621
 622             // gdb debugger scripts
 623             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 624                     0o755);
 625             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 626                     0o755);
 627
 628             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 629             cp_debugger_script("gdb_rust_pretty_printing.py");
 630
 631             // lldb debugger scripts
 632             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 633                     0o755);
 634
 635             cp_debugger_script("lldb_rust_formatters.py");
 636         }
 637     }
 638 }
 639
 640 fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool {
 641     // The only true set of target libraries came from the build triple, so
 642     // let's reduce redundant work by only producing archives from that host.
 643     if compiler.host != builder.config.build {
 644         builder.info("\tskipping, not a build host");
 645         true
 646     } else {
 647         false
 648     }
 649 }
 650
 651 /// Copy stamped files into an image's `target/lib` directory.
 652 fn copy_target_libs(builder: &Builder<'_>, target: &str, image: &Path, stamp: &Path) {
 653     let dst = image.join("lib/rustlib").join(target).join("lib");
 654     t!(fs::create_dir_all(&dst));
 655     for (path, host) in builder.read_stamp_file(stamp) {
 656         if !host || builder.config.build == target {
 657             builder.copy(&path, &dst.join(path.file_name().unwrap()));
 658         }
 659     }
 660 }
 661
 662 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 663 pub struct Std {
 664     pub compiler: Compiler,
 665     pub target: Interned<String>,
 666 }
 667
 668 impl Step for Std {
 669     type Output = PathBuf;
 670     const DEFAULT: bool = true;
 671
 672     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 673         run.path("src/libstd")
 674     }
 675
 676     fn make_run(run: RunConfig<'_>) {
 677         run.builder.ensure(Std {
 678             compiler: run.builder.compiler_for(
 679                 run.builder.top_stage,
 680                 run.builder.config.build,
 681                 run.target,
 682             ),
 683             target: run.target,
 684         });
 685     }
 686
 687     fn run(self, builder: &Builder<'_>) -> PathBuf {
 688         let compiler = self.compiler;
 689         let target = self.target;
 690
 691         let name = pkgname(builder, "rust-std");
 692         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 693         if skip_host_target_lib(builder, compiler) {
 694             return archive;
 695         }
 696
 697         builder.ensure(compile::Std { compiler, target });
 698
 699         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 700         let _ = fs::remove_dir_all(&image);
 701
 702         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 703         let stamp = compile::libstd_stamp(builder, compiler_to_use, target);
 704         copy_target_libs(builder, &target, &image, &stamp);
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717
 718         builder.info(&format!("Dist std stage{} ({} -> {})",
 719             compiler.stage, &compiler.host, target));
 720         let _time = timeit(builder);
 721         builder.run(&mut cmd);
 722         builder.remove_dir(&image);
 723         archive
 724     }
 725 }
 726
 727 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 728 pub struct RustcDev {
 729     pub compiler: Compiler,
 730     pub target: Interned<String>,
 731 }
 732
 733 impl Step for RustcDev {
 734     type Output = PathBuf;
 735     const DEFAULT: bool = true;
 736     const ONLY_HOSTS: bool = true;
 737
 738     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 739         run.path("rustc-dev")
 740     }
 741
 742     fn make_run(run: RunConfig<'_>) {
 743         run.builder.ensure(RustcDev {
 744             compiler: run.builder.compiler_for(
 745                 run.builder.top_stage,
 746                 run.builder.config.build,
 747                 run.target,
 748             ),
 749             target: run.target,
 750         });
 751     }
 752
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756
 757         let name = pkgname(builder, "rustc-dev");
 758         let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 759         if skip_host_target_lib(builder, compiler) {
 760             return archive;
 761         }
 762
 763         builder.ensure(compile::Rustc { compiler, target });
 764
 765         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 766         let _ = fs::remove_dir_all(&image);
 767
 768         let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
 769         let stamp = compile::librustc_stamp(builder, compiler_to_use, target);
 770         copy_target_libs(builder, &target, &image, &stamp);
 771
 772         let mut cmd = rust_installer(builder);
 773         cmd.arg("generate")
 774            .arg("--product-name=Rust")
 775            .arg("--rel-manifest-dir=rustlib")
 776            .arg("--success-message=Rust-is-ready-to-develop.")
 777            .arg("--image-dir").arg(&image)
 778            .arg("--work-dir").arg(&tmpdir(builder))
 779            .arg("--output-dir").arg(&distdir(builder))
 780            .arg(format!("--package-name={}-{}", name, target))
 781            .arg(format!("--component-name=rustc-dev-{}", target))
 782            .arg("--legacy-manifest-dirs=rustlib,cargo");
 783
 784         builder.info(&format!("Dist rustc-dev stage{} ({} -> {})",
 785             compiler.stage, &compiler.host, target));
 786         let _time = timeit(builder);
 787         builder.run(&mut cmd);
 788         builder.remove_dir(&image);
 789         archive
 790     }
 791 }
 792
 793 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 794 pub struct Analysis {
 795     pub compiler: Compiler,
 796     pub target: Interned<String>,
 797 }
 798
 799 impl Step for Analysis {
 800     type Output = PathBuf;
 801     const DEFAULT: bool = true;
 802
 803     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 804         let builder = run.builder;
 805         run.path("analysis").default_condition(builder.config.extended)
 806     }
 807
 808     fn make_run(run: RunConfig<'_>) {
 809         run.builder.ensure(Analysis {
 810             // Find the actual compiler (handling the full bootstrap option) which
 811             // produced the save-analysis data because that data isn't copied
 812             // through the sysroot uplifting.
 813             compiler: run.builder.compiler_for(
 814                 run.builder.top_stage,
 815                 run.builder.config.build,
 816                 run.target,
 817             ),
 818             target: run.target,
 819         });
 820     }
 821
 822     /// Creates a tarball of save-analysis metadata, if available.
 823     fn run(self, builder: &Builder<'_>) -> PathBuf {
 824         let compiler = self.compiler;
 825         let target = self.target;
 826         assert!(builder.config.extended);
 827         let name = pkgname(builder, "rust-analysis");
 828
 829         if &compiler.host != builder.config.build {
 830             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 831         }
 832
 833         builder.ensure(compile::Std { compiler, target });
 834
 835         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 836
 837         let src = builder.stage_out(compiler, Mode::Std)
 838             .join(target).join(builder.cargo_dir()).join("deps");
 839
 840         let image_src = src.join("save-analysis");
 841         let dst = image.join("lib/rustlib").join(target).join("analysis");
 842         t!(fs::create_dir_all(&dst));
 843         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 844         builder.cp_r(&image_src, &dst);
 845
 846         let mut cmd = rust_installer(builder);
 847         cmd.arg("generate")
 848            .arg("--product-name=Rust")
 849            .arg("--rel-manifest-dir=rustlib")
 850            .arg("--success-message=save-analysis-saved.")
 851            .arg("--image-dir").arg(&image)
 852            .arg("--work-dir").arg(&tmpdir(builder))
 853            .arg("--output-dir").arg(&distdir(builder))
 854            .arg(format!("--package-name={}-{}", name, target))
 855            .arg(format!("--component-name=rust-analysis-{}", target))
 856            .arg("--legacy-manifest-dirs=rustlib,cargo");
 857
 858         builder.info("Dist analysis");
 859         let _time = timeit(builder);
 860         builder.run(&mut cmd);
 861         builder.remove_dir(&image);
 862         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 863     }
 864 }
 865
 866 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 867     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 868         let spath = match path.to_str() {
 869             Some(path) => path,
 870             None => return false,
 871         };
 872         if spath.ends_with("~") || spath.ends_with(".pyc") {
 873             return false
 874         }
 875
 876         const LLVM_PROJECTS: &[&str] = &[
 877             "llvm-project/clang", "llvm-project\\clang",
 878             "llvm-project/libunwind", "llvm-project\\libunwind",
 879             "llvm-project/lld", "llvm-project\\lld",
 880             "llvm-project/lldb", "llvm-project\\lldb",
 881             "llvm-project/llvm", "llvm-project\\llvm",
 882             "llvm-project/compiler-rt", "llvm-project\\compiler-rt",
 883         ];
 884         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 885             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 886         {
 887             return false;
 888         }
 889
 890         const LLVM_TEST: &[&str] = &[
 891             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 892         ];
 893         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 894             (spath.ends_with(".ll") ||
 895              spath.ends_with(".td") ||
 896              spath.ends_with(".s")) {
 897             return false
 898         }
 899
 900         let full_path = Path::new(dir).join(path);
 901         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 902             return false;
 903         }
 904
 905         let excludes = [
 906             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 907             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 908             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 909             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 910         ];
 911         !path.iter()
 912              .map(|s| s.to_str().unwrap())
 913              .any(|s| excludes.contains(&s))
 914     }
 915
 916     // Copy the directories using our filter
 917     for item in src_dirs {
 918         let dst = &dst_dir.join(item);
 919         t!(fs::create_dir_all(dst));
 920         builder.cp_filtered(
 921             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 922     }
 923 }
 924
 925 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 926 pub struct Src;
 927
 928 impl Step for Src {
 929     /// The output path of the src installer tarball
 930     type Output = PathBuf;
 931     const DEFAULT: bool = true;
 932     const ONLY_HOSTS: bool = true;
 933
 934     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 935         run.path("src")
 936     }
 937
 938     fn make_run(run: RunConfig<'_>) {
 939         run.builder.ensure(Src);
 940     }
 941
 942     /// Creates the `rust-src` installer component
 943     fn run(self, builder: &Builder<'_>) -> PathBuf {
 944         let name = pkgname(builder, "rust-src");
 945         let image = tmpdir(builder).join(format!("{}-image", name));
 946         let _ = fs::remove_dir_all(&image);
 947
 948         let dst = image.join("lib/rustlib/src");
 949         let dst_src = dst.join("rust");
 950         t!(fs::create_dir_all(&dst_src));
 951
 952         let src_files = [
 953             "Cargo.lock",
 954         ];
 955         // This is the reduced set of paths which will become the rust-src component
 956         // (essentially libstd and all of its path dependencies)
 957         let std_src_dirs = [
 958             "src/build_helper",
 959             "src/liballoc",
 960             "src/libcore",
 961             "src/libpanic_abort",
 962             "src/libpanic_unwind",
 963             "src/librustc_asan",
 964             "src/librustc_lsan",
 965             "src/librustc_msan",
 966             "src/librustc_tsan",
 967             "src/libstd",
 968             "src/libunwind",
 969             "src/libtest",
 970             "src/libterm",
 971             "src/libprofiler_builtins",
 972             "src/stdarch",
 973             "src/libproc_macro",
 974             "src/tools/rustc-std-workspace-core",
 975             "src/tools/rustc-std-workspace-alloc",
 976             "src/tools/rustc-std-workspace-std",
 977             "src/librustc",
 978             "src/libsyntax",
 979         ];
 980
 981         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 982         for file in src_files.iter() {
 983             builder.copy(&builder.src.join(file), &dst_src.join(file));
 984         }
 985
 986         // Create source tarball in rust-installer format
 987         let mut cmd = rust_installer(builder);
 988         cmd.arg("generate")
 989            .arg("--product-name=Rust")
 990            .arg("--rel-manifest-dir=rustlib")
 991            .arg("--success-message=Awesome-Source.")
 992            .arg("--image-dir").arg(&image)
 993            .arg("--work-dir").arg(&tmpdir(builder))
 994            .arg("--output-dir").arg(&distdir(builder))
 995            .arg(format!("--package-name={}", name))
 996            .arg("--component-name=rust-src")
 997            .arg("--legacy-manifest-dirs=rustlib,cargo");
 998
 999         builder.info("Dist src");
1000         let _time = timeit(builder);
1001         builder.run(&mut cmd);
1002
1003         builder.remove_dir(&image);
1004         distdir(builder).join(&format!("{}.tar.gz", name))
1005     }
1006 }
1007
1008 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1009 pub struct PlainSourceTarball;
1010
1011 impl Step for PlainSourceTarball {
1012     /// Produces the location of the tarball generated
1013     type Output = PathBuf;
1014     const DEFAULT: bool = true;
1015     const ONLY_HOSTS: bool = true;
1016
1017     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1018         let builder = run.builder;
1019         run.path("src").default_condition(builder.config.rust_dist_src)
1020     }
1021
1022     fn make_run(run: RunConfig<'_>) {
1023         run.builder.ensure(PlainSourceTarball);
1024     }
1025
1026     /// Creates the plain source tarball
1027     fn run(self, builder: &Builder<'_>) -> PathBuf {
1028         // Make sure that the root folder of tarball has the correct name
1029         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
1030         let plain_dst_src = tmpdir(builder).join(&plain_name);
1031         let _ = fs::remove_dir_all(&plain_dst_src);
1032         t!(fs::create_dir_all(&plain_dst_src));
1033
1034         // This is the set of root paths which will become part of the source package
1035         let src_files = [
1036             "COPYRIGHT",
1037             "LICENSE-APACHE",
1038             "LICENSE-MIT",
1039             "CONTRIBUTING.md",
1040             "README.md",
1041             "RELEASES.md",
1042             "configure",
1043             "x.py",
1044             "config.toml.example",
1045             "Cargo.toml",
1046             "Cargo.lock",
1047         ];
1048         let src_dirs = [
1049             "src",
1050         ];
1051
1052         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
1053
1054         // Copy the files normally
1055         for item in &src_files {
1056             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
1057         }
1058
1059         // Create the version file
1060         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
1061         if let Some(sha) = builder.rust_sha() {
1062             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
1063         }
1064
1065         // If we're building from git sources, we need to vendor a complete distribution.
1066         if builder.rust_info.is_git() {
1067             // Vendor all Cargo dependencies
1068             let mut cmd = Command::new(&builder.initial_cargo);
1069             cmd.arg("vendor")
1070                .current_dir(&plain_dst_src);
1071             builder.run(&mut cmd);
1072         }
1073
1074         // Create plain source tarball
1075         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1076         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1077         tarball.set_extension(""); // strip .gz
1078         tarball.set_extension(""); // strip .tar
1079         if let Some(dir) = tarball.parent() {
1080             builder.create_dir(&dir);
1081         }
1082         builder.info("running installer");
1083         let mut cmd = rust_installer(builder);
1084         cmd.arg("tarball")
1085            .arg("--input").arg(&plain_name)
1086            .arg("--output").arg(&tarball)
1087            .arg("--work-dir=.")
1088            .current_dir(tmpdir(builder));
1089
1090         builder.info("Create plain source tarball");
1091         let _time = timeit(builder);
1092         builder.run(&mut cmd);
1093         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1094     }
1095 }
1096
1097 // We have to run a few shell scripts, which choke quite a bit on both `\`
1098 // characters and on `C:\` paths, so normalize both of them away.
1099 pub fn sanitize_sh(path: &Path) -> String {
1100     let path = path.to_str().unwrap().replace("\\", "/");
1101     return change_drive(&path).unwrap_or(path);
1102
1103     fn change_drive(s: &str) -> Option<String> {
1104         let mut ch = s.chars();
1105         let drive = ch.next().unwrap_or('C');
1106         if ch.next() != Some(':') {
1107             return None
1108         }
1109         if ch.next() != Some('/') {
1110             return None
1111         }
1112         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1113     }
1114 }
1115
1116 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1117 pub struct Cargo {
1118     pub compiler: Compiler,
1119     pub target: Interned<String>,
1120 }
1121
1122 impl Step for Cargo {
1123     type Output = PathBuf;
1124     const ONLY_HOSTS: bool = true;
1125
1126     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1127         run.path("cargo")
1128     }
1129
1130     fn make_run(run: RunConfig<'_>) {
1131         run.builder.ensure(Cargo {
1132             compiler: run.builder.compiler_for(
1133                 run.builder.top_stage,
1134                 run.builder.config.build,
1135                 run.target,
1136             ),
1137             target: run.target,
1138         });
1139     }
1140
1141     fn run(self, builder: &Builder<'_>) -> PathBuf {
1142         let compiler = self.compiler;
1143         let target = self.target;
1144
1145         let src = builder.src.join("src/tools/cargo");
1146         let etc = src.join("src/etc");
1147         let release_num = builder.release_num("cargo");
1148         let name = pkgname(builder, "cargo");
1149         let version = builder.cargo_info.version(builder, &release_num);
1150
1151         let tmp = tmpdir(builder);
1152         let image = tmp.join("cargo-image");
1153         drop(fs::remove_dir_all(&image));
1154         builder.create_dir(&image);
1155
1156         // Prepare the image directory
1157         builder.create_dir(&image.join("share/zsh/site-functions"));
1158         builder.create_dir(&image.join("etc/bash_completion.d"));
1159         let cargo = builder.ensure(tool::Cargo { compiler, target });
1160         builder.install(&cargo, &image.join("bin"), 0o755);
1161         for man in t!(etc.join("man").read_dir()) {
1162             let man = t!(man);
1163             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1164         }
1165         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1166         builder.copy(&etc.join("cargo.bashcomp.sh"),
1167              &image.join("etc/bash_completion.d/cargo"));
1168         let doc = image.join("share/doc/cargo");
1169         builder.install(&src.join("README.md"), &doc, 0o644);
1170         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1171         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1172         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1173
1174         // Prepare the overlay
1175         let overlay = tmp.join("cargo-overlay");
1176         drop(fs::remove_dir_all(&overlay));
1177         builder.create_dir(&overlay);
1178         builder.install(&src.join("README.md"), &overlay, 0o644);
1179         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1180         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1181         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1182         builder.create(&overlay.join("version"), &version);
1183
1184         // Generate the installer tarball
1185         let mut cmd = rust_installer(builder);
1186         cmd.arg("generate")
1187            .arg("--product-name=Rust")
1188            .arg("--rel-manifest-dir=rustlib")
1189            .arg("--success-message=Rust-is-ready-to-roll.")
1190            .arg("--image-dir").arg(&image)
1191            .arg("--work-dir").arg(&tmpdir(builder))
1192            .arg("--output-dir").arg(&distdir(builder))
1193            .arg("--non-installed-overlay").arg(&overlay)
1194            .arg(format!("--package-name={}-{}", name, target))
1195            .arg("--component-name=cargo")
1196            .arg("--legacy-manifest-dirs=rustlib,cargo");
1197
1198         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1199         let _time = timeit(builder);
1200         builder.run(&mut cmd);
1201         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1202     }
1203 }
1204
1205 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1206 pub struct Rls {
1207     pub compiler: Compiler,
1208     pub target: Interned<String>,
1209 }
1210
1211 impl Step for Rls {
1212     type Output = Option<PathBuf>;
1213     const ONLY_HOSTS: bool = true;
1214
1215     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1216         run.path("rls")
1217     }
1218
1219     fn make_run(run: RunConfig<'_>) {
1220         run.builder.ensure(Rls {
1221             compiler: run.builder.compiler_for(
1222                 run.builder.top_stage,
1223                 run.builder.config.build,
1224                 run.target,
1225             ),
1226             target: run.target,
1227         });
1228     }
1229
1230     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1231         let compiler = self.compiler;
1232         let target = self.target;
1233         assert!(builder.config.extended);
1234
1235         let src = builder.src.join("src/tools/rls");
1236         let release_num = builder.release_num("rls");
1237         let name = pkgname(builder, "rls");
1238         let version = builder.rls_info.version(builder, &release_num);
1239
1240         let tmp = tmpdir(builder);
1241         let image = tmp.join("rls-image");
1242         drop(fs::remove_dir_all(&image));
1243         t!(fs::create_dir_all(&image));
1244
1245         // Prepare the image directory
1246         // We expect RLS to build, because we've exited this step above if tool
1247         // state for RLS isn't testing.
1248         let rls = builder.ensure(tool::Rls {
1249             compiler,
1250             target,
1251             extra_features: Vec::new(),
1252         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1253
1254         builder.install(&rls, &image.join("bin"), 0o755);
1255         let doc = image.join("share/doc/rls");
1256         builder.install(&src.join("README.md"), &doc, 0o644);
1257         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1259
1260         // Prepare the overlay
1261         let overlay = tmp.join("rls-overlay");
1262         drop(fs::remove_dir_all(&overlay));
1263         t!(fs::create_dir_all(&overlay));
1264         builder.install(&src.join("README.md"), &overlay, 0o644);
1265         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1266         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1267         builder.create(&overlay.join("version"), &version);
1268
1269         // Generate the installer tarball
1270         let mut cmd = rust_installer(builder);
1271         cmd.arg("generate")
1272            .arg("--product-name=Rust")
1273            .arg("--rel-manifest-dir=rustlib")
1274            .arg("--success-message=RLS-ready-to-serve.")
1275            .arg("--image-dir").arg(&image)
1276            .arg("--work-dir").arg(&tmpdir(builder))
1277            .arg("--output-dir").arg(&distdir(builder))
1278            .arg("--non-installed-overlay").arg(&overlay)
1279            .arg(format!("--package-name={}-{}", name, target))
1280            .arg("--legacy-manifest-dirs=rustlib,cargo")
1281            .arg("--component-name=rls-preview");
1282
1283         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1284         let _time = timeit(builder);
1285         builder.run(&mut cmd);
1286         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1287     }
1288 }
1289
1290 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1291 pub struct Clippy {
1292     pub compiler: Compiler,
1293     pub target: Interned<String>,
1294 }
1295
1296 impl Step for Clippy {
1297     type Output = Option<PathBuf>;
1298     const ONLY_HOSTS: bool = true;
1299
1300     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1301         run.path("clippy")
1302     }
1303
1304     fn make_run(run: RunConfig<'_>) {
1305         run.builder.ensure(Clippy {
1306             compiler: run.builder.compiler_for(
1307                 run.builder.top_stage,
1308                 run.builder.config.build,
1309                 run.target,
1310             ),
1311             target: run.target,
1312         });
1313     }
1314
1315     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1316         let compiler = self.compiler;
1317         let target = self.target;
1318         assert!(builder.config.extended);
1319
1320         let src = builder.src.join("src/tools/clippy");
1321         let release_num = builder.release_num("clippy");
1322         let name = pkgname(builder, "clippy");
1323         let version = builder.clippy_info.version(builder, &release_num);
1324
1325         let tmp = tmpdir(builder);
1326         let image = tmp.join("clippy-image");
1327         drop(fs::remove_dir_all(&image));
1328         builder.create_dir(&image);
1329
1330         // Prepare the image directory
1331         // We expect clippy to build, because we've exited this step above if tool
1332         // state for clippy isn't testing.
1333         let clippy = builder.ensure(tool::Clippy {
1334             compiler,
1335             target,
1336             extra_features: Vec::new(),
1337         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1338         let cargoclippy = builder.ensure(tool::CargoClippy {
1339             compiler,
1340             target, extra_features: Vec::new()
1341         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1342
1343         builder.install(&clippy, &image.join("bin"), 0o755);
1344         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1345         let doc = image.join("share/doc/clippy");
1346         builder.install(&src.join("README.md"), &doc, 0o644);
1347         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1348         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1349
1350         // Prepare the overlay
1351         let overlay = tmp.join("clippy-overlay");
1352         drop(fs::remove_dir_all(&overlay));
1353         t!(fs::create_dir_all(&overlay));
1354         builder.install(&src.join("README.md"), &overlay, 0o644);
1355         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1356         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1357         builder.create(&overlay.join("version"), &version);
1358
1359         // Generate the installer tarball
1360         let mut cmd = rust_installer(builder);
1361         cmd.arg("generate")
1362            .arg("--product-name=Rust")
1363            .arg("--rel-manifest-dir=rustlib")
1364            .arg("--success-message=clippy-ready-to-serve.")
1365            .arg("--image-dir").arg(&image)
1366            .arg("--work-dir").arg(&tmpdir(builder))
1367            .arg("--output-dir").arg(&distdir(builder))
1368            .arg("--non-installed-overlay").arg(&overlay)
1369            .arg(format!("--package-name={}-{}", name, target))
1370            .arg("--legacy-manifest-dirs=rustlib,cargo")
1371            .arg("--component-name=clippy-preview");
1372
1373         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1374         let _time = timeit(builder);
1375         builder.run(&mut cmd);
1376         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1377     }
1378 }
1379
1380 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1381 pub struct Miri {
1382     pub compiler: Compiler,
1383     pub target: Interned<String>,
1384 }
1385
1386 impl Step for Miri {
1387     type Output = Option<PathBuf>;
1388     const ONLY_HOSTS: bool = true;
1389
1390     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1391         run.path("miri")
1392     }
1393
1394     fn make_run(run: RunConfig<'_>) {
1395         run.builder.ensure(Miri {
1396             compiler: run.builder.compiler_for(
1397                 run.builder.top_stage,
1398                 run.builder.config.build,
1399                 run.target,
1400             ),
1401             target: run.target,
1402         });
1403     }
1404
1405     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1406         let compiler = self.compiler;
1407         let target = self.target;
1408         assert!(builder.config.extended);
1409
1410         let src = builder.src.join("src/tools/miri");
1411         let release_num = builder.release_num("miri");
1412         let name = pkgname(builder, "miri");
1413         let version = builder.miri_info.version(builder, &release_num);
1414
1415         let tmp = tmpdir(builder);
1416         let image = tmp.join("miri-image");
1417         drop(fs::remove_dir_all(&image));
1418         builder.create_dir(&image);
1419
1420         // Prepare the image directory
1421         // We expect miri to build, because we've exited this step above if tool
1422         // state for miri isn't testing.
1423         let miri = builder.ensure(tool::Miri {
1424             compiler,
1425             target,
1426             extra_features: Vec::new(),
1427         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1428         let cargomiri = builder.ensure(tool::CargoMiri {
1429             compiler,
1430             target,
1431             extra_features: Vec::new()
1432         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1433
1434         builder.install(&miri, &image.join("bin"), 0o755);
1435         builder.install(&cargomiri, &image.join("bin"), 0o755);
1436         let doc = image.join("share/doc/miri");
1437         builder.install(&src.join("README.md"), &doc, 0o644);
1438         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1439         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1440
1441         // Prepare the overlay
1442         let overlay = tmp.join("miri-overlay");
1443         drop(fs::remove_dir_all(&overlay));
1444         t!(fs::create_dir_all(&overlay));
1445         builder.install(&src.join("README.md"), &overlay, 0o644);
1446         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1447         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1448         builder.create(&overlay.join("version"), &version);
1449
1450         // Generate the installer tarball
1451         let mut cmd = rust_installer(builder);
1452         cmd.arg("generate")
1453            .arg("--product-name=Rust")
1454            .arg("--rel-manifest-dir=rustlib")
1455            .arg("--success-message=miri-ready-to-serve.")
1456            .arg("--image-dir").arg(&image)
1457            .arg("--work-dir").arg(&tmpdir(builder))
1458            .arg("--output-dir").arg(&distdir(builder))
1459            .arg("--non-installed-overlay").arg(&overlay)
1460            .arg(format!("--package-name={}-{}", name, target))
1461            .arg("--legacy-manifest-dirs=rustlib,cargo")
1462            .arg("--component-name=miri-preview");
1463
1464         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1465         let _time = timeit(builder);
1466         builder.run(&mut cmd);
1467         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1468     }
1469 }
1470
1471 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1472 pub struct Rustfmt {
1473     pub compiler: Compiler,
1474     pub target: Interned<String>,
1475 }
1476
1477 impl Step for Rustfmt {
1478     type Output = Option<PathBuf>;
1479     const ONLY_HOSTS: bool = true;
1480
1481     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1482         run.path("rustfmt")
1483     }
1484
1485     fn make_run(run: RunConfig<'_>) {
1486         run.builder.ensure(Rustfmt {
1487             compiler: run.builder.compiler_for(
1488                 run.builder.top_stage,
1489                 run.builder.config.build,
1490                 run.target,
1491             ),
1492             target: run.target,
1493         });
1494     }
1495
1496     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1497         let compiler = self.compiler;
1498         let target = self.target;
1499
1500         let src = builder.src.join("src/tools/rustfmt");
1501         let release_num = builder.release_num("rustfmt");
1502         let name = pkgname(builder, "rustfmt");
1503         let version = builder.rustfmt_info.version(builder, &release_num);
1504
1505         let tmp = tmpdir(builder);
1506         let image = tmp.join("rustfmt-image");
1507         drop(fs::remove_dir_all(&image));
1508         builder.create_dir(&image);
1509
1510         // Prepare the image directory
1511         let rustfmt = builder.ensure(tool::Rustfmt {
1512             compiler,
1513             target,
1514             extra_features: Vec::new(),
1515         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1516         let cargofmt = builder.ensure(tool::Cargofmt {
1517             compiler,
1518             target,
1519             extra_features: Vec::new(),
1520         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1521
1522         builder.install(&rustfmt, &image.join("bin"), 0o755);
1523         builder.install(&cargofmt, &image.join("bin"), 0o755);
1524         let doc = image.join("share/doc/rustfmt");
1525         builder.install(&src.join("README.md"), &doc, 0o644);
1526         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1527         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1528
1529         // Prepare the overlay
1530         let overlay = tmp.join("rustfmt-overlay");
1531         drop(fs::remove_dir_all(&overlay));
1532         builder.create_dir(&overlay);
1533         builder.install(&src.join("README.md"), &overlay, 0o644);
1534         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1535         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1536         builder.create(&overlay.join("version"), &version);
1537
1538         // Generate the installer tarball
1539         let mut cmd = rust_installer(builder);
1540         cmd.arg("generate")
1541            .arg("--product-name=Rust")
1542            .arg("--rel-manifest-dir=rustlib")
1543            .arg("--success-message=rustfmt-ready-to-fmt.")
1544            .arg("--image-dir").arg(&image)
1545            .arg("--work-dir").arg(&tmpdir(builder))
1546            .arg("--output-dir").arg(&distdir(builder))
1547            .arg("--non-installed-overlay").arg(&overlay)
1548            .arg(format!("--package-name={}-{}", name, target))
1549            .arg("--legacy-manifest-dirs=rustlib,cargo")
1550            .arg("--component-name=rustfmt-preview");
1551
1552         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1553         let _time = timeit(builder);
1554         builder.run(&mut cmd);
1555         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1556     }
1557 }
1558
1559 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1560 pub struct Extended {
1561     stage: u32,
1562     host: Interned<String>,
1563     target: Interned<String>,
1564 }
1565
1566 impl Step for Extended {
1567     type Output = ();
1568     const DEFAULT: bool = true;
1569     const ONLY_HOSTS: bool = true;
1570
1571     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1572         let builder = run.builder;
1573         run.path("extended").default_condition(builder.config.extended)
1574     }
1575
1576     fn make_run(run: RunConfig<'_>) {
1577         run.builder.ensure(Extended {
1578             stage: run.builder.top_stage,
1579             host: run.builder.config.build,
1580             target: run.target,
1581         });
1582     }
1583
1584     /// Creates a combined installer for the specified target in the provided stage.
1585     fn run(self, builder: &Builder<'_>) {
1586         let target = self.target;
1587         let stage = self.stage;
1588         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1589
1590         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1591
1592         let rustc_installer = builder.ensure(Rustc {
1593             compiler: builder.compiler(stage, target),
1594         });
1595         let cargo_installer = builder.ensure(Cargo { compiler, target });
1596         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1597         let rls_installer = builder.ensure(Rls { compiler, target });
1598         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1599         let clippy_installer = builder.ensure(Clippy { compiler, target });
1600         let miri_installer = builder.ensure(Miri { compiler, target });
1601         let lldb_installer = builder.ensure(Lldb { target });
1602         let mingw_installer = builder.ensure(Mingw { host: target });
1603         let analysis_installer = builder.ensure(Analysis { compiler, target });
1604
1605         let docs_installer = builder.ensure(Docs { host: target, });
1606         let std_installer = builder.ensure(Std {
1607             compiler: builder.compiler(stage, target),
1608             target,
1609         });
1610
1611         let tmp = tmpdir(builder);
1612         let overlay = tmp.join("extended-overlay");
1613         let etc = builder.src.join("src/etc/installer");
1614         let work = tmp.join("work");
1615
1616         let _ = fs::remove_dir_all(&overlay);
1617         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1618         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1619         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1620         let version = builder.rust_version();
1621         builder.create(&overlay.join("version"), &version);
1622         if let Some(sha) = builder.rust_sha() {
1623             builder.create(&overlay.join("git-commit-hash"), &sha);
1624         }
1625         builder.install(&etc.join("README.md"), &overlay, 0o644);
1626
1627         // When rust-std package split from rustc, we needed to ensure that during
1628         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1629         // the std files during uninstall. To do this ensure that rustc comes
1630         // before rust-std in the list below.
1631         let mut tarballs = Vec::new();
1632         tarballs.push(rustc_installer);
1633         tarballs.push(cargo_installer);
1634         tarballs.extend(rls_installer.clone());
1635         tarballs.extend(clippy_installer.clone());
1636         tarballs.extend(miri_installer.clone());
1637         tarballs.extend(rustfmt_installer.clone());
1638         tarballs.extend(llvm_tools_installer);
1639         tarballs.extend(lldb_installer);
1640         tarballs.push(analysis_installer);
1641         tarballs.push(std_installer);
1642         if builder.config.docs {
1643             tarballs.push(docs_installer);
1644         }
1645         if target.contains("pc-windows-gnu") {
1646             tarballs.push(mingw_installer.unwrap());
1647         }
1648         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1649         for tarball in &tarballs[1..] {
1650             input_tarballs.push(",");
1651             input_tarballs.push(tarball);
1652         }
1653
1654         builder.info("building combined installer");
1655         let mut cmd = rust_installer(builder);
1656         cmd.arg("combine")
1657             .arg("--product-name=Rust")
1658             .arg("--rel-manifest-dir=rustlib")
1659             .arg("--success-message=Rust-is-ready-to-roll.")
1660             .arg("--work-dir").arg(&work)
1661             .arg("--output-dir").arg(&distdir(builder))
1662             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1663             .arg("--legacy-manifest-dirs=rustlib,cargo")
1664             .arg("--input-tarballs").arg(input_tarballs)
1665             .arg("--non-installed-overlay").arg(&overlay);
1666         let time = timeit(&builder);
1667         builder.run(&mut cmd);
1668         drop(time);
1669
1670         let mut license = String::new();
1671         license += &builder.read(&builder.src.join("COPYRIGHT"));
1672         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1673         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1674         license.push_str("\n");
1675         license.push_str("\n");
1676
1677         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1678         let mut rtf = rtf.to_string();
1679         rtf.push_str("\n");
1680         for line in license.lines() {
1681             rtf.push_str(line);
1682             rtf.push_str("\\line ");
1683         }
1684         rtf.push_str("}");
1685
1686         fn filter(contents: &str, marker: &str) -> String {
1687             let start = format!("tool-{}-start", marker);
1688             let end = format!("tool-{}-end", marker);
1689             let mut lines = Vec::new();
1690             let mut omitted = false;
1691             for line in contents.lines() {
1692                 if line.contains(&start) {
1693                     omitted = true;
1694                 } else if line.contains(&end) {
1695                     omitted = false;
1696                 } else if !omitted {
1697                     lines.push(line);
1698                 }
1699             }
1700
1701             lines.join("\n")
1702         }
1703
1704         let xform = |p: &Path| {
1705             let mut contents = t!(fs::read_to_string(p));
1706             if rls_installer.is_none() {
1707                 contents = filter(&contents, "rls");
1708             }
1709             if clippy_installer.is_none() {
1710                 contents = filter(&contents, "clippy");
1711             }
1712             if miri_installer.is_none() {
1713                 contents = filter(&contents, "miri");
1714             }
1715             if rustfmt_installer.is_none() {
1716                 contents = filter(&contents, "rustfmt");
1717             }
1718             let ret = tmp.join(p.file_name().unwrap());
1719             t!(fs::write(&ret, &contents));
1720             ret
1721         };
1722
1723         if target.contains("apple-darwin") {
1724             builder.info("building pkg installer");
1725             let pkg = tmp.join("pkg");
1726             let _ = fs::remove_dir_all(&pkg);
1727
1728             let pkgbuild = |component: &str| {
1729                 let mut cmd = Command::new("pkgbuild");
1730                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1731                     .arg("--scripts").arg(pkg.join(component))
1732                     .arg("--nopayload")
1733                     .arg(pkg.join(component).with_extension("pkg"));
1734                 builder.run(&mut cmd);
1735             };
1736
1737             let prepare = |name: &str| {
1738                 builder.create_dir(&pkg.join(name));
1739                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1740                         &pkg.join(name));
1741                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1742                 pkgbuild(name);
1743             };
1744             prepare("rustc");
1745             prepare("cargo");
1746             prepare("rust-docs");
1747             prepare("rust-std");
1748             prepare("rust-analysis");
1749
1750             if rls_installer.is_some() {
1751                 prepare("rls");
1752             }
1753             if clippy_installer.is_some() {
1754                 prepare("clippy");
1755             }
1756             if miri_installer.is_some() {
1757                 prepare("miri");
1758             }
1759
1760             // create an 'uninstall' package
1761             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1762             pkgbuild("uninstall");
1763
1764             builder.create_dir(&pkg.join("res"));
1765             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1766             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1767             let mut cmd = Command::new("productbuild");
1768             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1769                 .arg("--resources").arg(pkg.join("res"))
1770                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1771                                                     pkgname(builder, "rust"),
1772                                                     target)))
1773                 .arg("--package-path").arg(&pkg);
1774             let _time = timeit(builder);
1775             builder.run(&mut cmd);
1776         }
1777
1778         if target.contains("windows") {
1779             let exe = tmp.join("exe");
1780             let _ = fs::remove_dir_all(&exe);
1781
1782             let prepare = |name: &str| {
1783                 builder.create_dir(&exe.join(name));
1784                 let dir = if name == "rust-std" || name == "rust-analysis" {
1785                     format!("{}-{}", name, target)
1786                 } else if name == "rls" {
1787                     "rls-preview".to_string()
1788                 } else if name == "clippy" {
1789                     "clippy-preview".to_string()
1790                 } else if name == "miri" {
1791                     "miri-preview".to_string()
1792                 } else {
1793                     name.to_string()
1794                 };
1795                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1796                             .join(dir),
1797                         &exe.join(name));
1798                 builder.remove(&exe.join(name).join("manifest.in"));
1799             };
1800             prepare("rustc");
1801             prepare("cargo");
1802             prepare("rust-analysis");
1803             prepare("rust-docs");
1804             prepare("rust-std");
1805             if rls_installer.is_some() {
1806                 prepare("rls");
1807             }
1808             if clippy_installer.is_some() {
1809                 prepare("clippy");
1810             }
1811             if miri_installer.is_some() {
1812                 prepare("miri");
1813             }
1814             if target.contains("windows-gnu") {
1815                 prepare("rust-mingw");
1816             }
1817
1818             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1819             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1820             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1821             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1822             builder.create(&exe.join("LICENSE.txt"), &license);
1823
1824             // Generate exe installer
1825             builder.info("building `exe` installer with `iscc`");
1826             let mut cmd = Command::new("iscc");
1827             cmd.arg("rust.iss")
1828                 .arg("/Q")
1829                 .current_dir(&exe);
1830             if target.contains("windows-gnu") {
1831                 cmd.arg("/dMINGW");
1832             }
1833             add_env(builder, &mut cmd, target);
1834             let time = timeit(builder);
1835             builder.run(&mut cmd);
1836             drop(time);
1837             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1838                     &distdir(builder),
1839                     0o755);
1840
1841             // Generate msi installer
1842             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1843             let heat = wix.join("bin/heat.exe");
1844             let candle = wix.join("bin/candle.exe");
1845             let light = wix.join("bin/light.exe");
1846
1847             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1848             builder.run(Command::new(&heat)
1849                             .current_dir(&exe)
1850                             .arg("dir")
1851                             .arg("rustc")
1852                             .args(&heat_flags)
1853                             .arg("-cg").arg("RustcGroup")
1854                             .arg("-dr").arg("Rustc")
1855                             .arg("-var").arg("var.RustcDir")
1856                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1857             builder.run(Command::new(&heat)
1858                             .current_dir(&exe)
1859                             .arg("dir")
1860                             .arg("rust-docs")
1861                             .args(&heat_flags)
1862                             .arg("-cg").arg("DocsGroup")
1863                             .arg("-dr").arg("Docs")
1864                             .arg("-var").arg("var.DocsDir")
1865                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1866                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1867             builder.run(Command::new(&heat)
1868                             .current_dir(&exe)
1869                             .arg("dir")
1870                             .arg("cargo")
1871                             .args(&heat_flags)
1872                             .arg("-cg").arg("CargoGroup")
1873                             .arg("-dr").arg("Cargo")
1874                             .arg("-var").arg("var.CargoDir")
1875                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1876                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1877             builder.run(Command::new(&heat)
1878                             .current_dir(&exe)
1879                             .arg("dir")
1880                             .arg("rust-std")
1881                             .args(&heat_flags)
1882                             .arg("-cg").arg("StdGroup")
1883                             .arg("-dr").arg("Std")
1884                             .arg("-var").arg("var.StdDir")
1885                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1886             if rls_installer.is_some() {
1887                 builder.run(Command::new(&heat)
1888                                 .current_dir(&exe)
1889                                 .arg("dir")
1890                                 .arg("rls")
1891                                 .args(&heat_flags)
1892                                 .arg("-cg").arg("RlsGroup")
1893                                 .arg("-dr").arg("Rls")
1894                                 .arg("-var").arg("var.RlsDir")
1895                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1896                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1897             }
1898             if clippy_installer.is_some() {
1899                 builder.run(Command::new(&heat)
1900                                 .current_dir(&exe)
1901                                 .arg("dir")
1902                                 .arg("clippy")
1903                                 .args(&heat_flags)
1904                                 .arg("-cg").arg("ClippyGroup")
1905                                 .arg("-dr").arg("Clippy")
1906                                 .arg("-var").arg("var.ClippyDir")
1907                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1908                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1909             }
1910             if miri_installer.is_some() {
1911                 builder.run(Command::new(&heat)
1912                                 .current_dir(&exe)
1913                                 .arg("dir")
1914                                 .arg("miri")
1915                                 .args(&heat_flags)
1916                                 .arg("-cg").arg("MiriGroup")
1917                                 .arg("-dr").arg("Miri")
1918                                 .arg("-var").arg("var.MiriDir")
1919                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1920                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1921             }
1922             builder.run(Command::new(&heat)
1923                             .current_dir(&exe)
1924                             .arg("dir")
1925                             .arg("rust-analysis")
1926                             .args(&heat_flags)
1927                             .arg("-cg").arg("AnalysisGroup")
1928                             .arg("-dr").arg("Analysis")
1929                             .arg("-var").arg("var.AnalysisDir")
1930                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1931                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1932             if target.contains("windows-gnu") {
1933                 builder.run(Command::new(&heat)
1934                                 .current_dir(&exe)
1935                                 .arg("dir")
1936                                 .arg("rust-mingw")
1937                                 .args(&heat_flags)
1938                                 .arg("-cg").arg("GccGroup")
1939                                 .arg("-dr").arg("Gcc")
1940                                 .arg("-var").arg("var.GccDir")
1941                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1942             }
1943
1944             let candle = |input: &Path| {
1945                 let output = exe.join(input.file_stem().unwrap())
1946                                 .with_extension("wixobj");
1947                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1948                 let mut cmd = Command::new(&candle);
1949                 cmd.current_dir(&exe)
1950                     .arg("-nologo")
1951                     .arg("-dRustcDir=rustc")
1952                     .arg("-dDocsDir=rust-docs")
1953                     .arg("-dCargoDir=cargo")
1954                     .arg("-dStdDir=rust-std")
1955                     .arg("-dAnalysisDir=rust-analysis")
1956                     .arg("-arch").arg(&arch)
1957                     .arg("-out").arg(&output)
1958                     .arg(&input);
1959                 add_env(builder, &mut cmd, target);
1960
1961                 if rls_installer.is_some() {
1962                     cmd.arg("-dRlsDir=rls");
1963                 }
1964                 if clippy_installer.is_some() {
1965                     cmd.arg("-dClippyDir=clippy");
1966                 }
1967                 if miri_installer.is_some() {
1968                     cmd.arg("-dMiriDir=miri");
1969                 }
1970                 if target.contains("windows-gnu") {
1971                     cmd.arg("-dGccDir=rust-mingw");
1972                 }
1973                 builder.run(&mut cmd);
1974             };
1975             candle(&xform(&etc.join("msi/rust.wxs")));
1976             candle(&etc.join("msi/ui.wxs"));
1977             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1978             candle("RustcGroup.wxs".as_ref());
1979             candle("DocsGroup.wxs".as_ref());
1980             candle("CargoGroup.wxs".as_ref());
1981             candle("StdGroup.wxs".as_ref());
1982             if rls_installer.is_some() {
1983                 candle("RlsGroup.wxs".as_ref());
1984             }
1985             if clippy_installer.is_some() {
1986                 candle("ClippyGroup.wxs".as_ref());
1987             }
1988             if miri_installer.is_some() {
1989                 candle("MiriGroup.wxs".as_ref());
1990             }
1991             candle("AnalysisGroup.wxs".as_ref());
1992
1993             if target.contains("windows-gnu") {
1994                 candle("GccGroup.wxs".as_ref());
1995             }
1996
1997             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1998             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1999             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
2000
2001             builder.info(&format!("building `msi` installer with {:?}", light));
2002             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
2003             let mut cmd = Command::new(&light);
2004             cmd.arg("-nologo")
2005                 .arg("-ext").arg("WixUIExtension")
2006                 .arg("-ext").arg("WixUtilExtension")
2007                 .arg("-out").arg(exe.join(&filename))
2008                 .arg("rust.wixobj")
2009                 .arg("ui.wixobj")
2010                 .arg("rustwelcomedlg.wixobj")
2011                 .arg("RustcGroup.wixobj")
2012                 .arg("DocsGroup.wixobj")
2013                 .arg("CargoGroup.wixobj")
2014                 .arg("StdGroup.wixobj")
2015                 .arg("AnalysisGroup.wixobj")
2016                 .current_dir(&exe);
2017
2018             if rls_installer.is_some() {
2019                 cmd.arg("RlsGroup.wixobj");
2020             }
2021             if clippy_installer.is_some() {
2022                 cmd.arg("ClippyGroup.wixobj");
2023             }
2024             if miri_installer.is_some() {
2025                 cmd.arg("MiriGroup.wixobj");
2026             }
2027
2028             if target.contains("windows-gnu") {
2029                 cmd.arg("GccGroup.wixobj");
2030             }
2031             // ICE57 wrongly complains about the shortcuts
2032             cmd.arg("-sice:ICE57");
2033
2034             let _time = timeit(builder);
2035             builder.run(&mut cmd);
2036
2037             if !builder.config.dry_run {
2038                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
2039             }
2040         }
2041     }
2042 }
2043
2044 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
2045     let mut parts = channel::CFG_RELEASE_NUM.split('.');
2046     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
2047        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
2048        .env("CFG_RELEASE", builder.rust_release())
2049        .env("CFG_VER_MAJOR", parts.next().unwrap())
2050        .env("CFG_VER_MINOR", parts.next().unwrap())
2051        .env("CFG_VER_PATCH", parts.next().unwrap())
2052        .env("CFG_VER_BUILD", "0") // just needed to build
2053        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
2054        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
2055        .env("CFG_BUILD", target)
2056        .env("CFG_CHANNEL", &builder.config.channel);
2057
2058     if target.contains("windows-gnu") {
2059        cmd.env("CFG_MINGW", "1")
2060           .env("CFG_ABI", "GNU");
2061     } else {
2062        cmd.env("CFG_MINGW", "0")
2063           .env("CFG_ABI", "MSVC");
2064     }
2065
2066     if target.contains("x86_64") {
2067        cmd.env("CFG_PLATFORM", "x64");
2068     } else {
2069        cmd.env("CFG_PLATFORM", "x86");
2070     }
2071 }
2072
2073 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
2074 pub struct HashSign;
2075
2076 impl Step for HashSign {
2077     type Output = ();
2078     const ONLY_HOSTS: bool = true;
2079
2080     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2081         run.path("hash-and-sign")
2082     }
2083
2084     fn make_run(run: RunConfig<'_>) {
2085         run.builder.ensure(HashSign);
2086     }
2087
2088     fn run(self, builder: &Builder<'_>) {
2089         // This gets called by `promote-release`
2090         // (https://github.com/rust-lang/rust-central-station/tree/master/promote-release).
2091         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2092         if builder.config.dry_run {
2093             return;
2094         }
2095         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2096             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2097         });
2098         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2099             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2100         });
2101         let pass = if env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err() {
2102             let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2103                 panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2104             });
2105             t!(fs::read_to_string(&file))
2106         } else {
2107             String::new()
2108         };
2109
2110         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2111
2112         cmd.arg(sign);
2113         cmd.arg(distdir(builder));
2114         cmd.arg(today.trim());
2115         cmd.arg(builder.rust_package_vers());
2116         cmd.arg(addr);
2117         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2118         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2119         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2120         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2121         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2122         cmd.arg(builder.llvm_tools_package_vers());
2123         cmd.arg(builder.lldb_package_vers());
2124
2125         builder.create_dir(&distdir(builder));
2126
2127         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2128         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2129         let status = t!(child.wait());
2130         assert!(status.success());
2131     }
2132 }
2133
2134 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2135 // LLVM tools are linked dynamically.
2136 // Note: This function does no yet support Windows but we also don't support
2137 //       linking LLVM tools dynamically on Windows yet.
2138 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2139                                 target: Interned<String>,
2140                                 sysroot: &Path) {
2141     let src_libdir = builder
2142         .llvm_out(target)
2143         .join("lib");
2144     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2145     t!(fs::create_dir_all(&dst_libdir));
2146
2147     if target.contains("apple-darwin") {
2148         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2149         if llvm_dylib_path.exists() {
2150             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2151         }
2152         return
2153     }
2154
2155     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2156     // Since tools link to the latter rather than the former, we have to
2157     // follow the symlink to find out what to distribute.
2158     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2159     if llvm_dylib_path.exists() {
2160         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2161             panic!("dist: Error calling canonicalize path `{}`: {}",
2162                    llvm_dylib_path.display(), e);
2163         });
2164
2165
2166         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2167     }
2168 }
2169
2170 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2171 pub struct LlvmTools {
2172     pub target: Interned<String>,
2173 }
2174
2175 impl Step for LlvmTools {
2176     type Output = Option<PathBuf>;
2177     const ONLY_HOSTS: bool = true;
2178
2179     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2180         run.path("llvm-tools")
2181     }
2182
2183     fn make_run(run: RunConfig<'_>) {
2184         run.builder.ensure(LlvmTools {
2185             target: run.target,
2186         });
2187     }
2188
2189     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2190         let target = self.target;
2191         assert!(builder.config.extended);
2192
2193         /* run only if llvm-config isn't used */
2194         if let Some(config) = builder.config.target_config.get(&target) {
2195             if let Some(ref _s) = config.llvm_config {
2196                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2197                     target));
2198                 return None;
2199             }
2200         }
2201
2202         builder.info(&format!("Dist LlvmTools ({})", target));
2203         let _time = timeit(builder);
2204         let src = builder.src.join("src/llvm-project/llvm");
2205         let name = pkgname(builder, "llvm-tools");
2206
2207         let tmp = tmpdir(builder);
2208         let image = tmp.join("llvm-tools-image");
2209         drop(fs::remove_dir_all(&image));
2210
2211         // Prepare the image directory
2212         let src_bindir = builder
2213             .llvm_out(target)
2214             .join("bin");
2215         let dst_bindir = image.join("lib/rustlib")
2216             .join(&*target)
2217             .join("bin");
2218         t!(fs::create_dir_all(&dst_bindir));
2219         for tool in LLVM_TOOLS {
2220             let exe = src_bindir.join(exe(tool, &target));
2221             builder.install(&exe, &dst_bindir, 0o755);
2222         }
2223
2224         // Prepare the overlay
2225         let overlay = tmp.join("llvm-tools-overlay");
2226         drop(fs::remove_dir_all(&overlay));
2227         builder.create_dir(&overlay);
2228         builder.install(&src.join("README.txt"), &overlay, 0o644);
2229         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2230         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2231
2232         // Generate the installer tarball
2233         let mut cmd = rust_installer(builder);
2234         cmd.arg("generate")
2235             .arg("--product-name=Rust")
2236             .arg("--rel-manifest-dir=rustlib")
2237             .arg("--success-message=llvm-tools-installed.")
2238             .arg("--image-dir").arg(&image)
2239             .arg("--work-dir").arg(&tmpdir(builder))
2240             .arg("--output-dir").arg(&distdir(builder))
2241             .arg("--non-installed-overlay").arg(&overlay)
2242             .arg(format!("--package-name={}-{}", name, target))
2243             .arg("--legacy-manifest-dirs=rustlib,cargo")
2244             .arg("--component-name=llvm-tools-preview");
2245
2246
2247         builder.run(&mut cmd);
2248         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2249     }
2250 }
2251
2252 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2253 pub struct Lldb {
2254     pub target: Interned<String>,
2255 }
2256
2257 impl Step for Lldb {
2258     type Output = Option<PathBuf>;
2259     const ONLY_HOSTS: bool = true;
2260     const DEFAULT: bool = true;
2261
2262     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2263         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2264     }
2265
2266     fn make_run(run: RunConfig<'_>) {
2267         run.builder.ensure(Lldb {
2268             target: run.target,
2269         });
2270     }
2271
2272     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2273         let target = self.target;
2274
2275         if builder.config.dry_run {
2276             return None;
2277         }
2278
2279         let bindir = builder
2280             .llvm_out(target)
2281             .join("bin");
2282         let lldb_exe = bindir.join(exe("lldb", &target));
2283         if !lldb_exe.exists() {
2284             return None;
2285         }
2286
2287         builder.info(&format!("Dist Lldb ({})", target));
2288         let src = builder.src.join("src/llvm-project/lldb");
2289         let name = pkgname(builder, "lldb");
2290
2291         let tmp = tmpdir(builder);
2292         let image = tmp.join("lldb-image");
2293         drop(fs::remove_dir_all(&image));
2294
2295         // Prepare the image directory
2296         let root = image.join("lib/rustlib").join(&*target);
2297         let dst = root.join("bin");
2298         t!(fs::create_dir_all(&dst));
2299         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2300             let exe = bindir.join(exe(program, &target));
2301             builder.install(&exe, &dst, 0o755);
2302         }
2303
2304         // The libraries.
2305         let libdir = builder.llvm_out(target).join("lib");
2306         let dst = root.join("lib");
2307         t!(fs::create_dir_all(&dst));
2308         for entry in t!(fs::read_dir(&libdir)) {
2309             let entry = entry.unwrap();
2310             if let Ok(name) = entry.file_name().into_string() {
2311                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2312                     if t!(entry.file_type()).is_symlink() {
2313                         builder.copy_to_folder(&entry.path(), &dst);
2314                     } else {
2315                        builder.install(&entry.path(), &dst, 0o755);
2316                     }
2317                 }
2318             }
2319         }
2320
2321         // The lldb scripts might be installed in lib/python$version
2322         // or in lib64/python$version.  If lib64 exists, use it;
2323         // otherwise lib.
2324         let libdir = builder.llvm_out(target).join("lib64");
2325         let (libdir, libdir_name) = if libdir.exists() {
2326             (libdir, "lib64")
2327         } else {
2328             (builder.llvm_out(target).join("lib"), "lib")
2329         };
2330         for entry in t!(fs::read_dir(&libdir)) {
2331             let entry = t!(entry);
2332             if let Ok(name) = entry.file_name().into_string() {
2333                 if name.starts_with("python") {
2334                     let dst = root.join(libdir_name)
2335                         .join(entry.file_name());
2336                     t!(fs::create_dir_all(&dst));
2337                     builder.cp_r(&entry.path(), &dst);
2338                     break;
2339                 }
2340             }
2341         }
2342
2343         // Prepare the overlay
2344         let overlay = tmp.join("lldb-overlay");
2345         drop(fs::remove_dir_all(&overlay));
2346         builder.create_dir(&overlay);
2347         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2348         builder.create(&overlay.join("version"), &builder.lldb_vers());
2349
2350         // Generate the installer tarball
2351         let mut cmd = rust_installer(builder);
2352         cmd.arg("generate")
2353             .arg("--product-name=Rust")
2354             .arg("--rel-manifest-dir=rustlib")
2355             .arg("--success-message=lldb-installed.")
2356             .arg("--image-dir").arg(&image)
2357             .arg("--work-dir").arg(&tmpdir(builder))
2358             .arg("--output-dir").arg(&distdir(builder))
2359             .arg("--non-installed-overlay").arg(&overlay)
2360             .arg(format!("--package-name={}-{}", name, target))
2361             .arg("--legacy-manifest-dirs=rustlib,cargo")
2362             .arg("--component-name=lldb-preview");
2363
2364
2365         builder.run(&mut cmd);
2366         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2367     }
2368 }