src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub stage: u32,
  72     pub host: Interned<String>,
  73 }
  74
  75 impl Step for Docs {
  76     type Output = PathBuf;
  77     const DEFAULT: bool = true;
  78
  79     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  80         run.path("src/doc")
  81     }
  82
  83     fn make_run(run: RunConfig<'_>) {
  84         run.builder.ensure(Docs {
  85             stage: run.builder.top_stage,
  86             host: run.target,
  87         });
  88     }
  89
  90     /// Builds the `rust-docs` installer component.
  91     fn run(self, builder: &Builder<'_>) -> PathBuf {
  92         let host = self.host;
  93
  94         let name = pkgname(builder, "rust-docs");
  95
  96         builder.info(&format!("Dist docs ({})", host));
  97         if !builder.config.docs {
  98             builder.info("\tskipping - docs disabled");
  99             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 100         }
 101
 102         builder.default_doc(None);
 103
 104         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 105         let _ = fs::remove_dir_all(&image);
 106
 107         let dst = image.join("share/doc/rust/html");
 108         t!(fs::create_dir_all(&dst));
 109         let src = builder.doc_out(host);
 110         builder.cp_r(&src, &dst);
 111
 112         let mut cmd = rust_installer(builder);
 113         cmd.arg("generate")
 114            .arg("--product-name=Rust-Documentation")
 115            .arg("--rel-manifest-dir=rustlib")
 116            .arg("--success-message=Rust-documentation-is-installed.")
 117            .arg("--image-dir").arg(&image)
 118            .arg("--work-dir").arg(&tmpdir(builder))
 119            .arg("--output-dir").arg(&distdir(builder))
 120            .arg(format!("--package-name={}-{}", name, host))
 121            .arg("--component-name=rust-docs")
 122            .arg("--legacy-manifest-dirs=rustlib,cargo")
 123            .arg("--bulk-dirs=share/doc/rust/html");
 124         builder.run(&mut cmd);
 125         builder.remove_dir(&image);
 126
 127         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 128     }
 129 }
 130
 131 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 132 pub struct RustcDocs {
 133     pub stage: u32,
 134     pub host: Interned<String>,
 135 }
 136
 137 impl Step for RustcDocs {
 138     type Output = PathBuf;
 139     const DEFAULT: bool = true;
 140
 141     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 142         run.path("src/librustc")
 143     }
 144
 145     fn make_run(run: RunConfig<'_>) {
 146         run.builder.ensure(RustcDocs {
 147             stage: run.builder.top_stage,
 148             host: run.target,
 149         });
 150     }
 151
 152     /// Builds the `rustc-docs` installer component.
 153     fn run(self, builder: &Builder<'_>) -> PathBuf {
 154         let host = self.host;
 155
 156         let name = pkgname(builder, "rustc-docs");
 157
 158         builder.info(&format!("Dist compiler docs ({})", host));
 159         if !builder.config.compiler_docs {
 160             builder.info("\tskipping - compiler docs disabled");
 161             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 162         }
 163
 164         builder.default_doc(None);
 165
 166         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 167         let _ = fs::remove_dir_all(&image);
 168
 169         let dst = image.join("share/doc/rust/html");
 170         t!(fs::create_dir_all(&dst));
 171         let src = builder.compiler_doc_out(host);
 172         builder.cp_r(&src, &dst);
 173
 174         let mut cmd = rust_installer(builder);
 175         cmd.arg("generate")
 176            .arg("--product-name=Rustc-Documentation")
 177            .arg("--rel-manifest-dir=rustlib")
 178            .arg("--success-message=Rustc-documentation-is-installed.")
 179            .arg("--image-dir").arg(&image)
 180            .arg("--work-dir").arg(&tmpdir(builder))
 181            .arg("--output-dir").arg(&distdir(builder))
 182            .arg(format!("--package-name={}-{}", name, host))
 183            .arg("--component-name=rustc-docs")
 184            .arg("--legacy-manifest-dirs=rustlib,cargo")
 185            .arg("--bulk-dirs=share/doc/rust/html");
 186         builder.run(&mut cmd);
 187         builder.remove_dir(&image);
 188
 189         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 190     }
 191 }
 192
 193 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 194     let mut found = Vec::with_capacity(files.len());
 195
 196     for file in files {
 197         let file_path =
 198             path.iter()
 199                 .map(|dir| dir.join(file))
 200                 .find(|p| p.exists());
 201
 202         if let Some(file_path) = file_path {
 203             found.push(file_path);
 204         } else {
 205             panic!("Could not find '{}' in {:?}", file, path);
 206         }
 207     }
 208
 209     found
 210 }
 211
 212 fn make_win_dist(
 213     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 214 ) {
 215     //Ask gcc where it keeps its stuff
 216     let mut cmd = Command::new(builder.cc(target_triple));
 217     cmd.arg("-print-search-dirs");
 218     let gcc_out = output(&mut cmd);
 219
 220     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 221     let mut lib_path = Vec::new();
 222
 223     for line in gcc_out.lines() {
 224         let idx = line.find(':').unwrap();
 225         let key = &line[..idx];
 226         let trim_chars: &[_] = &[' ', '='];
 227         let value =
 228             line[(idx + 1)..]
 229                 .trim_start_matches(trim_chars)
 230                 .split(';')
 231                 .map(PathBuf::from);
 232
 233         if key == "programs" {
 234             bin_path.extend(value);
 235         } else if key == "libraries" {
 236             lib_path.extend(value);
 237         }
 238     }
 239
 240     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 241     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 242     if target_triple.starts_with("i686-") {
 243         rustc_dlls.push("libgcc_s_dw2-1.dll");
 244     } else {
 245         rustc_dlls.push("libgcc_s_seh-1.dll");
 246     }
 247
 248     let target_libs = [ //MinGW libs
 249         "libgcc.a",
 250         "libgcc_eh.a",
 251         "libgcc_s.a",
 252         "libm.a",
 253         "libmingw32.a",
 254         "libmingwex.a",
 255         "libstdc++.a",
 256         "libiconv.a",
 257         "libmoldname.a",
 258         "libpthread.a",
 259         //Windows import libs
 260         "libadvapi32.a",
 261         "libbcrypt.a",
 262         "libcomctl32.a",
 263         "libcomdlg32.a",
 264         "libcredui.a",
 265         "libcrypt32.a",
 266         "libdbghelp.a",
 267         "libgdi32.a",
 268         "libimagehlp.a",
 269         "libiphlpapi.a",
 270         "libkernel32.a",
 271         "libmsimg32.a",
 272         "libmsvcrt.a",
 273         "libodbc32.a",
 274         "libole32.a",
 275         "liboleaut32.a",
 276         "libopengl32.a",
 277         "libpsapi.a",
 278         "librpcrt4.a",
 279         "libsecur32.a",
 280         "libsetupapi.a",
 281         "libshell32.a",
 282         "libsynchronization.a",
 283         "libuser32.a",
 284         "libuserenv.a",
 285         "libuuid.a",
 286         "libwinhttp.a",
 287         "libwinmm.a",
 288         "libwinspool.a",
 289         "libws2_32.a",
 290         "libwsock32.a",
 291     ];
 292
 293     //Find mingw artifacts we want to bundle
 294     let target_tools = find_files(&target_tools, &bin_path);
 295     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 296     let target_libs = find_files(&target_libs, &lib_path);
 297
 298     // Copy runtime dlls next to rustc.exe
 299     let dist_bin_dir = rust_root.join("bin/");
 300     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 301     for src in rustc_dlls {
 302         builder.copy_to_folder(&src, &dist_bin_dir);
 303     }
 304
 305     //Copy platform tools to platform-specific bin directory
 306     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 307     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 308     for src in target_tools {
 309         builder.copy_to_folder(&src, &target_bin_dir);
 310     }
 311
 312     // Warn windows-gnu users that the bundled GCC cannot compile C files
 313     builder.create(
 314         &target_bin_dir.join("GCC-WARNING.txt"),
 315         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 316          used as a linker. In order to be able to compile projects containing C code use\
 317          the GCC provided by MinGW or Cygwin."
 318     );
 319
 320     //Copy platform libs to platform-specific lib directory
 321     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 322     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 323     for src in target_libs {
 324         builder.copy_to_folder(&src, &target_lib_dir);
 325     }
 326 }
 327
 328 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 329 pub struct Mingw {
 330     pub host: Interned<String>,
 331 }
 332
 333 impl Step for Mingw {
 334     type Output = Option<PathBuf>;
 335     const DEFAULT: bool = true;
 336
 337     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 338         run.never()
 339     }
 340
 341     fn make_run(run: RunConfig<'_>) {
 342         run.builder.ensure(Mingw { host: run.target });
 343     }
 344
 345     /// Builds the `rust-mingw` installer component.
 346     ///
 347     /// This contains all the bits and pieces to run the MinGW Windows targets
 348     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 349     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 350         let host = self.host;
 351
 352         if !host.contains("pc-windows-gnu") {
 353             return None;
 354         }
 355
 356         builder.info(&format!("Dist mingw ({})", host));
 357         let name = pkgname(builder, "rust-mingw");
 358         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 359         let _ = fs::remove_dir_all(&image);
 360         t!(fs::create_dir_all(&image));
 361
 362         // The first argument is a "temporary directory" which is just
 363         // thrown away (this contains the runtime DLLs included in the rustc package
 364         // above) and the second argument is where to place all the MinGW components
 365         // (which is what we want).
 366         make_win_dist(&tmpdir(builder), &image, host, &builder);
 367
 368         let mut cmd = rust_installer(builder);
 369         cmd.arg("generate")
 370            .arg("--product-name=Rust-MinGW")
 371            .arg("--rel-manifest-dir=rustlib")
 372            .arg("--success-message=Rust-MinGW-is-installed.")
 373            .arg("--image-dir").arg(&image)
 374            .arg("--work-dir").arg(&tmpdir(builder))
 375            .arg("--output-dir").arg(&distdir(builder))
 376            .arg(format!("--package-name={}-{}", name, host))
 377            .arg("--component-name=rust-mingw")
 378            .arg("--legacy-manifest-dirs=rustlib,cargo");
 379         builder.run(&mut cmd);
 380         t!(fs::remove_dir_all(&image));
 381         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 382     }
 383 }
 384
 385 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct Rustc {
 387     pub compiler: Compiler,
 388 }
 389
 390 impl Step for Rustc {
 391     type Output = PathBuf;
 392     const DEFAULT: bool = true;
 393     const ONLY_HOSTS: bool = true;
 394
 395     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 396         run.path("src/librustc")
 397     }
 398
 399     fn make_run(run: RunConfig<'_>) {
 400         run.builder.ensure(Rustc {
 401             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 402         });
 403     }
 404
 405     /// Creates the `rustc` installer component.
 406     fn run(self, builder: &Builder<'_>) -> PathBuf {
 407         let compiler = self.compiler;
 408         let host = self.compiler.host;
 409
 410         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 411         let name = pkgname(builder, "rustc");
 412         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 413         let _ = fs::remove_dir_all(&image);
 414         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 415         let _ = fs::remove_dir_all(&overlay);
 416
 417         // Prepare the rustc "image", what will actually end up getting installed
 418         prepare_image(builder, compiler, &image);
 419
 420         // Prepare the overlay which is part of the tarball but won't actually be
 421         // installed
 422         let cp = |file: &str| {
 423             builder.install(&builder.src.join(file), &overlay, 0o644);
 424         };
 425         cp("COPYRIGHT");
 426         cp("LICENSE-APACHE");
 427         cp("LICENSE-MIT");
 428         cp("README.md");
 429         // tiny morsel of metadata is used by rust-packaging
 430         let version = builder.rust_version();
 431         builder.create(&overlay.join("version"), &version);
 432         if let Some(sha) = builder.rust_sha() {
 433             builder.create(&overlay.join("git-commit-hash"), &sha);
 434         }
 435
 436         // On MinGW we've got a few runtime DLL dependencies that we need to
 437         // include. The first argument to this script is where to put these DLLs
 438         // (the image we're creating), and the second argument is a junk directory
 439         // to ignore all other MinGW stuff the script creates.
 440         //
 441         // On 32-bit MinGW we're always including a DLL which needs some extra
 442         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 443         // anything requiring us to distribute a license, but it's likely the
 444         // install will *also* include the rust-mingw package, which also needs
 445         // licenses, so to be safe we just include it here in all MinGW packages.
 446         if host.contains("pc-windows-gnu") {
 447             make_win_dist(&image, &tmpdir(builder), host, builder);
 448
 449             let dst = image.join("share/doc");
 450             t!(fs::create_dir_all(&dst));
 451             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 452         }
 453
 454         // Finally, wrap everything up in a nice tarball!
 455         let mut cmd = rust_installer(builder);
 456         cmd.arg("generate")
 457            .arg("--product-name=Rust")
 458            .arg("--rel-manifest-dir=rustlib")
 459            .arg("--success-message=Rust-is-ready-to-roll.")
 460            .arg("--image-dir").arg(&image)
 461            .arg("--work-dir").arg(&tmpdir(builder))
 462            .arg("--output-dir").arg(&distdir(builder))
 463            .arg("--non-installed-overlay").arg(&overlay)
 464            .arg(format!("--package-name={}-{}", name, host))
 465            .arg("--component-name=rustc")
 466            .arg("--legacy-manifest-dirs=rustlib,cargo");
 467         builder.run(&mut cmd);
 468         builder.remove_dir(&image);
 469         builder.remove_dir(&overlay);
 470
 471         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 472
 473         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 474             let host = compiler.host;
 475             let src = builder.sysroot(compiler);
 476             let libdir = builder.rustc_libdir(compiler);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 483
 484             let libdir_relative = builder.libdir_relative(compiler);
 485
 486             // Copy runtime DLLs needed by the compiler
 487             if libdir_relative.to_str() != Some("bin") {
 488                 for entry in builder.read_dir(&libdir) {
 489                     let name = entry.file_name();
 490                     if let Some(s) = name.to_str() {
 491                         if is_dylib(s) {
 492                             builder.install(&entry.path(), &image.join(&libdir_relative), 0o644);
 493                         }
 494                     }
 495                 }
 496             }
 497
 498             // Copy over the codegen backends
 499             let backends_src = builder.sysroot_codegen_backends(compiler);
 500             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 501             let backends_dst = image.join(&backends_rel);
 502             t!(fs::create_dir_all(&backends_dst));
 503             builder.cp_r(&backends_src, &backends_dst);
 504
 505             // Copy libLLVM.so to the lib dir as well, if needed. While not
 506             // technically needed by rustc itself it's needed by lots of other
 507             // components like the llvm tools and LLD. LLD is included below and
 508             // tools/LLDB come later, so let's just throw it in the rustc
 509             // component for now.
 510             maybe_install_llvm_dylib(builder, host, image);
 511
 512             // Copy over lld if it's there
 513             if builder.config.lld_enabled {
 514                 let exe = exe("rust-lld", &compiler.host);
 515                 let src = builder.sysroot_libdir(compiler, host)
 516                     .parent()
 517                     .unwrap()
 518                     .join("bin")
 519                     .join(&exe);
 520                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 521                 let dst = image.join("lib/rustlib")
 522                     .join(&*host)
 523                     .join("bin")
 524                     .join(&exe);
 525                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 526                 builder.copy(&src, &dst);
 527             }
 528
 529             // Man pages
 530             t!(fs::create_dir_all(image.join("share/man/man1")));
 531             let man_src = builder.src.join("src/doc/man");
 532             let man_dst = image.join("share/man/man1");
 533
 534             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 535             let time = env::var("SOURCE_DATE_EPOCH")
 536                 .map(|timestamp| {
 537                     let epoch = timestamp.parse().map_err(|err| {
 538                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 539                     }).unwrap();
 540
 541                     time::at(Timespec::new(epoch, 0))
 542                 })
 543                 .unwrap_or_else(|_| time::now());
 544
 545             let month_year = t!(time::strftime("%B %Y", &time));
 546             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 547             // to hardlink, and we don't want to edit the source templates
 548             for file_entry in builder.read_dir(&man_src) {
 549                 let page_src = file_entry.path();
 550                 let page_dst = man_dst.join(file_entry.file_name());
 551                 t!(fs::copy(&page_src, &page_dst));
 552                 // template in month/year and version number
 553                 builder.replace_in_file(&page_dst,
 554                                 &[("<INSERT DATE HERE>", &month_year),
 555                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 556             }
 557
 558             // Debugger scripts
 559             builder.ensure(DebuggerScripts {
 560                 sysroot: INTERNER.intern_path(image.to_owned()),
 561                 host,
 562             });
 563
 564             // Misc license info
 565             let cp = |file: &str| {
 566                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 567             };
 568             cp("COPYRIGHT");
 569             cp("LICENSE-APACHE");
 570             cp("LICENSE-MIT");
 571             cp("README.md");
 572         }
 573     }
 574 }
 575
 576 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 577 pub struct DebuggerScripts {
 578     pub sysroot: Interned<PathBuf>,
 579     pub host: Interned<String>,
 580 }
 581
 582 impl Step for DebuggerScripts {
 583     type Output = ();
 584
 585     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 586         run.path("src/lldb_batchmode.py")
 587     }
 588
 589     fn make_run(run: RunConfig<'_>) {
 590         run.builder.ensure(DebuggerScripts {
 591             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 592             host: run.target,
 593         });
 594     }
 595
 596     /// Copies debugger scripts for `target` into the `sysroot` specified.
 597     fn run(self, builder: &Builder<'_>) {
 598         let host = self.host;
 599         let sysroot = self.sysroot;
 600         let dst = sysroot.join("lib/rustlib/etc");
 601         t!(fs::create_dir_all(&dst));
 602         let cp_debugger_script = |file: &str| {
 603             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 604         };
 605         if host.contains("windows-msvc") {
 606             // windbg debugger scripts
 607             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 608                 0o755);
 609
 610             cp_debugger_script("natvis/intrinsic.natvis");
 611             cp_debugger_script("natvis/liballoc.natvis");
 612             cp_debugger_script("natvis/libcore.natvis");
 613         } else {
 614             cp_debugger_script("debugger_pretty_printers_common.py");
 615
 616             // gdb debugger scripts
 617             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 618                     0o755);
 619             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 620                     0o755);
 621
 622             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 623             cp_debugger_script("gdb_rust_pretty_printing.py");
 624
 625             // lldb debugger scripts
 626             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 627                     0o755);
 628
 629             cp_debugger_script("lldb_rust_formatters.py");
 630         }
 631     }
 632 }
 633
 634 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 635 pub struct Std {
 636     pub compiler: Compiler,
 637     pub target: Interned<String>,
 638 }
 639
 640 impl Step for Std {
 641     type Output = PathBuf;
 642     const DEFAULT: bool = true;
 643
 644     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 645         run.path("src/libstd")
 646     }
 647
 648     fn make_run(run: RunConfig<'_>) {
 649         run.builder.ensure(Std {
 650             compiler: run.builder.compiler_for(
 651                 run.builder.top_stage,
 652                 run.builder.config.build,
 653                 run.target,
 654             ),
 655             target: run.target,
 656         });
 657     }
 658
 659     fn run(self, builder: &Builder<'_>) -> PathBuf {
 660         let compiler = self.compiler;
 661         let target = self.target;
 662
 663         let name = pkgname(builder, "rust-std");
 664         builder.info(&format!("Dist std stage{} ({} -> {})",
 665             compiler.stage, &compiler.host, target));
 666
 667         // The only true set of target libraries came from the build triple, so
 668         // let's reduce redundant work by only producing archives from that host.
 669         if compiler.host != builder.config.build {
 670             builder.info("\tskipping, not a build host");
 671             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 672         }
 673
 674         // We want to package up as many target libraries as possible
 675         // for the `rust-std` package, so if this is a host target we
 676         // depend on librustc and otherwise we just depend on libtest.
 677         if builder.hosts.iter().any(|t| t == target) {
 678             builder.ensure(compile::Rustc { compiler, target });
 679         } else {
 680             if builder.no_std(target) == Some(true) {
 681                 // the `test` doesn't compile for no-std targets
 682                 builder.ensure(compile::Std { compiler, target });
 683             } else {
 684                 builder.ensure(compile::Test { compiler, target });
 685             }
 686         }
 687
 688         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 689         let _ = fs::remove_dir_all(&image);
 690
 691         let dst = image.join("lib/rustlib").join(target);
 692         t!(fs::create_dir_all(&dst));
 693         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 694         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 695         builder.cp_filtered(&src, &dst, &|path| {
 696             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 697                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 698                     return false
 699                 }
 700                 if name == "bin" {
 701                     return false
 702                 }
 703                 if name.contains("LLVM") {
 704                     return false
 705                 }
 706             }
 707             true
 708         });
 709
 710         let mut cmd = rust_installer(builder);
 711         cmd.arg("generate")
 712            .arg("--product-name=Rust")
 713            .arg("--rel-manifest-dir=rustlib")
 714            .arg("--success-message=std-is-standing-at-the-ready.")
 715            .arg("--image-dir").arg(&image)
 716            .arg("--work-dir").arg(&tmpdir(builder))
 717            .arg("--output-dir").arg(&distdir(builder))
 718            .arg(format!("--package-name={}-{}", name, target))
 719            .arg(format!("--component-name=rust-std-{}", target))
 720            .arg("--legacy-manifest-dirs=rustlib,cargo");
 721         builder.run(&mut cmd);
 722         builder.remove_dir(&image);
 723         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 724     }
 725 }
 726
 727 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 728 pub struct Analysis {
 729     pub compiler: Compiler,
 730     pub target: Interned<String>,
 731 }
 732
 733 impl Step for Analysis {
 734     type Output = PathBuf;
 735     const DEFAULT: bool = true;
 736
 737     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 738         let builder = run.builder;
 739         run.path("analysis").default_condition(builder.config.extended)
 740     }
 741
 742     fn make_run(run: RunConfig<'_>) {
 743         run.builder.ensure(Analysis {
 744             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 745             target: run.target,
 746         });
 747     }
 748
 749     /// Creates a tarball of save-analysis metadata, if available.
 750     fn run(self, builder: &Builder<'_>) -> PathBuf {
 751         let compiler = self.compiler;
 752         let target = self.target;
 753         assert!(builder.config.extended);
 754         builder.info("Dist analysis");
 755         let name = pkgname(builder, "rust-analysis");
 756
 757         if &compiler.host != builder.config.build {
 758             builder.info("\tskipping, not a build host");
 759             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 760         }
 761
 762         builder.ensure(Std { compiler, target });
 763
 764         // Find the actual compiler (handling the full bootstrap option) which
 765         // produced the save-analysis data because that data isn't copied
 766         // through the sysroot uplifting.
 767         let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
 768
 769         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 770
 771         let src = builder.stage_out(compiler, Mode::Std)
 772             .join(target).join(builder.cargo_dir()).join("deps");
 773
 774         let image_src = src.join("save-analysis");
 775         let dst = image.join("lib/rustlib").join(target).join("analysis");
 776         t!(fs::create_dir_all(&dst));
 777         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 778         builder.cp_r(&image_src, &dst);
 779
 780         let mut cmd = rust_installer(builder);
 781         cmd.arg("generate")
 782            .arg("--product-name=Rust")
 783            .arg("--rel-manifest-dir=rustlib")
 784            .arg("--success-message=save-analysis-saved.")
 785            .arg("--image-dir").arg(&image)
 786            .arg("--work-dir").arg(&tmpdir(builder))
 787            .arg("--output-dir").arg(&distdir(builder))
 788            .arg(format!("--package-name={}-{}", name, target))
 789            .arg(format!("--component-name=rust-analysis-{}", target))
 790            .arg("--legacy-manifest-dirs=rustlib,cargo");
 791         builder.run(&mut cmd);
 792         builder.remove_dir(&image);
 793         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 794     }
 795 }
 796
 797 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 798     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 799         let spath = match path.to_str() {
 800             Some(path) => path,
 801             None => return false,
 802         };
 803         if spath.ends_with("~") || spath.ends_with(".pyc") {
 804             return false
 805         }
 806
 807         const LLVM_PROJECTS: &[&str] = &[
 808             "llvm-project/clang", "llvm-project\\clang",
 809             "llvm-project/lld", "llvm-project\\lld",
 810             "llvm-project/lldb", "llvm-project\\lldb",
 811             "llvm-project/llvm", "llvm-project\\llvm",
 812         ];
 813         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 814             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 815         {
 816             return false;
 817         }
 818
 819         const LLVM_TEST: &[&str] = &[
 820             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 821             "llvm-emscripten/test", "llvm-emscripten\\test",
 822         ];
 823         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 824             (spath.ends_with(".ll") ||
 825              spath.ends_with(".td") ||
 826              spath.ends_with(".s")) {
 827             return false
 828         }
 829         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 830             return false
 831         }
 832
 833         let full_path = Path::new(dir).join(path);
 834         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 835             return false;
 836         }
 837
 838         let excludes = [
 839             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 840             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 841             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 842             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 843         ];
 844         !path.iter()
 845              .map(|s| s.to_str().unwrap())
 846              .any(|s| excludes.contains(&s))
 847     }
 848
 849     // Copy the directories using our filter
 850     for item in src_dirs {
 851         let dst = &dst_dir.join(item);
 852         t!(fs::create_dir_all(dst));
 853         builder.cp_filtered(
 854             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 855     }
 856 }
 857
 858 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 859 pub struct Src;
 860
 861 impl Step for Src {
 862     /// The output path of the src installer tarball
 863     type Output = PathBuf;
 864     const DEFAULT: bool = true;
 865     const ONLY_HOSTS: bool = true;
 866
 867     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 868         run.path("src")
 869     }
 870
 871     fn make_run(run: RunConfig<'_>) {
 872         run.builder.ensure(Src);
 873     }
 874
 875     /// Creates the `rust-src` installer component
 876     fn run(self, builder: &Builder<'_>) -> PathBuf {
 877         builder.info("Dist src");
 878
 879         let name = pkgname(builder, "rust-src");
 880         let image = tmpdir(builder).join(format!("{}-image", name));
 881         let _ = fs::remove_dir_all(&image);
 882
 883         let dst = image.join("lib/rustlib/src");
 884         let dst_src = dst.join("rust");
 885         t!(fs::create_dir_all(&dst_src));
 886
 887         let src_files = [
 888             "Cargo.lock",
 889         ];
 890         // This is the reduced set of paths which will become the rust-src component
 891         // (essentially libstd and all of its path dependencies)
 892         let std_src_dirs = [
 893             "src/build_helper",
 894             "src/liballoc",
 895             "src/libcore",
 896             "src/libpanic_abort",
 897             "src/libpanic_unwind",
 898             "src/librustc_asan",
 899             "src/librustc_lsan",
 900             "src/librustc_msan",
 901             "src/librustc_tsan",
 902             "src/libstd",
 903             "src/libunwind",
 904             "src/libtest",
 905             "src/libterm",
 906             "src/libprofiler_builtins",
 907             "src/stdsimd",
 908             "src/libproc_macro",
 909             "src/tools/rustc-std-workspace-core",
 910             "src/tools/rustc-std-workspace-alloc",
 911             "src/librustc",
 912             "src/libsyntax",
 913         ];
 914
 915         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 916         for file in src_files.iter() {
 917             builder.copy(&builder.src.join(file), &dst_src.join(file));
 918         }
 919
 920         // Create source tarball in rust-installer format
 921         let mut cmd = rust_installer(builder);
 922         cmd.arg("generate")
 923            .arg("--product-name=Rust")
 924            .arg("--rel-manifest-dir=rustlib")
 925            .arg("--success-message=Awesome-Source.")
 926            .arg("--image-dir").arg(&image)
 927            .arg("--work-dir").arg(&tmpdir(builder))
 928            .arg("--output-dir").arg(&distdir(builder))
 929            .arg(format!("--package-name={}", name))
 930            .arg("--component-name=rust-src")
 931            .arg("--legacy-manifest-dirs=rustlib,cargo");
 932         builder.run(&mut cmd);
 933
 934         builder.remove_dir(&image);
 935         distdir(builder).join(&format!("{}.tar.gz", name))
 936     }
 937 }
 938
 939 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 940
 941 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 942 pub struct PlainSourceTarball;
 943
 944 impl Step for PlainSourceTarball {
 945     /// Produces the location of the tarball generated
 946     type Output = PathBuf;
 947     const DEFAULT: bool = true;
 948     const ONLY_HOSTS: bool = true;
 949
 950     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 951         let builder = run.builder;
 952         run.path("src").default_condition(builder.config.rust_dist_src)
 953     }
 954
 955     fn make_run(run: RunConfig<'_>) {
 956         run.builder.ensure(PlainSourceTarball);
 957     }
 958
 959     /// Creates the plain source tarball
 960     fn run(self, builder: &Builder<'_>) -> PathBuf {
 961         builder.info("Create plain source tarball");
 962
 963         // Make sure that the root folder of tarball has the correct name
 964         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 965         let plain_dst_src = tmpdir(builder).join(&plain_name);
 966         let _ = fs::remove_dir_all(&plain_dst_src);
 967         t!(fs::create_dir_all(&plain_dst_src));
 968
 969         // This is the set of root paths which will become part of the source package
 970         let src_files = [
 971             "COPYRIGHT",
 972             "LICENSE-APACHE",
 973             "LICENSE-MIT",
 974             "CONTRIBUTING.md",
 975             "README.md",
 976             "RELEASES.md",
 977             "configure",
 978             "x.py",
 979             "config.toml.example",
 980             "Cargo.toml",
 981             "Cargo.lock",
 982         ];
 983         let src_dirs = [
 984             "src",
 985         ];
 986
 987         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 988
 989         // Copy the files normally
 990         for item in &src_files {
 991             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 992         }
 993
 994         // Create the version file
 995         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 996         if let Some(sha) = builder.rust_sha() {
 997             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 998         }
 999
1000         // If we're building from git sources, we need to vendor a complete distribution.
1001         if builder.rust_info.is_git() {
1002             // Get cargo-vendor installed, if it isn't already.
1003             let mut has_cargo_vendor = false;
1004             let mut cmd = Command::new(&builder.initial_cargo);
1005             for line in output(cmd.arg("install").arg("--list")).lines() {
1006                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
1007             }
1008             if !has_cargo_vendor {
1009                 let mut cmd = builder.cargo(
1010                     builder.compiler(0, builder.config.build),
1011                     Mode::ToolBootstrap,
1012                     builder.config.build,
1013                     "install"
1014                 );
1015                 cmd.arg("--force")
1016                    .arg("--debug")
1017                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1018                    .arg("cargo-vendor");
1019                 builder.run(&mut cmd);
1020             }
1021
1022             // Vendor all Cargo dependencies
1023             let mut cmd = Command::new(&builder.initial_cargo);
1024             cmd.arg("vendor")
1025                .current_dir(&plain_dst_src);
1026             builder.run(&mut cmd);
1027         }
1028
1029         // Create plain source tarball
1030         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1031         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1032         tarball.set_extension(""); // strip .gz
1033         tarball.set_extension(""); // strip .tar
1034         if let Some(dir) = tarball.parent() {
1035             builder.create_dir(&dir);
1036         }
1037         builder.info("running installer");
1038         let mut cmd = rust_installer(builder);
1039         cmd.arg("tarball")
1040            .arg("--input").arg(&plain_name)
1041            .arg("--output").arg(&tarball)
1042            .arg("--work-dir=.")
1043            .current_dir(tmpdir(builder));
1044         builder.run(&mut cmd);
1045         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1046     }
1047 }
1048
1049 // We have to run a few shell scripts, which choke quite a bit on both `\`
1050 // characters and on `C:\` paths, so normalize both of them away.
1051 pub fn sanitize_sh(path: &Path) -> String {
1052     let path = path.to_str().unwrap().replace("\\", "/");
1053     return change_drive(&path).unwrap_or(path);
1054
1055     fn change_drive(s: &str) -> Option<String> {
1056         let mut ch = s.chars();
1057         let drive = ch.next().unwrap_or('C');
1058         if ch.next() != Some(':') {
1059             return None
1060         }
1061         if ch.next() != Some('/') {
1062             return None
1063         }
1064         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1065     }
1066 }
1067
1068 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1069 pub struct Cargo {
1070     pub stage: u32,
1071     pub target: Interned<String>,
1072 }
1073
1074 impl Step for Cargo {
1075     type Output = PathBuf;
1076     const ONLY_HOSTS: bool = true;
1077
1078     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1079         run.path("cargo")
1080     }
1081
1082     fn make_run(run: RunConfig<'_>) {
1083         run.builder.ensure(Cargo {
1084             stage: run.builder.top_stage,
1085             target: run.target,
1086         });
1087     }
1088
1089     fn run(self, builder: &Builder<'_>) -> PathBuf {
1090         let stage = self.stage;
1091         let target = self.target;
1092
1093         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1094         let src = builder.src.join("src/tools/cargo");
1095         let etc = src.join("src/etc");
1096         let release_num = builder.release_num("cargo");
1097         let name = pkgname(builder, "cargo");
1098         let version = builder.cargo_info.version(builder, &release_num);
1099
1100         let tmp = tmpdir(builder);
1101         let image = tmp.join("cargo-image");
1102         drop(fs::remove_dir_all(&image));
1103         builder.create_dir(&image);
1104
1105         // Prepare the image directory
1106         builder.create_dir(&image.join("share/zsh/site-functions"));
1107         builder.create_dir(&image.join("etc/bash_completion.d"));
1108         let cargo = builder.ensure(tool::Cargo {
1109             compiler: builder.compiler(stage, builder.config.build),
1110             target
1111         });
1112         builder.install(&cargo, &image.join("bin"), 0o755);
1113         for man in t!(etc.join("man").read_dir()) {
1114             let man = t!(man);
1115             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1116         }
1117         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1118         builder.copy(&etc.join("cargo.bashcomp.sh"),
1119              &image.join("etc/bash_completion.d/cargo"));
1120         let doc = image.join("share/doc/cargo");
1121         builder.install(&src.join("README.md"), &doc, 0o644);
1122         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1123         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1124         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1125
1126         // Prepare the overlay
1127         let overlay = tmp.join("cargo-overlay");
1128         drop(fs::remove_dir_all(&overlay));
1129         builder.create_dir(&overlay);
1130         builder.install(&src.join("README.md"), &overlay, 0o644);
1131         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1132         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1133         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1134         builder.create(&overlay.join("version"), &version);
1135
1136         // Generate the installer tarball
1137         let mut cmd = rust_installer(builder);
1138         cmd.arg("generate")
1139            .arg("--product-name=Rust")
1140            .arg("--rel-manifest-dir=rustlib")
1141            .arg("--success-message=Rust-is-ready-to-roll.")
1142            .arg("--image-dir").arg(&image)
1143            .arg("--work-dir").arg(&tmpdir(builder))
1144            .arg("--output-dir").arg(&distdir(builder))
1145            .arg("--non-installed-overlay").arg(&overlay)
1146            .arg(format!("--package-name={}-{}", name, target))
1147            .arg("--component-name=cargo")
1148            .arg("--legacy-manifest-dirs=rustlib,cargo");
1149         builder.run(&mut cmd);
1150         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1151     }
1152 }
1153
1154 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1155 pub struct Rls {
1156     pub stage: u32,
1157     pub target: Interned<String>,
1158 }
1159
1160 impl Step for Rls {
1161     type Output = Option<PathBuf>;
1162     const ONLY_HOSTS: bool = true;
1163
1164     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1165         run.path("rls")
1166     }
1167
1168     fn make_run(run: RunConfig<'_>) {
1169         run.builder.ensure(Rls {
1170             stage: run.builder.top_stage,
1171             target: run.target,
1172         });
1173     }
1174
1175     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1176         let stage = self.stage;
1177         let target = self.target;
1178         assert!(builder.config.extended);
1179
1180         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1181         let src = builder.src.join("src/tools/rls");
1182         let release_num = builder.release_num("rls");
1183         let name = pkgname(builder, "rls");
1184         let version = builder.rls_info.version(builder, &release_num);
1185
1186         let tmp = tmpdir(builder);
1187         let image = tmp.join("rls-image");
1188         drop(fs::remove_dir_all(&image));
1189         t!(fs::create_dir_all(&image));
1190
1191         // Prepare the image directory
1192         // We expect RLS to build, because we've exited this step above if tool
1193         // state for RLS isn't testing.
1194         let rls = builder.ensure(tool::Rls {
1195             compiler: builder.compiler(stage, builder.config.build),
1196             target, extra_features: Vec::new()
1197         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1198
1199         builder.install(&rls, &image.join("bin"), 0o755);
1200         let doc = image.join("share/doc/rls");
1201         builder.install(&src.join("README.md"), &doc, 0o644);
1202         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1203         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1204
1205         // Prepare the overlay
1206         let overlay = tmp.join("rls-overlay");
1207         drop(fs::remove_dir_all(&overlay));
1208         t!(fs::create_dir_all(&overlay));
1209         builder.install(&src.join("README.md"), &overlay, 0o644);
1210         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1211         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1212         builder.create(&overlay.join("version"), &version);
1213
1214         // Generate the installer tarball
1215         let mut cmd = rust_installer(builder);
1216         cmd.arg("generate")
1217            .arg("--product-name=Rust")
1218            .arg("--rel-manifest-dir=rustlib")
1219            .arg("--success-message=RLS-ready-to-serve.")
1220            .arg("--image-dir").arg(&image)
1221            .arg("--work-dir").arg(&tmpdir(builder))
1222            .arg("--output-dir").arg(&distdir(builder))
1223            .arg("--non-installed-overlay").arg(&overlay)
1224            .arg(format!("--package-name={}-{}", name, target))
1225            .arg("--legacy-manifest-dirs=rustlib,cargo")
1226            .arg("--component-name=rls-preview");
1227
1228         builder.run(&mut cmd);
1229         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1230     }
1231 }
1232
1233 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1234 pub struct Clippy {
1235     pub stage: u32,
1236     pub target: Interned<String>,
1237 }
1238
1239 impl Step for Clippy {
1240     type Output = Option<PathBuf>;
1241     const ONLY_HOSTS: bool = true;
1242
1243     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1244         run.path("clippy")
1245     }
1246
1247     fn make_run(run: RunConfig<'_>) {
1248         run.builder.ensure(Clippy {
1249             stage: run.builder.top_stage,
1250             target: run.target,
1251         });
1252     }
1253
1254     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1255         let stage = self.stage;
1256         let target = self.target;
1257         assert!(builder.config.extended);
1258
1259         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1260         let src = builder.src.join("src/tools/clippy");
1261         let release_num = builder.release_num("clippy");
1262         let name = pkgname(builder, "clippy");
1263         let version = builder.clippy_info.version(builder, &release_num);
1264
1265         let tmp = tmpdir(builder);
1266         let image = tmp.join("clippy-image");
1267         drop(fs::remove_dir_all(&image));
1268         builder.create_dir(&image);
1269
1270         // Prepare the image directory
1271         // We expect clippy to build, because we've exited this step above if tool
1272         // state for clippy isn't testing.
1273         let clippy = builder.ensure(tool::Clippy {
1274             compiler: builder.compiler(stage, builder.config.build),
1275             target, extra_features: Vec::new()
1276         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1277         let cargoclippy = builder.ensure(tool::CargoClippy {
1278             compiler: builder.compiler(stage, builder.config.build),
1279             target, extra_features: Vec::new()
1280         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1281
1282         builder.install(&clippy, &image.join("bin"), 0o755);
1283         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1284         let doc = image.join("share/doc/clippy");
1285         builder.install(&src.join("README.md"), &doc, 0o644);
1286         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1287         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1288
1289         // Prepare the overlay
1290         let overlay = tmp.join("clippy-overlay");
1291         drop(fs::remove_dir_all(&overlay));
1292         t!(fs::create_dir_all(&overlay));
1293         builder.install(&src.join("README.md"), &overlay, 0o644);
1294         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1295         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1296         builder.create(&overlay.join("version"), &version);
1297
1298         // Generate the installer tarball
1299         let mut cmd = rust_installer(builder);
1300         cmd.arg("generate")
1301            .arg("--product-name=Rust")
1302            .arg("--rel-manifest-dir=rustlib")
1303            .arg("--success-message=clippy-ready-to-serve.")
1304            .arg("--image-dir").arg(&image)
1305            .arg("--work-dir").arg(&tmpdir(builder))
1306            .arg("--output-dir").arg(&distdir(builder))
1307            .arg("--non-installed-overlay").arg(&overlay)
1308            .arg(format!("--package-name={}-{}", name, target))
1309            .arg("--legacy-manifest-dirs=rustlib,cargo")
1310            .arg("--component-name=clippy-preview");
1311
1312         builder.run(&mut cmd);
1313         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1314     }
1315 }
1316
1317 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1318 pub struct Miri {
1319     pub stage: u32,
1320     pub target: Interned<String>,
1321 }
1322
1323 impl Step for Miri {
1324     type Output = Option<PathBuf>;
1325     const ONLY_HOSTS: bool = true;
1326
1327     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1328         run.path("miri")
1329     }
1330
1331     fn make_run(run: RunConfig<'_>) {
1332         run.builder.ensure(Miri {
1333             stage: run.builder.top_stage,
1334             target: run.target,
1335         });
1336     }
1337
1338     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1339         let stage = self.stage;
1340         let target = self.target;
1341         assert!(builder.config.extended);
1342
1343         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1344         let src = builder.src.join("src/tools/miri");
1345         let release_num = builder.release_num("miri");
1346         let name = pkgname(builder, "miri");
1347         let version = builder.miri_info.version(builder, &release_num);
1348
1349         let tmp = tmpdir(builder);
1350         let image = tmp.join("miri-image");
1351         drop(fs::remove_dir_all(&image));
1352         builder.create_dir(&image);
1353
1354         // Prepare the image directory
1355         // We expect miri to build, because we've exited this step above if tool
1356         // state for miri isn't testing.
1357         let miri = builder.ensure(tool::Miri {
1358             compiler: builder.compiler(stage, builder.config.build),
1359             target, extra_features: Vec::new()
1360         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1361         let cargomiri = builder.ensure(tool::CargoMiri {
1362             compiler: builder.compiler(stage, builder.config.build),
1363             target, extra_features: Vec::new()
1364         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1365
1366         builder.install(&miri, &image.join("bin"), 0o755);
1367         builder.install(&cargomiri, &image.join("bin"), 0o755);
1368         let doc = image.join("share/doc/miri");
1369         builder.install(&src.join("README.md"), &doc, 0o644);
1370         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1371         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1372
1373         // Prepare the overlay
1374         let overlay = tmp.join("miri-overlay");
1375         drop(fs::remove_dir_all(&overlay));
1376         t!(fs::create_dir_all(&overlay));
1377         builder.install(&src.join("README.md"), &overlay, 0o644);
1378         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1379         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1380         builder.create(&overlay.join("version"), &version);
1381
1382         // Generate the installer tarball
1383         let mut cmd = rust_installer(builder);
1384         cmd.arg("generate")
1385            .arg("--product-name=Rust")
1386            .arg("--rel-manifest-dir=rustlib")
1387            .arg("--success-message=miri-ready-to-serve.")
1388            .arg("--image-dir").arg(&image)
1389            .arg("--work-dir").arg(&tmpdir(builder))
1390            .arg("--output-dir").arg(&distdir(builder))
1391            .arg("--non-installed-overlay").arg(&overlay)
1392            .arg(format!("--package-name={}-{}", name, target))
1393            .arg("--legacy-manifest-dirs=rustlib,cargo")
1394            .arg("--component-name=miri-preview");
1395
1396         builder.run(&mut cmd);
1397         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1398     }
1399 }
1400
1401 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1402 pub struct Rustfmt {
1403     pub stage: u32,
1404     pub target: Interned<String>,
1405 }
1406
1407 impl Step for Rustfmt {
1408     type Output = Option<PathBuf>;
1409     const ONLY_HOSTS: bool = true;
1410
1411     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1412         run.path("rustfmt")
1413     }
1414
1415     fn make_run(run: RunConfig<'_>) {
1416         run.builder.ensure(Rustfmt {
1417             stage: run.builder.top_stage,
1418             target: run.target,
1419         });
1420     }
1421
1422     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1423         let stage = self.stage;
1424         let target = self.target;
1425
1426         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1427         let src = builder.src.join("src/tools/rustfmt");
1428         let release_num = builder.release_num("rustfmt");
1429         let name = pkgname(builder, "rustfmt");
1430         let version = builder.rustfmt_info.version(builder, &release_num);
1431
1432         let tmp = tmpdir(builder);
1433         let image = tmp.join("rustfmt-image");
1434         drop(fs::remove_dir_all(&image));
1435         builder.create_dir(&image);
1436
1437         // Prepare the image directory
1438         let rustfmt = builder.ensure(tool::Rustfmt {
1439             compiler: builder.compiler(stage, builder.config.build),
1440             target, extra_features: Vec::new()
1441         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1442         let cargofmt = builder.ensure(tool::Cargofmt {
1443             compiler: builder.compiler(stage, builder.config.build),
1444             target, extra_features: Vec::new()
1445         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1446
1447         builder.install(&rustfmt, &image.join("bin"), 0o755);
1448         builder.install(&cargofmt, &image.join("bin"), 0o755);
1449         let doc = image.join("share/doc/rustfmt");
1450         builder.install(&src.join("README.md"), &doc, 0o644);
1451         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1452         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1453
1454         // Prepare the overlay
1455         let overlay = tmp.join("rustfmt-overlay");
1456         drop(fs::remove_dir_all(&overlay));
1457         builder.create_dir(&overlay);
1458         builder.install(&src.join("README.md"), &overlay, 0o644);
1459         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1460         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1461         builder.create(&overlay.join("version"), &version);
1462
1463         // Generate the installer tarball
1464         let mut cmd = rust_installer(builder);
1465         cmd.arg("generate")
1466            .arg("--product-name=Rust")
1467            .arg("--rel-manifest-dir=rustlib")
1468            .arg("--success-message=rustfmt-ready-to-fmt.")
1469            .arg("--image-dir").arg(&image)
1470            .arg("--work-dir").arg(&tmpdir(builder))
1471            .arg("--output-dir").arg(&distdir(builder))
1472            .arg("--non-installed-overlay").arg(&overlay)
1473            .arg(format!("--package-name={}-{}", name, target))
1474            .arg("--legacy-manifest-dirs=rustlib,cargo")
1475            .arg("--component-name=rustfmt-preview");
1476
1477         builder.run(&mut cmd);
1478         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1479     }
1480 }
1481
1482 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1483 pub struct Extended {
1484     stage: u32,
1485     host: Interned<String>,
1486     target: Interned<String>,
1487 }
1488
1489 impl Step for Extended {
1490     type Output = ();
1491     const DEFAULT: bool = true;
1492     const ONLY_HOSTS: bool = true;
1493
1494     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1495         let builder = run.builder;
1496         run.path("extended").default_condition(builder.config.extended)
1497     }
1498
1499     fn make_run(run: RunConfig<'_>) {
1500         run.builder.ensure(Extended {
1501             stage: run.builder.top_stage,
1502             host: run.builder.config.build,
1503             target: run.target,
1504         });
1505     }
1506
1507     /// Creates a combined installer for the specified target in the provided stage.
1508     fn run(self, builder: &Builder<'_>) {
1509         let stage = self.stage;
1510         let target = self.target;
1511
1512         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1513
1514         let rustc_installer = builder.ensure(Rustc {
1515             compiler: builder.compiler(stage, target),
1516         });
1517         let cargo_installer = builder.ensure(Cargo { stage, target });
1518         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1519         let rls_installer = builder.ensure(Rls { stage, target });
1520         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1521         let clippy_installer = builder.ensure(Clippy { stage, target });
1522         let miri_installer = builder.ensure(Miri { stage, target });
1523         let lldb_installer = builder.ensure(Lldb { target });
1524         let mingw_installer = builder.ensure(Mingw { host: target });
1525         let analysis_installer = builder.ensure(Analysis {
1526             compiler: builder.compiler(stage, self.host),
1527             target
1528         });
1529
1530         let docs_installer = builder.ensure(Docs { stage, host: target, });
1531         let std_installer = builder.ensure(Std {
1532             compiler: builder.compiler(stage, self.host),
1533             target,
1534         });
1535
1536         let tmp = tmpdir(builder);
1537         let overlay = tmp.join("extended-overlay");
1538         let etc = builder.src.join("src/etc/installer");
1539         let work = tmp.join("work");
1540
1541         let _ = fs::remove_dir_all(&overlay);
1542         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1543         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1544         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1545         let version = builder.rust_version();
1546         builder.create(&overlay.join("version"), &version);
1547         if let Some(sha) = builder.rust_sha() {
1548             builder.create(&overlay.join("git-commit-hash"), &sha);
1549         }
1550         builder.install(&etc.join("README.md"), &overlay, 0o644);
1551
1552         // When rust-std package split from rustc, we needed to ensure that during
1553         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1554         // the std files during uninstall. To do this ensure that rustc comes
1555         // before rust-std in the list below.
1556         let mut tarballs = Vec::new();
1557         tarballs.push(rustc_installer);
1558         tarballs.push(cargo_installer);
1559         tarballs.extend(rls_installer.clone());
1560         tarballs.extend(clippy_installer.clone());
1561         tarballs.extend(miri_installer.clone());
1562         tarballs.extend(rustfmt_installer.clone());
1563         tarballs.extend(llvm_tools_installer);
1564         tarballs.extend(lldb_installer);
1565         tarballs.push(analysis_installer);
1566         tarballs.push(std_installer);
1567         if builder.config.docs {
1568             tarballs.push(docs_installer);
1569         }
1570         if target.contains("pc-windows-gnu") {
1571             tarballs.push(mingw_installer.unwrap());
1572         }
1573         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1574         for tarball in &tarballs[1..] {
1575             input_tarballs.push(",");
1576             input_tarballs.push(tarball);
1577         }
1578
1579         let mut cmd = rust_installer(builder);
1580         cmd.arg("combine")
1581             .arg("--product-name=Rust")
1582             .arg("--rel-manifest-dir=rustlib")
1583             .arg("--success-message=Rust-is-ready-to-roll.")
1584             .arg("--work-dir").arg(&work)
1585             .arg("--output-dir").arg(&distdir(builder))
1586             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1587             .arg("--legacy-manifest-dirs=rustlib,cargo")
1588             .arg("--input-tarballs").arg(input_tarballs)
1589             .arg("--non-installed-overlay").arg(&overlay);
1590         builder.run(&mut cmd);
1591
1592         let mut license = String::new();
1593         license += &builder.read(&builder.src.join("COPYRIGHT"));
1594         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1595         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1596         license.push_str("\n");
1597         license.push_str("\n");
1598
1599         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1600         let mut rtf = rtf.to_string();
1601         rtf.push_str("\n");
1602         for line in license.lines() {
1603             rtf.push_str(line);
1604             rtf.push_str("\\line ");
1605         }
1606         rtf.push_str("}");
1607
1608         fn filter(contents: &str, marker: &str) -> String {
1609             let start = format!("tool-{}-start", marker);
1610             let end = format!("tool-{}-end", marker);
1611             let mut lines = Vec::new();
1612             let mut omitted = false;
1613             for line in contents.lines() {
1614                 if line.contains(&start) {
1615                     omitted = true;
1616                 } else if line.contains(&end) {
1617                     omitted = false;
1618                 } else if !omitted {
1619                     lines.push(line);
1620                 }
1621             }
1622
1623             lines.join("\n")
1624         }
1625
1626         let xform = |p: &Path| {
1627             let mut contents = t!(fs::read_to_string(p));
1628             if rls_installer.is_none() {
1629                 contents = filter(&contents, "rls");
1630             }
1631             if clippy_installer.is_none() {
1632                 contents = filter(&contents, "clippy");
1633             }
1634             if miri_installer.is_none() {
1635                 contents = filter(&contents, "miri");
1636             }
1637             if rustfmt_installer.is_none() {
1638                 contents = filter(&contents, "rustfmt");
1639             }
1640             let ret = tmp.join(p.file_name().unwrap());
1641             t!(fs::write(&ret, &contents));
1642             ret
1643         };
1644
1645         if target.contains("apple-darwin") {
1646             let pkg = tmp.join("pkg");
1647             let _ = fs::remove_dir_all(&pkg);
1648
1649             let pkgbuild = |component: &str| {
1650                 let mut cmd = Command::new("pkgbuild");
1651                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1652                     .arg("--scripts").arg(pkg.join(component))
1653                     .arg("--nopayload")
1654                     .arg(pkg.join(component).with_extension("pkg"));
1655                 builder.run(&mut cmd);
1656             };
1657
1658             let prepare = |name: &str| {
1659                 builder.create_dir(&pkg.join(name));
1660                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1661                         &pkg.join(name));
1662                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1663                 pkgbuild(name);
1664             };
1665             prepare("rustc");
1666             prepare("cargo");
1667             prepare("rust-docs");
1668             prepare("rust-std");
1669             prepare("rust-analysis");
1670
1671             if rls_installer.is_some() {
1672                 prepare("rls");
1673             }
1674             if clippy_installer.is_some() {
1675                 prepare("clippy");
1676             }
1677             if miri_installer.is_some() {
1678                 prepare("miri");
1679             }
1680
1681             // create an 'uninstall' package
1682             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1683             pkgbuild("uninstall");
1684
1685             builder.create_dir(&pkg.join("res"));
1686             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1687             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1688             let mut cmd = Command::new("productbuild");
1689             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1690                 .arg("--resources").arg(pkg.join("res"))
1691                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1692                                                     pkgname(builder, "rust"),
1693                                                     target)))
1694                 .arg("--package-path").arg(&pkg);
1695             builder.run(&mut cmd);
1696         }
1697
1698         if target.contains("windows") {
1699             let exe = tmp.join("exe");
1700             let _ = fs::remove_dir_all(&exe);
1701
1702             let prepare = |name: &str| {
1703                 builder.create_dir(&exe.join(name));
1704                 let dir = if name == "rust-std" || name == "rust-analysis" {
1705                     format!("{}-{}", name, target)
1706                 } else if name == "rls" {
1707                     "rls-preview".to_string()
1708                 } else if name == "clippy" {
1709                     "clippy-preview".to_string()
1710                 } else if name == "miri" {
1711                     "miri-preview".to_string()
1712                 } else {
1713                     name.to_string()
1714                 };
1715                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1716                             .join(dir),
1717                         &exe.join(name));
1718                 builder.remove(&exe.join(name).join("manifest.in"));
1719             };
1720             prepare("rustc");
1721             prepare("cargo");
1722             prepare("rust-analysis");
1723             prepare("rust-docs");
1724             prepare("rust-std");
1725             if rls_installer.is_some() {
1726                 prepare("rls");
1727             }
1728             if clippy_installer.is_some() {
1729                 prepare("clippy");
1730             }
1731             if miri_installer.is_some() {
1732                 prepare("miri");
1733             }
1734             if target.contains("windows-gnu") {
1735                 prepare("rust-mingw");
1736             }
1737
1738             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1739             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1740             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1741             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1742             builder.create(&exe.join("LICENSE.txt"), &license);
1743
1744             // Generate exe installer
1745             let mut cmd = Command::new("iscc");
1746             cmd.arg("rust.iss")
1747                 .current_dir(&exe);
1748             if target.contains("windows-gnu") {
1749                 cmd.arg("/dMINGW");
1750             }
1751             add_env(builder, &mut cmd, target);
1752             builder.run(&mut cmd);
1753             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1754                     &distdir(builder),
1755                     0o755);
1756
1757             // Generate msi installer
1758             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1759             let heat = wix.join("bin/heat.exe");
1760             let candle = wix.join("bin/candle.exe");
1761             let light = wix.join("bin/light.exe");
1762
1763             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1764             builder.run(Command::new(&heat)
1765                             .current_dir(&exe)
1766                             .arg("dir")
1767                             .arg("rustc")
1768                             .args(&heat_flags)
1769                             .arg("-cg").arg("RustcGroup")
1770                             .arg("-dr").arg("Rustc")
1771                             .arg("-var").arg("var.RustcDir")
1772                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1773             builder.run(Command::new(&heat)
1774                             .current_dir(&exe)
1775                             .arg("dir")
1776                             .arg("rust-docs")
1777                             .args(&heat_flags)
1778                             .arg("-cg").arg("DocsGroup")
1779                             .arg("-dr").arg("Docs")
1780                             .arg("-var").arg("var.DocsDir")
1781                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1782                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1783             builder.run(Command::new(&heat)
1784                             .current_dir(&exe)
1785                             .arg("dir")
1786                             .arg("cargo")
1787                             .args(&heat_flags)
1788                             .arg("-cg").arg("CargoGroup")
1789                             .arg("-dr").arg("Cargo")
1790                             .arg("-var").arg("var.CargoDir")
1791                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1792                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1793             builder.run(Command::new(&heat)
1794                             .current_dir(&exe)
1795                             .arg("dir")
1796                             .arg("rust-std")
1797                             .args(&heat_flags)
1798                             .arg("-cg").arg("StdGroup")
1799                             .arg("-dr").arg("Std")
1800                             .arg("-var").arg("var.StdDir")
1801                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1802             if rls_installer.is_some() {
1803                 builder.run(Command::new(&heat)
1804                                 .current_dir(&exe)
1805                                 .arg("dir")
1806                                 .arg("rls")
1807                                 .args(&heat_flags)
1808                                 .arg("-cg").arg("RlsGroup")
1809                                 .arg("-dr").arg("Rls")
1810                                 .arg("-var").arg("var.RlsDir")
1811                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1812                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1813             }
1814             if clippy_installer.is_some() {
1815                 builder.run(Command::new(&heat)
1816                                 .current_dir(&exe)
1817                                 .arg("dir")
1818                                 .arg("clippy")
1819                                 .args(&heat_flags)
1820                                 .arg("-cg").arg("ClippyGroup")
1821                                 .arg("-dr").arg("Clippy")
1822                                 .arg("-var").arg("var.ClippyDir")
1823                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1824                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1825             }
1826             if miri_installer.is_some() {
1827                 builder.run(Command::new(&heat)
1828                                 .current_dir(&exe)
1829                                 .arg("dir")
1830                                 .arg("miri")
1831                                 .args(&heat_flags)
1832                                 .arg("-cg").arg("MiriGroup")
1833                                 .arg("-dr").arg("Miri")
1834                                 .arg("-var").arg("var.MiriDir")
1835                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1836                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1837             }
1838             builder.run(Command::new(&heat)
1839                             .current_dir(&exe)
1840                             .arg("dir")
1841                             .arg("rust-analysis")
1842                             .args(&heat_flags)
1843                             .arg("-cg").arg("AnalysisGroup")
1844                             .arg("-dr").arg("Analysis")
1845                             .arg("-var").arg("var.AnalysisDir")
1846                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1847                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1848             if target.contains("windows-gnu") {
1849                 builder.run(Command::new(&heat)
1850                                 .current_dir(&exe)
1851                                 .arg("dir")
1852                                 .arg("rust-mingw")
1853                                 .args(&heat_flags)
1854                                 .arg("-cg").arg("GccGroup")
1855                                 .arg("-dr").arg("Gcc")
1856                                 .arg("-var").arg("var.GccDir")
1857                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1858             }
1859
1860             let candle = |input: &Path| {
1861                 let output = exe.join(input.file_stem().unwrap())
1862                                 .with_extension("wixobj");
1863                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1864                 let mut cmd = Command::new(&candle);
1865                 cmd.current_dir(&exe)
1866                     .arg("-nologo")
1867                     .arg("-dRustcDir=rustc")
1868                     .arg("-dDocsDir=rust-docs")
1869                     .arg("-dCargoDir=cargo")
1870                     .arg("-dStdDir=rust-std")
1871                     .arg("-dAnalysisDir=rust-analysis")
1872                     .arg("-arch").arg(&arch)
1873                     .arg("-out").arg(&output)
1874                     .arg(&input);
1875                 add_env(builder, &mut cmd, target);
1876
1877                 if rls_installer.is_some() {
1878                     cmd.arg("-dRlsDir=rls");
1879                 }
1880                 if clippy_installer.is_some() {
1881                     cmd.arg("-dClippyDir=clippy");
1882                 }
1883                 if miri_installer.is_some() {
1884                     cmd.arg("-dMiriDir=miri");
1885                 }
1886                 if target.contains("windows-gnu") {
1887                     cmd.arg("-dGccDir=rust-mingw");
1888                 }
1889                 builder.run(&mut cmd);
1890             };
1891             candle(&xform(&etc.join("msi/rust.wxs")));
1892             candle(&etc.join("msi/ui.wxs"));
1893             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1894             candle("RustcGroup.wxs".as_ref());
1895             candle("DocsGroup.wxs".as_ref());
1896             candle("CargoGroup.wxs".as_ref());
1897             candle("StdGroup.wxs".as_ref());
1898             if rls_installer.is_some() {
1899                 candle("RlsGroup.wxs".as_ref());
1900             }
1901             if clippy_installer.is_some() {
1902                 candle("ClippyGroup.wxs".as_ref());
1903             }
1904             if miri_installer.is_some() {
1905                 candle("MiriGroup.wxs".as_ref());
1906             }
1907             candle("AnalysisGroup.wxs".as_ref());
1908
1909             if target.contains("windows-gnu") {
1910                 candle("GccGroup.wxs".as_ref());
1911             }
1912
1913             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1914             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1915             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1916
1917             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1918             let mut cmd = Command::new(&light);
1919             cmd.arg("-nologo")
1920                 .arg("-ext").arg("WixUIExtension")
1921                 .arg("-ext").arg("WixUtilExtension")
1922                 .arg("-out").arg(exe.join(&filename))
1923                 .arg("rust.wixobj")
1924                 .arg("ui.wixobj")
1925                 .arg("rustwelcomedlg.wixobj")
1926                 .arg("RustcGroup.wixobj")
1927                 .arg("DocsGroup.wixobj")
1928                 .arg("CargoGroup.wixobj")
1929                 .arg("StdGroup.wixobj")
1930                 .arg("AnalysisGroup.wixobj")
1931                 .current_dir(&exe);
1932
1933             if rls_installer.is_some() {
1934                 cmd.arg("RlsGroup.wixobj");
1935             }
1936             if clippy_installer.is_some() {
1937                 cmd.arg("ClippyGroup.wixobj");
1938             }
1939             if miri_installer.is_some() {
1940                 cmd.arg("MiriGroup.wixobj");
1941             }
1942
1943             if target.contains("windows-gnu") {
1944                 cmd.arg("GccGroup.wixobj");
1945             }
1946             // ICE57 wrongly complains about the shortcuts
1947             cmd.arg("-sice:ICE57");
1948
1949             builder.run(&mut cmd);
1950
1951             if !builder.config.dry_run {
1952                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1953             }
1954         }
1955     }
1956 }
1957
1958 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1959     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1960     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1961        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1962        .env("CFG_RELEASE", builder.rust_release())
1963        .env("CFG_VER_MAJOR", parts.next().unwrap())
1964        .env("CFG_VER_MINOR", parts.next().unwrap())
1965        .env("CFG_VER_PATCH", parts.next().unwrap())
1966        .env("CFG_VER_BUILD", "0") // just needed to build
1967        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1968        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1969        .env("CFG_BUILD", target)
1970        .env("CFG_CHANNEL", &builder.config.channel);
1971
1972     if target.contains("windows-gnu") {
1973        cmd.env("CFG_MINGW", "1")
1974           .env("CFG_ABI", "GNU");
1975     } else {
1976        cmd.env("CFG_MINGW", "0")
1977           .env("CFG_ABI", "MSVC");
1978     }
1979
1980     if target.contains("x86_64") {
1981        cmd.env("CFG_PLATFORM", "x64");
1982     } else {
1983        cmd.env("CFG_PLATFORM", "x86");
1984     }
1985 }
1986
1987 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1988 pub struct HashSign;
1989
1990 impl Step for HashSign {
1991     type Output = ();
1992     const ONLY_HOSTS: bool = true;
1993
1994     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1995         run.path("hash-and-sign")
1996     }
1997
1998     fn make_run(run: RunConfig<'_>) {
1999         run.builder.ensure(HashSign);
2000     }
2001
2002     fn run(self, builder: &Builder<'_>) {
2003         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2004         if builder.config.dry_run {
2005             return;
2006         }
2007         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2008             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2009         });
2010         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2011             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2012         });
2013         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2014             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2015         });
2016         let pass = t!(fs::read_to_string(&file));
2017
2018         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2019
2020         cmd.arg(sign);
2021         cmd.arg(distdir(builder));
2022         cmd.arg(today.trim());
2023         cmd.arg(builder.rust_package_vers());
2024         cmd.arg(addr);
2025         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2026         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2027         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2028         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2029         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2030         cmd.arg(builder.llvm_tools_package_vers());
2031         cmd.arg(builder.lldb_package_vers());
2032
2033         builder.create_dir(&distdir(builder));
2034
2035         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2036         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2037         let status = t!(child.wait());
2038         assert!(status.success());
2039     }
2040 }
2041
2042 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2043 // LLVM tools are linked dynamically.
2044 // Note: This function does no yet support Windows but we also don't support
2045 //       linking LLVM tools dynamically on Windows yet.
2046 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2047                                 target: Interned<String>,
2048                                 sysroot: &Path) {
2049     let src_libdir = builder
2050         .llvm_out(target)
2051         .join("lib");
2052     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2053     t!(fs::create_dir_all(&dst_libdir));
2054
2055     if target.contains("apple-darwin") {
2056         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2057         if llvm_dylib_path.exists() {
2058             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2059         }
2060         return
2061     }
2062
2063     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2064     // Since tools link to the latter rather than the former, we have to
2065     // follow the symlink to find out what to distribute.
2066     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2067     if llvm_dylib_path.exists() {
2068         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2069             panic!("dist: Error calling canonicalize path `{}`: {}",
2070                    llvm_dylib_path.display(), e);
2071         });
2072
2073
2074         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2075     }
2076 }
2077
2078 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2079 pub struct LlvmTools {
2080     pub stage: u32,
2081     pub target: Interned<String>,
2082 }
2083
2084 impl Step for LlvmTools {
2085     type Output = Option<PathBuf>;
2086     const ONLY_HOSTS: bool = true;
2087
2088     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2089         run.path("llvm-tools")
2090     }
2091
2092     fn make_run(run: RunConfig<'_>) {
2093         run.builder.ensure(LlvmTools {
2094             stage: run.builder.top_stage,
2095             target: run.target,
2096         });
2097     }
2098
2099     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2100         let stage = self.stage;
2101         let target = self.target;
2102         assert!(builder.config.extended);
2103
2104         /* run only if llvm-config isn't used */
2105         if let Some(config) = builder.config.target_config.get(&target) {
2106             if let Some(ref _s) = config.llvm_config {
2107                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2108                     stage, target));
2109                 return None;
2110             }
2111         }
2112
2113         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2114         let src = builder.src.join("src/llvm-project/llvm");
2115         let name = pkgname(builder, "llvm-tools");
2116
2117         let tmp = tmpdir(builder);
2118         let image = tmp.join("llvm-tools-image");
2119         drop(fs::remove_dir_all(&image));
2120
2121         // Prepare the image directory
2122         let src_bindir = builder
2123             .llvm_out(target)
2124             .join("bin");
2125         let dst_bindir = image.join("lib/rustlib")
2126             .join(&*target)
2127             .join("bin");
2128         t!(fs::create_dir_all(&dst_bindir));
2129         for tool in LLVM_TOOLS {
2130             let exe = src_bindir.join(exe(tool, &target));
2131             builder.install(&exe, &dst_bindir, 0o755);
2132         }
2133
2134         // Prepare the overlay
2135         let overlay = tmp.join("llvm-tools-overlay");
2136         drop(fs::remove_dir_all(&overlay));
2137         builder.create_dir(&overlay);
2138         builder.install(&src.join("README.txt"), &overlay, 0o644);
2139         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2140         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2141
2142         // Generate the installer tarball
2143         let mut cmd = rust_installer(builder);
2144         cmd.arg("generate")
2145             .arg("--product-name=Rust")
2146             .arg("--rel-manifest-dir=rustlib")
2147             .arg("--success-message=llvm-tools-installed.")
2148             .arg("--image-dir").arg(&image)
2149             .arg("--work-dir").arg(&tmpdir(builder))
2150             .arg("--output-dir").arg(&distdir(builder))
2151             .arg("--non-installed-overlay").arg(&overlay)
2152             .arg(format!("--package-name={}-{}", name, target))
2153             .arg("--legacy-manifest-dirs=rustlib,cargo")
2154             .arg("--component-name=llvm-tools-preview");
2155
2156
2157         builder.run(&mut cmd);
2158         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2159     }
2160 }
2161
2162 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2163 pub struct Lldb {
2164     pub target: Interned<String>,
2165 }
2166
2167 impl Step for Lldb {
2168     type Output = Option<PathBuf>;
2169     const ONLY_HOSTS: bool = true;
2170     const DEFAULT: bool = true;
2171
2172     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2173         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2174     }
2175
2176     fn make_run(run: RunConfig<'_>) {
2177         run.builder.ensure(Lldb {
2178             target: run.target,
2179         });
2180     }
2181
2182     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2183         let target = self.target;
2184
2185         if builder.config.dry_run {
2186             return None;
2187         }
2188
2189         let bindir = builder
2190             .llvm_out(target)
2191             .join("bin");
2192         let lldb_exe = bindir.join(exe("lldb", &target));
2193         if !lldb_exe.exists() {
2194             return None;
2195         }
2196
2197         builder.info(&format!("Dist Lldb ({})", target));
2198         let src = builder.src.join("src/llvm-project/lldb");
2199         let name = pkgname(builder, "lldb");
2200
2201         let tmp = tmpdir(builder);
2202         let image = tmp.join("lldb-image");
2203         drop(fs::remove_dir_all(&image));
2204
2205         // Prepare the image directory
2206         let root = image.join("lib/rustlib").join(&*target);
2207         let dst = root.join("bin");
2208         t!(fs::create_dir_all(&dst));
2209         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2210             let exe = bindir.join(exe(program, &target));
2211             builder.install(&exe, &dst, 0o755);
2212         }
2213
2214         // The libraries.
2215         let libdir = builder.llvm_out(target).join("lib");
2216         let dst = root.join("lib");
2217         t!(fs::create_dir_all(&dst));
2218         for entry in t!(fs::read_dir(&libdir)) {
2219             let entry = entry.unwrap();
2220             if let Ok(name) = entry.file_name().into_string() {
2221                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2222                     if t!(entry.file_type()).is_symlink() {
2223                         builder.copy_to_folder(&entry.path(), &dst);
2224                     } else {
2225                        builder.install(&entry.path(), &dst, 0o755);
2226                     }
2227                 }
2228             }
2229         }
2230
2231         // The lldb scripts might be installed in lib/python$version
2232         // or in lib64/python$version.  If lib64 exists, use it;
2233         // otherwise lib.
2234         let libdir = builder.llvm_out(target).join("lib64");
2235         let (libdir, libdir_name) = if libdir.exists() {
2236             (libdir, "lib64")
2237         } else {
2238             (builder.llvm_out(target).join("lib"), "lib")
2239         };
2240         for entry in t!(fs::read_dir(&libdir)) {
2241             let entry = t!(entry);
2242             if let Ok(name) = entry.file_name().into_string() {
2243                 if name.starts_with("python") {
2244                     let dst = root.join(libdir_name)
2245                         .join(entry.file_name());
2246                     t!(fs::create_dir_all(&dst));
2247                     builder.cp_r(&entry.path(), &dst);
2248                     break;
2249                 }
2250             }
2251         }
2252
2253         // Prepare the overlay
2254         let overlay = tmp.join("lldb-overlay");
2255         drop(fs::remove_dir_all(&overlay));
2256         builder.create_dir(&overlay);
2257         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2258         builder.create(&overlay.join("version"), &builder.lldb_vers());
2259
2260         // Generate the installer tarball
2261         let mut cmd = rust_installer(builder);
2262         cmd.arg("generate")
2263             .arg("--product-name=Rust")
2264             .arg("--rel-manifest-dir=rustlib")
2265             .arg("--success-message=lldb-installed.")
2266             .arg("--image-dir").arg(&image)
2267             .arg("--work-dir").arg(&tmpdir(builder))
2268             .arg("--output-dir").arg(&distdir(builder))
2269             .arg("--non-installed-overlay").arg(&overlay)
2270             .arg(format!("--package-name={}-{}", name, target))
2271             .arg("--legacy-manifest-dirs=rustlib,cargo")
2272             .arg("--component-name=lldb-preview");
2273
2274
2275         builder.run(&mut cmd);
2276         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2277     }
2278 }