src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::output;
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{libdir, is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time;
  27
  28 pub fn pkgname(builder: &Builder, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub stage: u32,
  72     pub host: Interned<String>,
  73 }
  74
  75 impl Step for Docs {
  76     type Output = PathBuf;
  77     const DEFAULT: bool = true;
  78
  79     fn should_run(run: ShouldRun) -> ShouldRun {
  80         run.path("src/doc")
  81     }
  82
  83     fn make_run(run: RunConfig) {
  84         run.builder.ensure(Docs {
  85             stage: run.builder.top_stage,
  86             host: run.target,
  87         });
  88     }
  89
  90     /// Builds the `rust-docs` installer component.
  91     fn run(self, builder: &Builder) -> PathBuf {
  92         let host = self.host;
  93
  94         let name = pkgname(builder, "rust-docs");
  95
  96         builder.info(&format!("Dist docs ({})", host));
  97         if !builder.config.docs {
  98             builder.info("\tskipping - docs disabled");
  99             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 100         }
 101
 102         builder.default_doc(None);
 103
 104         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 105         let _ = fs::remove_dir_all(&image);
 106
 107         let dst = image.join("share/doc/rust/html");
 108         t!(fs::create_dir_all(&dst));
 109         let src = builder.doc_out(host);
 110         builder.cp_r(&src, &dst);
 111
 112         let mut cmd = rust_installer(builder);
 113         cmd.arg("generate")
 114            .arg("--product-name=Rust-Documentation")
 115            .arg("--rel-manifest-dir=rustlib")
 116            .arg("--success-message=Rust-documentation-is-installed.")
 117            .arg("--image-dir").arg(&image)
 118            .arg("--work-dir").arg(&tmpdir(builder))
 119            .arg("--output-dir").arg(&distdir(builder))
 120            .arg(format!("--package-name={}-{}", name, host))
 121            .arg("--component-name=rust-docs")
 122            .arg("--legacy-manifest-dirs=rustlib,cargo")
 123            .arg("--bulk-dirs=share/doc/rust/html");
 124         builder.run(&mut cmd);
 125         builder.remove_dir(&image);
 126
 127         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 128     }
 129 }
 130
 131 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 132 pub struct RustcDocs {
 133     pub stage: u32,
 134     pub host: Interned<String>,
 135 }
 136
 137 impl Step for RustcDocs {
 138     type Output = PathBuf;
 139     const DEFAULT: bool = true;
 140
 141     fn should_run(run: ShouldRun) -> ShouldRun {
 142         run.path("src/librustc")
 143     }
 144
 145     fn make_run(run: RunConfig) {
 146         run.builder.ensure(RustcDocs {
 147             stage: run.builder.top_stage,
 148             host: run.target,
 149         });
 150     }
 151
 152     /// Builds the `rustc-docs` installer component.
 153     fn run(self, builder: &Builder) -> PathBuf {
 154         let host = self.host;
 155
 156         let name = pkgname(builder, "rustc-docs");
 157
 158         builder.info(&format!("Dist compiler docs ({})", host));
 159         if !builder.config.compiler_docs {
 160             builder.info("\tskipping - compiler docs disabled");
 161             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 162         }
 163
 164         builder.default_doc(None);
 165
 166         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 167         let _ = fs::remove_dir_all(&image);
 168
 169         let dst = image.join("share/doc/rust/html");
 170         t!(fs::create_dir_all(&dst));
 171         let src = builder.compiler_doc_out(host);
 172         builder.cp_r(&src, &dst);
 173
 174         let mut cmd = rust_installer(builder);
 175         cmd.arg("generate")
 176            .arg("--product-name=Rustc-Documentation")
 177            .arg("--rel-manifest-dir=rustlib")
 178            .arg("--success-message=Rustc-documentation-is-installed.")
 179            .arg("--image-dir").arg(&image)
 180            .arg("--work-dir").arg(&tmpdir(builder))
 181            .arg("--output-dir").arg(&distdir(builder))
 182            .arg(format!("--package-name={}-{}", name, host))
 183            .arg("--component-name=rustc-docs")
 184            .arg("--legacy-manifest-dirs=rustlib,cargo")
 185            .arg("--bulk-dirs=share/doc/rust/html");
 186         builder.run(&mut cmd);
 187         builder.remove_dir(&image);
 188
 189         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 190     }
 191 }
 192
 193 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 194     let mut found = Vec::with_capacity(files.len());
 195
 196     for file in files {
 197         let file_path =
 198             path.iter()
 199                 .map(|dir| dir.join(file))
 200                 .find(|p| p.exists());
 201
 202         if let Some(file_path) = file_path {
 203             found.push(file_path);
 204         } else {
 205             panic!("Could not find '{}' in {:?}", file, path);
 206         }
 207     }
 208
 209     found
 210 }
 211
 212 fn make_win_dist(
 213     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 214 ) {
 215     //Ask gcc where it keeps its stuff
 216     let mut cmd = Command::new(builder.cc(target_triple));
 217     cmd.arg("-print-search-dirs");
 218     let gcc_out = output(&mut cmd);
 219
 220     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 221     let mut lib_path = Vec::new();
 222
 223     for line in gcc_out.lines() {
 224         let idx = line.find(':').unwrap();
 225         let key = &line[..idx];
 226         let trim_chars: &[_] = &[' ', '='];
 227         let value =
 228             line[(idx + 1)..]
 229                 .trim_left_matches(trim_chars)
 230                 .split(';')
 231                 .map(PathBuf::from);
 232
 233         if key == "programs" {
 234             bin_path.extend(value);
 235         } else if key == "libraries" {
 236             lib_path.extend(value);
 237         }
 238     }
 239
 240     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 241     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 242     if target_triple.starts_with("i686-") {
 243         rustc_dlls.push("libgcc_s_dw2-1.dll");
 244     } else {
 245         rustc_dlls.push("libgcc_s_seh-1.dll");
 246     }
 247
 248     let target_libs = [ //MinGW libs
 249         "libgcc.a",
 250         "libgcc_eh.a",
 251         "libgcc_s.a",
 252         "libm.a",
 253         "libmingw32.a",
 254         "libmingwex.a",
 255         "libstdc++.a",
 256         "libiconv.a",
 257         "libmoldname.a",
 258         "libpthread.a",
 259         //Windows import libs
 260         "libadvapi32.a",
 261         "libbcrypt.a",
 262         "libcomctl32.a",
 263         "libcomdlg32.a",
 264         "libcredui.a",
 265         "libcrypt32.a",
 266         "libdbghelp.a",
 267         "libgdi32.a",
 268         "libimagehlp.a",
 269         "libiphlpapi.a",
 270         "libkernel32.a",
 271         "libmsimg32.a",
 272         "libmsvcrt.a",
 273         "libodbc32.a",
 274         "libole32.a",
 275         "liboleaut32.a",
 276         "libopengl32.a",
 277         "libpsapi.a",
 278         "librpcrt4.a",
 279         "libsecur32.a",
 280         "libsetupapi.a",
 281         "libshell32.a",
 282         "libsynchronization.a",
 283         "libuser32.a",
 284         "libuserenv.a",
 285         "libuuid.a",
 286         "libwinhttp.a",
 287         "libwinmm.a",
 288         "libwinspool.a",
 289         "libws2_32.a",
 290         "libwsock32.a",
 291     ];
 292
 293     //Find mingw artifacts we want to bundle
 294     let target_tools = find_files(&target_tools, &bin_path);
 295     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 296     let target_libs = find_files(&target_libs, &lib_path);
 297
 298     // Copy runtime dlls next to rustc.exe
 299     let dist_bin_dir = rust_root.join("bin/");
 300     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 301     for src in rustc_dlls {
 302         builder.copy_to_folder(&src, &dist_bin_dir);
 303     }
 304
 305     //Copy platform tools to platform-specific bin directory
 306     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 307     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 308     for src in target_tools {
 309         builder.copy_to_folder(&src, &target_bin_dir);
 310     }
 311
 312     // Warn windows-gnu users that the bundled GCC cannot compile C files
 313     builder.create(
 314         &target_bin_dir.join("GCC-WARNING.txt"),
 315         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 316          used as a linker. In order to be able to compile projects containing C code use\
 317          the GCC provided by MinGW or Cygwin."
 318     );
 319
 320     //Copy platform libs to platform-specific lib directory
 321     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 322     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 323     for src in target_libs {
 324         builder.copy_to_folder(&src, &target_lib_dir);
 325     }
 326 }
 327
 328 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 329 pub struct Mingw {
 330     pub host: Interned<String>,
 331 }
 332
 333 impl Step for Mingw {
 334     type Output = Option<PathBuf>;
 335     const DEFAULT: bool = true;
 336
 337     fn should_run(run: ShouldRun) -> ShouldRun {
 338         run.never()
 339     }
 340
 341     fn make_run(run: RunConfig) {
 342         run.builder.ensure(Mingw { host: run.target });
 343     }
 344
 345     /// Build the `rust-mingw` installer component.
 346     ///
 347     /// This contains all the bits and pieces to run the MinGW Windows targets
 348     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 349     fn run(self, builder: &Builder) -> Option<PathBuf> {
 350         let host = self.host;
 351
 352         if !host.contains("pc-windows-gnu") {
 353             return None;
 354         }
 355
 356         builder.info(&format!("Dist mingw ({})", host));
 357         let name = pkgname(builder, "rust-mingw");
 358         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 359         let _ = fs::remove_dir_all(&image);
 360         t!(fs::create_dir_all(&image));
 361
 362         // The first argument is a "temporary directory" which is just
 363         // thrown away (this contains the runtime DLLs included in the rustc package
 364         // above) and the second argument is where to place all the MinGW components
 365         // (which is what we want).
 366         make_win_dist(&tmpdir(builder), &image, host, &builder);
 367
 368         let mut cmd = rust_installer(builder);
 369         cmd.arg("generate")
 370            .arg("--product-name=Rust-MinGW")
 371            .arg("--rel-manifest-dir=rustlib")
 372            .arg("--success-message=Rust-MinGW-is-installed.")
 373            .arg("--image-dir").arg(&image)
 374            .arg("--work-dir").arg(&tmpdir(builder))
 375            .arg("--output-dir").arg(&distdir(builder))
 376            .arg(format!("--package-name={}-{}", name, host))
 377            .arg("--component-name=rust-mingw")
 378            .arg("--legacy-manifest-dirs=rustlib,cargo");
 379         builder.run(&mut cmd);
 380         t!(fs::remove_dir_all(&image));
 381         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 382     }
 383 }
 384
 385 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct Rustc {
 387     pub compiler: Compiler,
 388 }
 389
 390 impl Step for Rustc {
 391     type Output = PathBuf;
 392     const DEFAULT: bool = true;
 393     const ONLY_HOSTS: bool = true;
 394
 395     fn should_run(run: ShouldRun) -> ShouldRun {
 396         run.path("src/librustc")
 397     }
 398
 399     fn make_run(run: RunConfig) {
 400         run.builder.ensure(Rustc {
 401             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 402         });
 403     }
 404
 405     /// Creates the `rustc` installer component.
 406     fn run(self, builder: &Builder) -> PathBuf {
 407         let compiler = self.compiler;
 408         let host = self.compiler.host;
 409
 410         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 411         let name = pkgname(builder, "rustc");
 412         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 413         let _ = fs::remove_dir_all(&image);
 414         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 415         let _ = fs::remove_dir_all(&overlay);
 416
 417         // Prepare the rustc "image", what will actually end up getting installed
 418         prepare_image(builder, compiler, &image);
 419
 420         // Prepare the overlay which is part of the tarball but won't actually be
 421         // installed
 422         let cp = |file: &str| {
 423             builder.install(&builder.src.join(file), &overlay, 0o644);
 424         };
 425         cp("COPYRIGHT");
 426         cp("LICENSE-APACHE");
 427         cp("LICENSE-MIT");
 428         cp("README.md");
 429         // tiny morsel of metadata is used by rust-packaging
 430         let version = builder.rust_version();
 431         builder.create(&overlay.join("version"), &version);
 432         if let Some(sha) = builder.rust_sha() {
 433             builder.create(&overlay.join("git-commit-hash"), &sha);
 434         }
 435
 436         // On MinGW we've got a few runtime DLL dependencies that we need to
 437         // include. The first argument to this script is where to put these DLLs
 438         // (the image we're creating), and the second argument is a junk directory
 439         // to ignore all other MinGW stuff the script creates.
 440         //
 441         // On 32-bit MinGW we're always including a DLL which needs some extra
 442         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 443         // anything requiring us to distribute a license, but it's likely the
 444         // install will *also* include the rust-mingw package, which also needs
 445         // licenses, so to be safe we just include it here in all MinGW packages.
 446         if host.contains("pc-windows-gnu") {
 447             make_win_dist(&image, &tmpdir(builder), host, builder);
 448
 449             let dst = image.join("share/doc");
 450             t!(fs::create_dir_all(&dst));
 451             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 452         }
 453
 454         // Finally, wrap everything up in a nice tarball!
 455         let mut cmd = rust_installer(builder);
 456         cmd.arg("generate")
 457            .arg("--product-name=Rust")
 458            .arg("--rel-manifest-dir=rustlib")
 459            .arg("--success-message=Rust-is-ready-to-roll.")
 460            .arg("--image-dir").arg(&image)
 461            .arg("--work-dir").arg(&tmpdir(builder))
 462            .arg("--output-dir").arg(&distdir(builder))
 463            .arg("--non-installed-overlay").arg(&overlay)
 464            .arg(format!("--package-name={}-{}", name, host))
 465            .arg("--component-name=rustc")
 466            .arg("--legacy-manifest-dirs=rustlib,cargo");
 467         builder.run(&mut cmd);
 468         builder.remove_dir(&image);
 469         builder.remove_dir(&overlay);
 470
 471         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 472
 473         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 474             let host = compiler.host;
 475             let src = builder.sysroot(compiler);
 476             let libdir = libdir(&host);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 483
 484             // Copy runtime DLLs needed by the compiler
 485             if libdir != "bin" {
 486                 for entry in builder.read_dir(&src.join(libdir)) {
 487                     let name = entry.file_name();
 488                     if let Some(s) = name.to_str() {
 489                         if is_dylib(s) {
 490                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 499             let backends_dst = image.join(&backends_rel);
 500             t!(fs::create_dir_all(&backends_dst));
 501             builder.cp_r(&backends_src, &backends_dst);
 502
 503             // Copy libLLVM.so to the lib dir as well, if needed. While not
 504             // technically needed by rustc itself it's needed by lots of other
 505             // components like the llvm tools and LLD. LLD is included below and
 506             // tools/LLDB come later, so let's just throw it in the rustc
 507             // component for now.
 508             maybe_install_llvm_dylib(builder, host, image);
 509
 510             // Copy over lld if it's there
 511             if builder.config.lld_enabled {
 512                 let exe = exe("rust-lld", &compiler.host);
 513                 let src = builder.sysroot_libdir(compiler, host)
 514                     .parent()
 515                     .unwrap()
 516                     .join("bin")
 517                     .join(&exe);
 518                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 519                 let dst = image.join("lib/rustlib")
 520                     .join(&*host)
 521                     .join("bin")
 522                     .join(&exe);
 523                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 524                 builder.copy(&src, &dst);
 525             }
 526
 527             // Man pages
 528             t!(fs::create_dir_all(image.join("share/man/man1")));
 529             let man_src = builder.src.join("src/doc/man");
 530             let man_dst = image.join("share/man/man1");
 531             let month_year = t!(time::strftime("%B %Y", &time::now()));
 532             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 533             // to hardlink, and we don't want to edit the source templates
 534             for file_entry in builder.read_dir(&man_src) {
 535                 let page_src = file_entry.path();
 536                 let page_dst = man_dst.join(file_entry.file_name());
 537                 t!(fs::copy(&page_src, &page_dst));
 538                 // template in month/year and version number
 539                 builder.replace_in_file(&page_dst,
 540                                 &[("<INSERT DATE HERE>", &month_year),
 541                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 542             }
 543
 544             // Debugger scripts
 545             builder.ensure(DebuggerScripts {
 546                 sysroot: INTERNER.intern_path(image.to_owned()),
 547                 host,
 548             });
 549
 550             // Misc license info
 551             let cp = |file: &str| {
 552                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 553             };
 554             cp("COPYRIGHT");
 555             cp("LICENSE-APACHE");
 556             cp("LICENSE-MIT");
 557             cp("README.md");
 558         }
 559     }
 560 }
 561
 562 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 563 pub struct DebuggerScripts {
 564     pub sysroot: Interned<PathBuf>,
 565     pub host: Interned<String>,
 566 }
 567
 568 impl Step for DebuggerScripts {
 569     type Output = ();
 570
 571     fn should_run(run: ShouldRun) -> ShouldRun {
 572         run.path("src/lldb_batchmode.py")
 573     }
 574
 575     fn make_run(run: RunConfig) {
 576         run.builder.ensure(DebuggerScripts {
 577             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 578             host: run.target,
 579         });
 580     }
 581
 582     /// Copies debugger scripts for `target` into the `sysroot` specified.
 583     fn run(self, builder: &Builder) {
 584         let host = self.host;
 585         let sysroot = self.sysroot;
 586         let dst = sysroot.join("lib/rustlib/etc");
 587         t!(fs::create_dir_all(&dst));
 588         let cp_debugger_script = |file: &str| {
 589             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 590         };
 591         if host.contains("windows-msvc") {
 592             // windbg debugger scripts
 593             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 594                 0o755);
 595
 596             cp_debugger_script("natvis/intrinsic.natvis");
 597             cp_debugger_script("natvis/liballoc.natvis");
 598             cp_debugger_script("natvis/libcore.natvis");
 599         } else {
 600             cp_debugger_script("debugger_pretty_printers_common.py");
 601
 602             // gdb debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 604                     0o755);
 605
 606             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 607             cp_debugger_script("gdb_rust_pretty_printing.py");
 608
 609             // lldb debugger scripts
 610             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 611                     0o755);
 612
 613             cp_debugger_script("lldb_rust_formatters.py");
 614         }
 615     }
 616 }
 617
 618 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 619 pub struct Std {
 620     pub compiler: Compiler,
 621     pub target: Interned<String>,
 622 }
 623
 624 impl Step for Std {
 625     type Output = PathBuf;
 626     const DEFAULT: bool = true;
 627
 628     fn should_run(run: ShouldRun) -> ShouldRun {
 629         run.path("src/libstd")
 630     }
 631
 632     fn make_run(run: RunConfig) {
 633         run.builder.ensure(Std {
 634             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 635             target: run.target,
 636         });
 637     }
 638
 639     fn run(self, builder: &Builder) -> PathBuf {
 640         let compiler = self.compiler;
 641         let target = self.target;
 642
 643         let name = pkgname(builder, "rust-std");
 644         builder.info(&format!("Dist std stage{} ({} -> {})",
 645             compiler.stage, &compiler.host, target));
 646
 647         // The only true set of target libraries came from the build triple, so
 648         // let's reduce redundant work by only producing archives from that host.
 649         if compiler.host != builder.config.build {
 650             builder.info("\tskipping, not a build host");
 651             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 652         }
 653
 654         // We want to package up as many target libraries as possible
 655         // for the `rust-std` package, so if this is a host target we
 656         // depend on librustc and otherwise we just depend on libtest.
 657         if builder.hosts.iter().any(|t| t == target) {
 658             builder.ensure(compile::Rustc { compiler, target });
 659         } else {
 660             if builder.no_std(target) == Some(true) {
 661                 // the `test` doesn't compile for no-std targets
 662                 builder.ensure(compile::Std { compiler, target });
 663             } else {
 664                 builder.ensure(compile::Test { compiler, target });
 665             }
 666         }
 667
 668         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 669         let _ = fs::remove_dir_all(&image);
 670
 671         let dst = image.join("lib/rustlib").join(target);
 672         t!(fs::create_dir_all(&dst));
 673         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 674         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 675         builder.cp_filtered(&src, &dst, &|path| {
 676             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 677                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 678                     return false
 679                 }
 680                 if name == "bin" {
 681                     return false
 682                 }
 683                 if name.contains("LLVM") {
 684                     return false
 685                 }
 686             }
 687             true
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/liballoc",
 861             "src/libcore",
 862             "src/libpanic_abort",
 863             "src/libpanic_unwind",
 864             "src/librustc_asan",
 865             "src/librustc_lsan",
 866             "src/librustc_msan",
 867             "src/librustc_tsan",
 868             "src/libstd",
 869             "src/libunwind",
 870             "src/libtest",
 871             "src/libterm",
 872             "src/libprofiler_builtins",
 873             "src/stdsimd",
 874             "src/libproc_macro",
 875             "src/tools/rustc-std-workspace-core",
 876         ];
 877
 878         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 879         for file in src_files.iter() {
 880             builder.copy(&builder.src.join(file), &dst_src.join(file));
 881         }
 882
 883         // Create source tarball in rust-installer format
 884         let mut cmd = rust_installer(builder);
 885         cmd.arg("generate")
 886            .arg("--product-name=Rust")
 887            .arg("--rel-manifest-dir=rustlib")
 888            .arg("--success-message=Awesome-Source.")
 889            .arg("--image-dir").arg(&image)
 890            .arg("--work-dir").arg(&tmpdir(builder))
 891            .arg("--output-dir").arg(&distdir(builder))
 892            .arg(format!("--package-name={}", name))
 893            .arg("--component-name=rust-src")
 894            .arg("--legacy-manifest-dirs=rustlib,cargo");
 895         builder.run(&mut cmd);
 896
 897         builder.remove_dir(&image);
 898         distdir(builder).join(&format!("{}.tar.gz", name))
 899     }
 900 }
 901
 902 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 903
 904 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 905 pub struct PlainSourceTarball;
 906
 907 impl Step for PlainSourceTarball {
 908     /// Produces the location of the tarball generated
 909     type Output = PathBuf;
 910     const DEFAULT: bool = true;
 911     const ONLY_HOSTS: bool = true;
 912
 913     fn should_run(run: ShouldRun) -> ShouldRun {
 914         let builder = run.builder;
 915         run.path("src").default_condition(builder.config.rust_dist_src)
 916     }
 917
 918     fn make_run(run: RunConfig) {
 919         run.builder.ensure(PlainSourceTarball);
 920     }
 921
 922     /// Creates the plain source tarball
 923     fn run(self, builder: &Builder) -> PathBuf {
 924         builder.info("Create plain source tarball");
 925
 926         // Make sure that the root folder of tarball has the correct name
 927         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 928         let plain_dst_src = tmpdir(builder).join(&plain_name);
 929         let _ = fs::remove_dir_all(&plain_dst_src);
 930         t!(fs::create_dir_all(&plain_dst_src));
 931
 932         // This is the set of root paths which will become part of the source package
 933         let src_files = [
 934             "COPYRIGHT",
 935             "LICENSE-APACHE",
 936             "LICENSE-MIT",
 937             "CONTRIBUTING.md",
 938             "README.md",
 939             "RELEASES.md",
 940             "configure",
 941             "x.py",
 942             "config.toml.example",
 943             "Cargo.toml",
 944             "Cargo.lock",
 945         ];
 946         let src_dirs = [
 947             "src",
 948         ];
 949
 950         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 951
 952         // Copy the files normally
 953         for item in &src_files {
 954             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 955         }
 956
 957         // Create the version file
 958         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 959         if let Some(sha) = builder.rust_sha() {
 960             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 961         }
 962
 963         // If we're building from git sources, we need to vendor a complete distribution.
 964         if builder.rust_info.is_git() {
 965             // Get cargo-vendor installed, if it isn't already.
 966             let mut has_cargo_vendor = false;
 967             let mut cmd = Command::new(&builder.initial_cargo);
 968             for line in output(cmd.arg("install").arg("--list")).lines() {
 969                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 970             }
 971             if !has_cargo_vendor {
 972                 let mut cmd = builder.cargo(
 973                     builder.compiler(0, builder.config.build),
 974                     Mode::ToolBootstrap,
 975                     builder.config.build,
 976                     "install"
 977                 );
 978                 cmd.arg("--force")
 979                    .arg("--debug")
 980                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 981                    .arg("cargo-vendor");
 982                 builder.run(&mut cmd);
 983             }
 984
 985             // Vendor all Cargo dependencies
 986             let mut cmd = Command::new(&builder.initial_cargo);
 987             cmd.arg("vendor")
 988                .current_dir(&plain_dst_src);
 989             builder.run(&mut cmd);
 990         }
 991
 992         // Create plain source tarball
 993         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
 994         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
 995         tarball.set_extension(""); // strip .gz
 996         tarball.set_extension(""); // strip .tar
 997         if let Some(dir) = tarball.parent() {
 998             builder.create_dir(&dir);
 999         }
1000         builder.info("running installer");
1001         let mut cmd = rust_installer(builder);
1002         cmd.arg("tarball")
1003            .arg("--input").arg(&plain_name)
1004            .arg("--output").arg(&tarball)
1005            .arg("--work-dir=.")
1006            .current_dir(tmpdir(builder));
1007         builder.run(&mut cmd);
1008         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1009     }
1010 }
1011
1012 // We have to run a few shell scripts, which choke quite a bit on both `\`
1013 // characters and on `C:\` paths, so normalize both of them away.
1014 pub fn sanitize_sh(path: &Path) -> String {
1015     let path = path.to_str().unwrap().replace("\\", "/");
1016     return change_drive(&path).unwrap_or(path);
1017
1018     fn change_drive(s: &str) -> Option<String> {
1019         let mut ch = s.chars();
1020         let drive = ch.next().unwrap_or('C');
1021         if ch.next() != Some(':') {
1022             return None
1023         }
1024         if ch.next() != Some('/') {
1025             return None
1026         }
1027         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1028     }
1029 }
1030
1031 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1032 pub struct Cargo {
1033     pub stage: u32,
1034     pub target: Interned<String>,
1035 }
1036
1037 impl Step for Cargo {
1038     type Output = PathBuf;
1039     const ONLY_HOSTS: bool = true;
1040
1041     fn should_run(run: ShouldRun) -> ShouldRun {
1042         run.path("cargo")
1043     }
1044
1045     fn make_run(run: RunConfig) {
1046         run.builder.ensure(Cargo {
1047             stage: run.builder.top_stage,
1048             target: run.target,
1049         });
1050     }
1051
1052     fn run(self, builder: &Builder) -> PathBuf {
1053         let stage = self.stage;
1054         let target = self.target;
1055
1056         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1057         let src = builder.src.join("src/tools/cargo");
1058         let etc = src.join("src/etc");
1059         let release_num = builder.release_num("cargo");
1060         let name = pkgname(builder, "cargo");
1061         let version = builder.cargo_info.version(builder, &release_num);
1062
1063         let tmp = tmpdir(builder);
1064         let image = tmp.join("cargo-image");
1065         drop(fs::remove_dir_all(&image));
1066         builder.create_dir(&image);
1067
1068         // Prepare the image directory
1069         builder.create_dir(&image.join("share/zsh/site-functions"));
1070         builder.create_dir(&image.join("etc/bash_completion.d"));
1071         let cargo = builder.ensure(tool::Cargo {
1072             compiler: builder.compiler(stage, builder.config.build),
1073             target
1074         });
1075         builder.install(&cargo, &image.join("bin"), 0o755);
1076         for man in t!(etc.join("man").read_dir()) {
1077             let man = t!(man);
1078             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1079         }
1080         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1081         builder.copy(&etc.join("cargo.bashcomp.sh"),
1082              &image.join("etc/bash_completion.d/cargo"));
1083         let doc = image.join("share/doc/cargo");
1084         builder.install(&src.join("README.md"), &doc, 0o644);
1085         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1086         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1087         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1088
1089         // Prepare the overlay
1090         let overlay = tmp.join("cargo-overlay");
1091         drop(fs::remove_dir_all(&overlay));
1092         builder.create_dir(&overlay);
1093         builder.install(&src.join("README.md"), &overlay, 0o644);
1094         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1095         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1096         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1097         builder.create(&overlay.join("version"), &version);
1098
1099         // Generate the installer tarball
1100         let mut cmd = rust_installer(builder);
1101         cmd.arg("generate")
1102            .arg("--product-name=Rust")
1103            .arg("--rel-manifest-dir=rustlib")
1104            .arg("--success-message=Rust-is-ready-to-roll.")
1105            .arg("--image-dir").arg(&image)
1106            .arg("--work-dir").arg(&tmpdir(builder))
1107            .arg("--output-dir").arg(&distdir(builder))
1108            .arg("--non-installed-overlay").arg(&overlay)
1109            .arg(format!("--package-name={}-{}", name, target))
1110            .arg("--component-name=cargo")
1111            .arg("--legacy-manifest-dirs=rustlib,cargo");
1112         builder.run(&mut cmd);
1113         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1114     }
1115 }
1116
1117 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1118 pub struct Rls {
1119     pub stage: u32,
1120     pub target: Interned<String>,
1121 }
1122
1123 impl Step for Rls {
1124     type Output = Option<PathBuf>;
1125     const ONLY_HOSTS: bool = true;
1126
1127     fn should_run(run: ShouldRun) -> ShouldRun {
1128         run.path("rls")
1129     }
1130
1131     fn make_run(run: RunConfig) {
1132         run.builder.ensure(Rls {
1133             stage: run.builder.top_stage,
1134             target: run.target,
1135         });
1136     }
1137
1138     fn run(self, builder: &Builder) -> Option<PathBuf> {
1139         let stage = self.stage;
1140         let target = self.target;
1141         assert!(builder.config.extended);
1142
1143         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1144         let src = builder.src.join("src/tools/rls");
1145         let release_num = builder.release_num("rls");
1146         let name = pkgname(builder, "rls");
1147         let version = builder.rls_info.version(builder, &release_num);
1148
1149         let tmp = tmpdir(builder);
1150         let image = tmp.join("rls-image");
1151         drop(fs::remove_dir_all(&image));
1152         t!(fs::create_dir_all(&image));
1153
1154         // Prepare the image directory
1155         // We expect RLS to build, because we've exited this step above if tool
1156         // state for RLS isn't testing.
1157         let rls = builder.ensure(tool::Rls {
1158             compiler: builder.compiler(stage, builder.config.build),
1159             target, extra_features: Vec::new()
1160         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1161
1162         builder.install(&rls, &image.join("bin"), 0o755);
1163         let doc = image.join("share/doc/rls");
1164         builder.install(&src.join("README.md"), &doc, 0o644);
1165         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1166         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1167
1168         // Prepare the overlay
1169         let overlay = tmp.join("rls-overlay");
1170         drop(fs::remove_dir_all(&overlay));
1171         t!(fs::create_dir_all(&overlay));
1172         builder.install(&src.join("README.md"), &overlay, 0o644);
1173         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1174         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1175         builder.create(&overlay.join("version"), &version);
1176
1177         // Generate the installer tarball
1178         let mut cmd = rust_installer(builder);
1179         cmd.arg("generate")
1180            .arg("--product-name=Rust")
1181            .arg("--rel-manifest-dir=rustlib")
1182            .arg("--success-message=RLS-ready-to-serve.")
1183            .arg("--image-dir").arg(&image)
1184            .arg("--work-dir").arg(&tmpdir(builder))
1185            .arg("--output-dir").arg(&distdir(builder))
1186            .arg("--non-installed-overlay").arg(&overlay)
1187            .arg(format!("--package-name={}-{}", name, target))
1188            .arg("--legacy-manifest-dirs=rustlib,cargo")
1189            .arg("--component-name=rls-preview");
1190
1191         builder.run(&mut cmd);
1192         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Clippy {
1198     pub stage: u32,
1199     pub target: Interned<String>,
1200 }
1201
1202 impl Step for Clippy {
1203     type Output = Option<PathBuf>;
1204     const ONLY_HOSTS: bool = true;
1205
1206     fn should_run(run: ShouldRun) -> ShouldRun {
1207         run.path("clippy")
1208     }
1209
1210     fn make_run(run: RunConfig) {
1211         run.builder.ensure(Clippy {
1212             stage: run.builder.top_stage,
1213             target: run.target,
1214         });
1215     }
1216
1217     fn run(self, builder: &Builder) -> Option<PathBuf> {
1218         let stage = self.stage;
1219         let target = self.target;
1220         assert!(builder.config.extended);
1221
1222         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1223         let src = builder.src.join("src/tools/clippy");
1224         let release_num = builder.release_num("clippy");
1225         let name = pkgname(builder, "clippy");
1226         let version = builder.clippy_info.version(builder, &release_num);
1227
1228         let tmp = tmpdir(builder);
1229         let image = tmp.join("clippy-image");
1230         drop(fs::remove_dir_all(&image));
1231         builder.create_dir(&image);
1232
1233         // Prepare the image directory
1234         // We expect clippy to build, because we've exited this step above if tool
1235         // state for clippy isn't testing.
1236         let clippy = builder.ensure(tool::Clippy {
1237             compiler: builder.compiler(stage, builder.config.build),
1238             target, extra_features: Vec::new()
1239         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1240         let cargoclippy = builder.ensure(tool::CargoClippy {
1241             compiler: builder.compiler(stage, builder.config.build),
1242             target, extra_features: Vec::new()
1243         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1244
1245         builder.install(&clippy, &image.join("bin"), 0o755);
1246         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1247         let doc = image.join("share/doc/clippy");
1248         builder.install(&src.join("README.md"), &doc, 0o644);
1249         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1250         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1251
1252         // Prepare the overlay
1253         let overlay = tmp.join("clippy-overlay");
1254         drop(fs::remove_dir_all(&overlay));
1255         t!(fs::create_dir_all(&overlay));
1256         builder.install(&src.join("README.md"), &overlay, 0o644);
1257         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1259         builder.create(&overlay.join("version"), &version);
1260
1261         // Generate the installer tarball
1262         let mut cmd = rust_installer(builder);
1263         cmd.arg("generate")
1264            .arg("--product-name=Rust")
1265            .arg("--rel-manifest-dir=rustlib")
1266            .arg("--success-message=clippy-ready-to-serve.")
1267            .arg("--image-dir").arg(&image)
1268            .arg("--work-dir").arg(&tmpdir(builder))
1269            .arg("--output-dir").arg(&distdir(builder))
1270            .arg("--non-installed-overlay").arg(&overlay)
1271            .arg(format!("--package-name={}-{}", name, target))
1272            .arg("--legacy-manifest-dirs=rustlib,cargo")
1273            .arg("--component-name=clippy-preview");
1274
1275         builder.run(&mut cmd);
1276         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1277     }
1278 }
1279
1280 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1281 pub struct Miri {
1282     pub stage: u32,
1283     pub target: Interned<String>,
1284 }
1285
1286 impl Step for Miri {
1287     type Output = Option<PathBuf>;
1288     const ONLY_HOSTS: bool = true;
1289
1290     fn should_run(run: ShouldRun) -> ShouldRun {
1291         run.path("miri")
1292     }
1293
1294     fn make_run(run: RunConfig) {
1295         run.builder.ensure(Miri {
1296             stage: run.builder.top_stage,
1297             target: run.target,
1298         });
1299     }
1300
1301     fn run(self, builder: &Builder) -> Option<PathBuf> {
1302         let stage = self.stage;
1303         let target = self.target;
1304         assert!(builder.config.extended);
1305
1306         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1307         let src = builder.src.join("src/tools/miri");
1308         let release_num = builder.release_num("miri");
1309         let name = pkgname(builder, "miri");
1310         let version = builder.miri_info.version(builder, &release_num);
1311
1312         let tmp = tmpdir(builder);
1313         let image = tmp.join("miri-image");
1314         drop(fs::remove_dir_all(&image));
1315         builder.create_dir(&image);
1316
1317         // Prepare the image directory
1318         // We expect miri to build, because we've exited this step above if tool
1319         // state for miri isn't testing.
1320         let miri = builder.ensure(tool::Miri {
1321             compiler: builder.compiler(stage, builder.config.build),
1322             target, extra_features: Vec::new()
1323         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1324         let cargomiri = builder.ensure(tool::CargoMiri {
1325             compiler: builder.compiler(stage, builder.config.build),
1326             target, extra_features: Vec::new()
1327         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1328
1329         builder.install(&miri, &image.join("bin"), 0o755);
1330         builder.install(&cargomiri, &image.join("bin"), 0o755);
1331         let doc = image.join("share/doc/miri");
1332         builder.install(&src.join("README.md"), &doc, 0o644);
1333         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1334         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1335
1336         // Prepare the overlay
1337         let overlay = tmp.join("miri-overlay");
1338         drop(fs::remove_dir_all(&overlay));
1339         t!(fs::create_dir_all(&overlay));
1340         builder.install(&src.join("README.md"), &overlay, 0o644);
1341         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1342         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1343         builder.create(&overlay.join("version"), &version);
1344
1345         // Generate the installer tarball
1346         let mut cmd = rust_installer(builder);
1347         cmd.arg("generate")
1348            .arg("--product-name=Rust")
1349            .arg("--rel-manifest-dir=rustlib")
1350            .arg("--success-message=miri-ready-to-serve.")
1351            .arg("--image-dir").arg(&image)
1352            .arg("--work-dir").arg(&tmpdir(builder))
1353            .arg("--output-dir").arg(&distdir(builder))
1354            .arg("--non-installed-overlay").arg(&overlay)
1355            .arg(format!("--package-name={}-{}", name, target))
1356            .arg("--legacy-manifest-dirs=rustlib,cargo")
1357            .arg("--component-name=miri-preview");
1358
1359         builder.run(&mut cmd);
1360         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1361     }
1362 }
1363
1364 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1365 pub struct Rustfmt {
1366     pub stage: u32,
1367     pub target: Interned<String>,
1368 }
1369
1370 impl Step for Rustfmt {
1371     type Output = Option<PathBuf>;
1372     const ONLY_HOSTS: bool = true;
1373
1374     fn should_run(run: ShouldRun) -> ShouldRun {
1375         run.path("rustfmt")
1376     }
1377
1378     fn make_run(run: RunConfig) {
1379         run.builder.ensure(Rustfmt {
1380             stage: run.builder.top_stage,
1381             target: run.target,
1382         });
1383     }
1384
1385     fn run(self, builder: &Builder) -> Option<PathBuf> {
1386         let stage = self.stage;
1387         let target = self.target;
1388
1389         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1390         let src = builder.src.join("src/tools/rustfmt");
1391         let release_num = builder.release_num("rustfmt");
1392         let name = pkgname(builder, "rustfmt");
1393         let version = builder.rustfmt_info.version(builder, &release_num);
1394
1395         let tmp = tmpdir(builder);
1396         let image = tmp.join("rustfmt-image");
1397         drop(fs::remove_dir_all(&image));
1398         builder.create_dir(&image);
1399
1400         // Prepare the image directory
1401         let rustfmt = builder.ensure(tool::Rustfmt {
1402             compiler: builder.compiler(stage, builder.config.build),
1403             target, extra_features: Vec::new()
1404         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1405         let cargofmt = builder.ensure(tool::Cargofmt {
1406             compiler: builder.compiler(stage, builder.config.build),
1407             target, extra_features: Vec::new()
1408         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1409
1410         builder.install(&rustfmt, &image.join("bin"), 0o755);
1411         builder.install(&cargofmt, &image.join("bin"), 0o755);
1412         let doc = image.join("share/doc/rustfmt");
1413         builder.install(&src.join("README.md"), &doc, 0o644);
1414         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1415         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1416
1417         // Prepare the overlay
1418         let overlay = tmp.join("rustfmt-overlay");
1419         drop(fs::remove_dir_all(&overlay));
1420         builder.create_dir(&overlay);
1421         builder.install(&src.join("README.md"), &overlay, 0o644);
1422         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1423         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1424         builder.create(&overlay.join("version"), &version);
1425
1426         // Generate the installer tarball
1427         let mut cmd = rust_installer(builder);
1428         cmd.arg("generate")
1429            .arg("--product-name=Rust")
1430            .arg("--rel-manifest-dir=rustlib")
1431            .arg("--success-message=rustfmt-ready-to-fmt.")
1432            .arg("--image-dir").arg(&image)
1433            .arg("--work-dir").arg(&tmpdir(builder))
1434            .arg("--output-dir").arg(&distdir(builder))
1435            .arg("--non-installed-overlay").arg(&overlay)
1436            .arg(format!("--package-name={}-{}", name, target))
1437            .arg("--legacy-manifest-dirs=rustlib,cargo")
1438            .arg("--component-name=rustfmt-preview");
1439
1440         builder.run(&mut cmd);
1441         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1442     }
1443 }
1444
1445 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1446 pub struct Extended {
1447     stage: u32,
1448     host: Interned<String>,
1449     target: Interned<String>,
1450 }
1451
1452 impl Step for Extended {
1453     type Output = ();
1454     const DEFAULT: bool = true;
1455     const ONLY_HOSTS: bool = true;
1456
1457     fn should_run(run: ShouldRun) -> ShouldRun {
1458         let builder = run.builder;
1459         run.path("extended").default_condition(builder.config.extended)
1460     }
1461
1462     fn make_run(run: RunConfig) {
1463         run.builder.ensure(Extended {
1464             stage: run.builder.top_stage,
1465             host: run.builder.config.build,
1466             target: run.target,
1467         });
1468     }
1469
1470     /// Creates a combined installer for the specified target in the provided stage.
1471     fn run(self, builder: &Builder) {
1472         let stage = self.stage;
1473         let target = self.target;
1474
1475         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1476
1477         let rustc_installer = builder.ensure(Rustc {
1478             compiler: builder.compiler(stage, target),
1479         });
1480         let cargo_installer = builder.ensure(Cargo { stage, target });
1481         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1482         let rls_installer = builder.ensure(Rls { stage, target });
1483         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1484         let clippy_installer = builder.ensure(Clippy { stage, target });
1485         let miri_installer = builder.ensure(Miri { stage, target });
1486         let lldb_installer = builder.ensure(Lldb { target });
1487         let mingw_installer = builder.ensure(Mingw { host: target });
1488         let analysis_installer = builder.ensure(Analysis {
1489             compiler: builder.compiler(stage, self.host),
1490             target
1491         });
1492
1493         let docs_installer = builder.ensure(Docs { stage, host: target, });
1494         let std_installer = builder.ensure(Std {
1495             compiler: builder.compiler(stage, self.host),
1496             target,
1497         });
1498
1499         let tmp = tmpdir(builder);
1500         let overlay = tmp.join("extended-overlay");
1501         let etc = builder.src.join("src/etc/installer");
1502         let work = tmp.join("work");
1503
1504         let _ = fs::remove_dir_all(&overlay);
1505         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1506         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1507         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1508         let version = builder.rust_version();
1509         builder.create(&overlay.join("version"), &version);
1510         if let Some(sha) = builder.rust_sha() {
1511             builder.create(&overlay.join("git-commit-hash"), &sha);
1512         }
1513         builder.install(&etc.join("README.md"), &overlay, 0o644);
1514
1515         // When rust-std package split from rustc, we needed to ensure that during
1516         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1517         // the std files during uninstall. To do this ensure that rustc comes
1518         // before rust-std in the list below.
1519         let mut tarballs = Vec::new();
1520         tarballs.push(rustc_installer);
1521         tarballs.push(cargo_installer);
1522         tarballs.extend(rls_installer.clone());
1523         tarballs.extend(clippy_installer.clone());
1524         tarballs.extend(miri_installer.clone());
1525         tarballs.extend(rustfmt_installer.clone());
1526         tarballs.extend(llvm_tools_installer);
1527         tarballs.extend(lldb_installer);
1528         tarballs.push(analysis_installer);
1529         tarballs.push(std_installer);
1530         if builder.config.docs {
1531             tarballs.push(docs_installer);
1532         }
1533         if target.contains("pc-windows-gnu") {
1534             tarballs.push(mingw_installer.unwrap());
1535         }
1536         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1537         for tarball in &tarballs[1..] {
1538             input_tarballs.push(",");
1539             input_tarballs.push(tarball);
1540         }
1541
1542         let mut cmd = rust_installer(builder);
1543         cmd.arg("combine")
1544             .arg("--product-name=Rust")
1545             .arg("--rel-manifest-dir=rustlib")
1546             .arg("--success-message=Rust-is-ready-to-roll.")
1547             .arg("--work-dir").arg(&work)
1548             .arg("--output-dir").arg(&distdir(builder))
1549             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1550             .arg("--legacy-manifest-dirs=rustlib,cargo")
1551             .arg("--input-tarballs").arg(input_tarballs)
1552             .arg("--non-installed-overlay").arg(&overlay);
1553         builder.run(&mut cmd);
1554
1555         let mut license = String::new();
1556         license += &builder.read(&builder.src.join("COPYRIGHT"));
1557         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1558         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1559         license.push_str("\n");
1560         license.push_str("\n");
1561
1562         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1563         let mut rtf = rtf.to_string();
1564         rtf.push_str("\n");
1565         for line in license.lines() {
1566             rtf.push_str(line);
1567             rtf.push_str("\\line ");
1568         }
1569         rtf.push_str("}");
1570
1571         fn filter(contents: &str, marker: &str) -> String {
1572             let start = format!("tool-{}-start", marker);
1573             let end = format!("tool-{}-end", marker);
1574             let mut lines = Vec::new();
1575             let mut omitted = false;
1576             for line in contents.lines() {
1577                 if line.contains(&start) {
1578                     omitted = true;
1579                 } else if line.contains(&end) {
1580                     omitted = false;
1581                 } else if !omitted {
1582                     lines.push(line);
1583                 }
1584             }
1585
1586             lines.join("\n")
1587         }
1588
1589         let xform = |p: &Path| {
1590             let mut contents = t!(fs::read_to_string(p));
1591             if rls_installer.is_none() {
1592                 contents = filter(&contents, "rls");
1593             }
1594             if clippy_installer.is_none() {
1595                 contents = filter(&contents, "clippy");
1596             }
1597             if miri_installer.is_none() {
1598                 contents = filter(&contents, "miri");
1599             }
1600             if rustfmt_installer.is_none() {
1601                 contents = filter(&contents, "rustfmt");
1602             }
1603             let ret = tmp.join(p.file_name().unwrap());
1604             t!(fs::write(&ret, &contents));
1605             ret
1606         };
1607
1608         if target.contains("apple-darwin") {
1609             let pkg = tmp.join("pkg");
1610             let _ = fs::remove_dir_all(&pkg);
1611
1612             let pkgbuild = |component: &str| {
1613                 let mut cmd = Command::new("pkgbuild");
1614                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1615                     .arg("--scripts").arg(pkg.join(component))
1616                     .arg("--nopayload")
1617                     .arg(pkg.join(component).with_extension("pkg"));
1618                 builder.run(&mut cmd);
1619             };
1620
1621             let prepare = |name: &str| {
1622                 builder.create_dir(&pkg.join(name));
1623                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1624                         &pkg.join(name));
1625                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1626                 pkgbuild(name);
1627             };
1628             prepare("rustc");
1629             prepare("cargo");
1630             prepare("rust-docs");
1631             prepare("rust-std");
1632             prepare("rust-analysis");
1633
1634             if rls_installer.is_some() {
1635                 prepare("rls");
1636             }
1637             if clippy_installer.is_some() {
1638                 prepare("clippy");
1639             }
1640             if miri_installer.is_some() {
1641                 prepare("miri");
1642             }
1643
1644             // create an 'uninstall' package
1645             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1646             pkgbuild("uninstall");
1647
1648             builder.create_dir(&pkg.join("res"));
1649             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1650             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1651             let mut cmd = Command::new("productbuild");
1652             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1653                 .arg("--resources").arg(pkg.join("res"))
1654                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1655                                                     pkgname(builder, "rust"),
1656                                                     target)))
1657                 .arg("--package-path").arg(&pkg);
1658             builder.run(&mut cmd);
1659         }
1660
1661         if target.contains("windows") {
1662             let exe = tmp.join("exe");
1663             let _ = fs::remove_dir_all(&exe);
1664
1665             let prepare = |name: &str| {
1666                 builder.create_dir(&exe.join(name));
1667                 let dir = if name == "rust-std" || name == "rust-analysis" {
1668                     format!("{}-{}", name, target)
1669                 } else if name == "rls" {
1670                     "rls-preview".to_string()
1671                 } else if name == "clippy" {
1672                     "clippy-preview".to_string()
1673                 } else if name == "miri" {
1674                     "miri-preview".to_string()
1675                 } else {
1676                     name.to_string()
1677                 };
1678                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1679                             .join(dir),
1680                         &exe.join(name));
1681                 builder.remove(&exe.join(name).join("manifest.in"));
1682             };
1683             prepare("rustc");
1684             prepare("cargo");
1685             prepare("rust-analysis");
1686             prepare("rust-docs");
1687             prepare("rust-std");
1688             if rls_installer.is_some() {
1689                 prepare("rls");
1690             }
1691             if clippy_installer.is_some() {
1692                 prepare("clippy");
1693             }
1694             if miri_installer.is_some() {
1695                 prepare("miri");
1696             }
1697             if target.contains("windows-gnu") {
1698                 prepare("rust-mingw");
1699             }
1700
1701             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1702             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1703             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1704             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1705             builder.create(&exe.join("LICENSE.txt"), &license);
1706
1707             // Generate exe installer
1708             let mut cmd = Command::new("iscc");
1709             cmd.arg("rust.iss")
1710                 .current_dir(&exe);
1711             if target.contains("windows-gnu") {
1712                 cmd.arg("/dMINGW");
1713             }
1714             add_env(builder, &mut cmd, target);
1715             builder.run(&mut cmd);
1716             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1717                     &distdir(builder),
1718                     0o755);
1719
1720             // Generate msi installer
1721             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1722             let heat = wix.join("bin/heat.exe");
1723             let candle = wix.join("bin/candle.exe");
1724             let light = wix.join("bin/light.exe");
1725
1726             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1727             builder.run(Command::new(&heat)
1728                             .current_dir(&exe)
1729                             .arg("dir")
1730                             .arg("rustc")
1731                             .args(&heat_flags)
1732                             .arg("-cg").arg("RustcGroup")
1733                             .arg("-dr").arg("Rustc")
1734                             .arg("-var").arg("var.RustcDir")
1735                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1736             builder.run(Command::new(&heat)
1737                             .current_dir(&exe)
1738                             .arg("dir")
1739                             .arg("rust-docs")
1740                             .args(&heat_flags)
1741                             .arg("-cg").arg("DocsGroup")
1742                             .arg("-dr").arg("Docs")
1743                             .arg("-var").arg("var.DocsDir")
1744                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1745                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1746             builder.run(Command::new(&heat)
1747                             .current_dir(&exe)
1748                             .arg("dir")
1749                             .arg("cargo")
1750                             .args(&heat_flags)
1751                             .arg("-cg").arg("CargoGroup")
1752                             .arg("-dr").arg("Cargo")
1753                             .arg("-var").arg("var.CargoDir")
1754                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1755                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1756             builder.run(Command::new(&heat)
1757                             .current_dir(&exe)
1758                             .arg("dir")
1759                             .arg("rust-std")
1760                             .args(&heat_flags)
1761                             .arg("-cg").arg("StdGroup")
1762                             .arg("-dr").arg("Std")
1763                             .arg("-var").arg("var.StdDir")
1764                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1765             if rls_installer.is_some() {
1766                 builder.run(Command::new(&heat)
1767                                 .current_dir(&exe)
1768                                 .arg("dir")
1769                                 .arg("rls")
1770                                 .args(&heat_flags)
1771                                 .arg("-cg").arg("RlsGroup")
1772                                 .arg("-dr").arg("Rls")
1773                                 .arg("-var").arg("var.RlsDir")
1774                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1775                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1776             }
1777             if clippy_installer.is_some() {
1778                 builder.run(Command::new(&heat)
1779                                 .current_dir(&exe)
1780                                 .arg("dir")
1781                                 .arg("clippy")
1782                                 .args(&heat_flags)
1783                                 .arg("-cg").arg("ClippyGroup")
1784                                 .arg("-dr").arg("Clippy")
1785                                 .arg("-var").arg("var.ClippyDir")
1786                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1787                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1788             }
1789             if miri_installer.is_some() {
1790                 builder.run(Command::new(&heat)
1791                                 .current_dir(&exe)
1792                                 .arg("dir")
1793                                 .arg("miri")
1794                                 .args(&heat_flags)
1795                                 .arg("-cg").arg("MiriGroup")
1796                                 .arg("-dr").arg("Miri")
1797                                 .arg("-var").arg("var.MiriDir")
1798                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1799                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1800             }
1801             builder.run(Command::new(&heat)
1802                             .current_dir(&exe)
1803                             .arg("dir")
1804                             .arg("rust-analysis")
1805                             .args(&heat_flags)
1806                             .arg("-cg").arg("AnalysisGroup")
1807                             .arg("-dr").arg("Analysis")
1808                             .arg("-var").arg("var.AnalysisDir")
1809                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1810                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1811             if target.contains("windows-gnu") {
1812                 builder.run(Command::new(&heat)
1813                                 .current_dir(&exe)
1814                                 .arg("dir")
1815                                 .arg("rust-mingw")
1816                                 .args(&heat_flags)
1817                                 .arg("-cg").arg("GccGroup")
1818                                 .arg("-dr").arg("Gcc")
1819                                 .arg("-var").arg("var.GccDir")
1820                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1821             }
1822
1823             let candle = |input: &Path| {
1824                 let output = exe.join(input.file_stem().unwrap())
1825                                 .with_extension("wixobj");
1826                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1827                 let mut cmd = Command::new(&candle);
1828                 cmd.current_dir(&exe)
1829                     .arg("-nologo")
1830                     .arg("-dRustcDir=rustc")
1831                     .arg("-dDocsDir=rust-docs")
1832                     .arg("-dCargoDir=cargo")
1833                     .arg("-dStdDir=rust-std")
1834                     .arg("-dAnalysisDir=rust-analysis")
1835                     .arg("-arch").arg(&arch)
1836                     .arg("-out").arg(&output)
1837                     .arg(&input);
1838                 add_env(builder, &mut cmd, target);
1839
1840                 if rls_installer.is_some() {
1841                     cmd.arg("-dRlsDir=rls");
1842                 }
1843                 if clippy_installer.is_some() {
1844                     cmd.arg("-dClippyDir=clippy");
1845                 }
1846                 if miri_installer.is_some() {
1847                     cmd.arg("-dMiriDir=miri");
1848                 }
1849                 if target.contains("windows-gnu") {
1850                     cmd.arg("-dGccDir=rust-mingw");
1851                 }
1852                 builder.run(&mut cmd);
1853             };
1854             candle(&xform(&etc.join("msi/rust.wxs")));
1855             candle(&etc.join("msi/ui.wxs"));
1856             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1857             candle("RustcGroup.wxs".as_ref());
1858             candle("DocsGroup.wxs".as_ref());
1859             candle("CargoGroup.wxs".as_ref());
1860             candle("StdGroup.wxs".as_ref());
1861             if rls_installer.is_some() {
1862                 candle("RlsGroup.wxs".as_ref());
1863             }
1864             if clippy_installer.is_some() {
1865                 candle("ClippyGroup.wxs".as_ref());
1866             }
1867             if miri_installer.is_some() {
1868                 candle("MiriGroup.wxs".as_ref());
1869             }
1870             candle("AnalysisGroup.wxs".as_ref());
1871
1872             if target.contains("windows-gnu") {
1873                 candle("GccGroup.wxs".as_ref());
1874             }
1875
1876             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1877             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1878             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1879
1880             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1881             let mut cmd = Command::new(&light);
1882             cmd.arg("-nologo")
1883                 .arg("-ext").arg("WixUIExtension")
1884                 .arg("-ext").arg("WixUtilExtension")
1885                 .arg("-out").arg(exe.join(&filename))
1886                 .arg("rust.wixobj")
1887                 .arg("ui.wixobj")
1888                 .arg("rustwelcomedlg.wixobj")
1889                 .arg("RustcGroup.wixobj")
1890                 .arg("DocsGroup.wixobj")
1891                 .arg("CargoGroup.wixobj")
1892                 .arg("StdGroup.wixobj")
1893                 .arg("AnalysisGroup.wixobj")
1894                 .current_dir(&exe);
1895
1896             if rls_installer.is_some() {
1897                 cmd.arg("RlsGroup.wixobj");
1898             }
1899             if clippy_installer.is_some() {
1900                 cmd.arg("ClippyGroup.wixobj");
1901             }
1902             if miri_installer.is_some() {
1903                 cmd.arg("MiriGroup.wixobj");
1904             }
1905
1906             if target.contains("windows-gnu") {
1907                 cmd.arg("GccGroup.wixobj");
1908             }
1909             // ICE57 wrongly complains about the shortcuts
1910             cmd.arg("-sice:ICE57");
1911
1912             builder.run(&mut cmd);
1913
1914             if !builder.config.dry_run {
1915                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1916             }
1917         }
1918     }
1919 }
1920
1921 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1922     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1923     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1924        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1925        .env("CFG_RELEASE", builder.rust_release())
1926        .env("CFG_VER_MAJOR", parts.next().unwrap())
1927        .env("CFG_VER_MINOR", parts.next().unwrap())
1928        .env("CFG_VER_PATCH", parts.next().unwrap())
1929        .env("CFG_VER_BUILD", "0") // just needed to build
1930        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1931        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1932        .env("CFG_BUILD", target)
1933        .env("CFG_CHANNEL", &builder.config.channel);
1934
1935     if target.contains("windows-gnu") {
1936        cmd.env("CFG_MINGW", "1")
1937           .env("CFG_ABI", "GNU");
1938     } else {
1939        cmd.env("CFG_MINGW", "0")
1940           .env("CFG_ABI", "MSVC");
1941     }
1942
1943     if target.contains("x86_64") {
1944        cmd.env("CFG_PLATFORM", "x64");
1945     } else {
1946        cmd.env("CFG_PLATFORM", "x86");
1947     }
1948 }
1949
1950 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1951 pub struct HashSign;
1952
1953 impl Step for HashSign {
1954     type Output = ();
1955     const ONLY_HOSTS: bool = true;
1956
1957     fn should_run(run: ShouldRun) -> ShouldRun {
1958         run.path("hash-and-sign")
1959     }
1960
1961     fn make_run(run: RunConfig) {
1962         run.builder.ensure(HashSign);
1963     }
1964
1965     fn run(self, builder: &Builder) {
1966         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1967         if builder.config.dry_run {
1968             return;
1969         }
1970         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1971             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1972         });
1973         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1974             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1975         });
1976         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1977             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1978         });
1979         let pass = t!(fs::read_to_string(&file));
1980
1981         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1982
1983         cmd.arg(sign);
1984         cmd.arg(distdir(builder));
1985         cmd.arg(today.trim());
1986         cmd.arg(builder.rust_package_vers());
1987         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1988         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1989         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1990         cmd.arg(builder.package_vers(&builder.release_num("miri")));
1991         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1992         cmd.arg(builder.llvm_tools_package_vers());
1993         cmd.arg(builder.lldb_package_vers());
1994         cmd.arg(addr);
1995
1996         builder.create_dir(&distdir(builder));
1997
1998         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1999         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2000         let status = t!(child.wait());
2001         assert!(status.success());
2002     }
2003 }
2004
2005 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2006 // LLVM tools are linked dynamically.
2007 // Note: This function does no yet support Windows but we also don't support
2008 //       linking LLVM tools dynamically on Windows yet.
2009 pub fn maybe_install_llvm_dylib(builder: &Builder,
2010                                 target: Interned<String>,
2011                                 sysroot: &Path) {
2012     let src_libdir = builder
2013         .llvm_out(target)
2014         .join("lib");
2015     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2016     t!(fs::create_dir_all(&dst_libdir));
2017
2018     if target.contains("apple-darwin") {
2019         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2020         if llvm_dylib_path.exists() {
2021             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2022         }
2023         return
2024     }
2025
2026     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2027     // Since tools link to the latter rather than the former, we have to
2028     // follow the symlink to find out what to distribute.
2029     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2030     if llvm_dylib_path.exists() {
2031         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2032             panic!("dist: Error calling canonicalize path `{}`: {}",
2033                    llvm_dylib_path.display(), e);
2034         });
2035
2036
2037         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2038     }
2039 }
2040
2041 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2042 pub struct LlvmTools {
2043     pub stage: u32,
2044     pub target: Interned<String>,
2045 }
2046
2047 impl Step for LlvmTools {
2048     type Output = Option<PathBuf>;
2049     const ONLY_HOSTS: bool = true;
2050
2051     fn should_run(run: ShouldRun) -> ShouldRun {
2052         run.path("llvm-tools")
2053     }
2054
2055     fn make_run(run: RunConfig) {
2056         run.builder.ensure(LlvmTools {
2057             stage: run.builder.top_stage,
2058             target: run.target,
2059         });
2060     }
2061
2062     fn run(self, builder: &Builder) -> Option<PathBuf> {
2063         let stage = self.stage;
2064         let target = self.target;
2065         assert!(builder.config.extended);
2066
2067         /* run only if llvm-config isn't used */
2068         if let Some(config) = builder.config.target_config.get(&target) {
2069             if let Some(ref _s) = config.llvm_config {
2070                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2071                     stage, target));
2072                 return None;
2073             }
2074         }
2075
2076         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2077         let src = builder.src.join("src/llvm");
2078         let name = pkgname(builder, "llvm-tools");
2079
2080         let tmp = tmpdir(builder);
2081         let image = tmp.join("llvm-tools-image");
2082         drop(fs::remove_dir_all(&image));
2083
2084         // Prepare the image directory
2085         let src_bindir = builder
2086             .llvm_out(target)
2087             .join("bin");
2088         let dst_bindir = image.join("lib/rustlib")
2089             .join(&*target)
2090             .join("bin");
2091         t!(fs::create_dir_all(&dst_bindir));
2092         for tool in LLVM_TOOLS {
2093             let exe = src_bindir.join(exe(tool, &target));
2094             builder.install(&exe, &dst_bindir, 0o755);
2095         }
2096
2097         // Prepare the overlay
2098         let overlay = tmp.join("llvm-tools-overlay");
2099         drop(fs::remove_dir_all(&overlay));
2100         builder.create_dir(&overlay);
2101         builder.install(&src.join("README.txt"), &overlay, 0o644);
2102         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2103         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2104
2105         // Generate the installer tarball
2106         let mut cmd = rust_installer(builder);
2107         cmd.arg("generate")
2108             .arg("--product-name=Rust")
2109             .arg("--rel-manifest-dir=rustlib")
2110             .arg("--success-message=llvm-tools-installed.")
2111             .arg("--image-dir").arg(&image)
2112             .arg("--work-dir").arg(&tmpdir(builder))
2113             .arg("--output-dir").arg(&distdir(builder))
2114             .arg("--non-installed-overlay").arg(&overlay)
2115             .arg(format!("--package-name={}-{}", name, target))
2116             .arg("--legacy-manifest-dirs=rustlib,cargo")
2117             .arg("--component-name=llvm-tools-preview");
2118
2119
2120         builder.run(&mut cmd);
2121         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2122     }
2123 }
2124
2125 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2126 pub struct Lldb {
2127     pub target: Interned<String>,
2128 }
2129
2130 impl Step for Lldb {
2131     type Output = Option<PathBuf>;
2132     const ONLY_HOSTS: bool = true;
2133     const DEFAULT: bool = true;
2134
2135     fn should_run(run: ShouldRun) -> ShouldRun {
2136         run.path("src/tools/lldb")
2137     }
2138
2139     fn make_run(run: RunConfig) {
2140         run.builder.ensure(Lldb {
2141             target: run.target,
2142         });
2143     }
2144
2145     fn run(self, builder: &Builder) -> Option<PathBuf> {
2146         let target = self.target;
2147
2148         if builder.config.dry_run {
2149             return None;
2150         }
2151
2152         let bindir = builder
2153             .llvm_out(target)
2154             .join("bin");
2155         let lldb_exe = bindir.join(exe("lldb", &target));
2156         if !lldb_exe.exists() {
2157             return None;
2158         }
2159
2160         builder.info(&format!("Dist Lldb ({})", target));
2161         let src = builder.src.join("src/tools/lldb");
2162         let name = pkgname(builder, "lldb");
2163
2164         let tmp = tmpdir(builder);
2165         let image = tmp.join("lldb-image");
2166         drop(fs::remove_dir_all(&image));
2167
2168         // Prepare the image directory
2169         let root = image.join("lib/rustlib").join(&*target);
2170         let dst = root.join("bin");
2171         t!(fs::create_dir_all(&dst));
2172         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2173             let exe = bindir.join(exe(program, &target));
2174             builder.install(&exe, &dst, 0o755);
2175         }
2176
2177         // The libraries.
2178         let libdir = builder.llvm_out(target).join("lib");
2179         let dst = root.join("lib");
2180         t!(fs::create_dir_all(&dst));
2181         for entry in t!(fs::read_dir(&libdir)) {
2182             let entry = entry.unwrap();
2183             if let Ok(name) = entry.file_name().into_string() {
2184                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2185                     if t!(entry.file_type()).is_symlink() {
2186                         builder.copy_to_folder(&entry.path(), &dst);
2187                     } else {
2188                        builder.install(&entry.path(), &dst, 0o755);
2189                     }
2190                 }
2191             }
2192         }
2193
2194         // The lldb scripts might be installed in lib/python$version
2195         // or in lib64/python$version.  If lib64 exists, use it;
2196         // otherwise lib.
2197         let libdir = builder.llvm_out(target).join("lib64");
2198         let (libdir, libdir_name) = if libdir.exists() {
2199             (libdir, "lib64")
2200         } else {
2201             (builder.llvm_out(target).join("lib"), "lib")
2202         };
2203         for entry in t!(fs::read_dir(&libdir)) {
2204             let entry = t!(entry);
2205             if let Ok(name) = entry.file_name().into_string() {
2206                 if name.starts_with("python") {
2207                     let dst = root.join(libdir_name)
2208                         .join(entry.file_name());
2209                     t!(fs::create_dir_all(&dst));
2210                     builder.cp_r(&entry.path(), &dst);
2211                     break;
2212                 }
2213             }
2214         }
2215
2216         // Prepare the overlay
2217         let overlay = tmp.join("lldb-overlay");
2218         drop(fs::remove_dir_all(&overlay));
2219         builder.create_dir(&overlay);
2220         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2221         builder.create(&overlay.join("version"), &builder.lldb_vers());
2222
2223         // Generate the installer tarball
2224         let mut cmd = rust_installer(builder);
2225         cmd.arg("generate")
2226             .arg("--product-name=Rust")
2227             .arg("--rel-manifest-dir=rustlib")
2228             .arg("--success-message=lldb-installed.")
2229             .arg("--image-dir").arg(&image)
2230             .arg("--work-dir").arg(&tmpdir(builder))
2231             .arg("--output-dir").arg(&distdir(builder))
2232             .arg("--non-installed-overlay").arg(&overlay)
2233             .arg(format!("--package-name={}-{}", name, target))
2234             .arg("--legacy-manifest-dirs=rustlib,cargo")
2235             .arg("--component-name=lldb-preview");
2236
2237
2238         builder.run(&mut cmd);
2239         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2240     }
2241 }