src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::output;
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{libdir, is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time;
  27
  28 pub fn pkgname(builder: &Builder, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "rustfmt" {
  36         format!("{}-{}", component, builder.rustfmt_package_vers())
  37     } else if component == "llvm-tools" {
  38         format!("{}-{}", component, builder.llvm_tools_package_vers())
  39     } else if component == "lldb" {
  40         format!("{}-{}", component, builder.lldb_package_vers())
  41     } else {
  42         assert!(component.starts_with("rust"));
  43         format!("{}-{}", component, builder.rust_package_vers())
  44     }
  45 }
  46
  47 fn distdir(builder: &Builder) -> PathBuf {
  48     builder.out.join("dist")
  49 }
  50
  51 pub fn tmpdir(builder: &Builder) -> PathBuf {
  52     builder.out.join("tmp/dist")
  53 }
  54
  55 fn rust_installer(builder: &Builder) -> Command {
  56     builder.tool_cmd(Tool::RustInstaller)
  57 }
  58
  59 fn missing_tool(tool_name: &str, skip: bool) {
  60     if skip {
  61         println!("Unable to build {}, skipping dist", tool_name)
  62     } else {
  63         panic!("Unable to build {}", tool_name)
  64     }
  65 }
  66
  67 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  68 pub struct Docs {
  69     pub stage: u32,
  70     pub host: Interned<String>,
  71 }
  72
  73 impl Step for Docs {
  74     type Output = PathBuf;
  75     const DEFAULT: bool = true;
  76
  77     fn should_run(run: ShouldRun) -> ShouldRun {
  78         run.path("src/doc")
  79     }
  80
  81     fn make_run(run: RunConfig) {
  82         run.builder.ensure(Docs {
  83             stage: run.builder.top_stage,
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub stage: u32,
 132     pub host: Interned<String>,
 133 }
 134
 135 impl Step for RustcDocs {
 136     type Output = PathBuf;
 137     const DEFAULT: bool = true;
 138
 139     fn should_run(run: ShouldRun) -> ShouldRun {
 140         run.path("src/librustc")
 141     }
 142
 143     fn make_run(run: RunConfig) {
 144         run.builder.ensure(RustcDocs {
 145             stage: run.builder.top_stage,
 146             host: run.target,
 147         });
 148     }
 149
 150     /// Builds the `rustc-docs` installer component.
 151     fn run(self, builder: &Builder) -> PathBuf {
 152         let host = self.host;
 153
 154         let name = pkgname(builder, "rustc-docs");
 155
 156         builder.info(&format!("Dist compiler docs ({})", host));
 157         if !builder.config.compiler_docs {
 158             builder.info("\tskipping - compiler docs disabled");
 159             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 160         }
 161
 162         builder.default_doc(None);
 163
 164         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 165         let _ = fs::remove_dir_all(&image);
 166
 167         let dst = image.join("share/doc/rust/html");
 168         t!(fs::create_dir_all(&dst));
 169         let src = builder.compiler_doc_out(host);
 170         builder.cp_r(&src, &dst);
 171
 172         let mut cmd = rust_installer(builder);
 173         cmd.arg("generate")
 174            .arg("--product-name=Rustc-Documentation")
 175            .arg("--rel-manifest-dir=rustlib")
 176            .arg("--success-message=Rustc-documentation-is-installed.")
 177            .arg("--image-dir").arg(&image)
 178            .arg("--work-dir").arg(&tmpdir(builder))
 179            .arg("--output-dir").arg(&distdir(builder))
 180            .arg(format!("--package-name={}-{}", name, host))
 181            .arg("--component-name=rustc-docs")
 182            .arg("--legacy-manifest-dirs=rustlib,cargo")
 183            .arg("--bulk-dirs=share/doc/rust/html");
 184         builder.run(&mut cmd);
 185         builder.remove_dir(&image);
 186
 187         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 188     }
 189 }
 190
 191 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 192     let mut found = Vec::with_capacity(files.len());
 193
 194     for file in files {
 195         let file_path =
 196             path.iter()
 197                 .map(|dir| dir.join(file))
 198                 .find(|p| p.exists());
 199
 200         if let Some(file_path) = file_path {
 201             found.push(file_path);
 202         } else {
 203             panic!("Could not find '{}' in {:?}", file, path);
 204         }
 205     }
 206
 207     found
 208 }
 209
 210 fn make_win_dist(
 211     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 212 ) {
 213     //Ask gcc where it keeps its stuff
 214     let mut cmd = Command::new(builder.cc(target_triple));
 215     cmd.arg("-print-search-dirs");
 216     let gcc_out = output(&mut cmd);
 217
 218     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 219     let mut lib_path = Vec::new();
 220
 221     for line in gcc_out.lines() {
 222         let idx = line.find(':').unwrap();
 223         let key = &line[..idx];
 224         let trim_chars: &[_] = &[' ', '='];
 225         let value =
 226             line[(idx + 1)..]
 227                 .trim_left_matches(trim_chars)
 228                 .split(';')
 229                 .map(PathBuf::from);
 230
 231         if key == "programs" {
 232             bin_path.extend(value);
 233         } else if key == "libraries" {
 234             lib_path.extend(value);
 235         }
 236     }
 237
 238     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 239     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 240     if target_triple.starts_with("i686-") {
 241         rustc_dlls.push("libgcc_s_dw2-1.dll");
 242     } else {
 243         rustc_dlls.push("libgcc_s_seh-1.dll");
 244     }
 245
 246     let target_libs = [ //MinGW libs
 247         "libgcc.a",
 248         "libgcc_eh.a",
 249         "libgcc_s.a",
 250         "libm.a",
 251         "libmingw32.a",
 252         "libmingwex.a",
 253         "libstdc++.a",
 254         "libiconv.a",
 255         "libmoldname.a",
 256         "libpthread.a",
 257         //Windows import libs
 258         "libadvapi32.a",
 259         "libbcrypt.a",
 260         "libcomctl32.a",
 261         "libcomdlg32.a",
 262         "libcredui.a",
 263         "libcrypt32.a",
 264         "libdbghelp.a",
 265         "libgdi32.a",
 266         "libimagehlp.a",
 267         "libiphlpapi.a",
 268         "libkernel32.a",
 269         "libmsimg32.a",
 270         "libmsvcrt.a",
 271         "libodbc32.a",
 272         "libole32.a",
 273         "liboleaut32.a",
 274         "libopengl32.a",
 275         "libpsapi.a",
 276         "librpcrt4.a",
 277         "libsecur32.a",
 278         "libsetupapi.a",
 279         "libshell32.a",
 280         "libsynchronization.a",
 281         "libuser32.a",
 282         "libuserenv.a",
 283         "libuuid.a",
 284         "libwinhttp.a",
 285         "libwinmm.a",
 286         "libwinspool.a",
 287         "libws2_32.a",
 288         "libwsock32.a",
 289     ];
 290
 291     //Find mingw artifacts we want to bundle
 292     let target_tools = find_files(&target_tools, &bin_path);
 293     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 294     let target_libs = find_files(&target_libs, &lib_path);
 295
 296     // Copy runtime dlls next to rustc.exe
 297     let dist_bin_dir = rust_root.join("bin/");
 298     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 299     for src in rustc_dlls {
 300         builder.copy_to_folder(&src, &dist_bin_dir);
 301     }
 302
 303     //Copy platform tools to platform-specific bin directory
 304     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 305     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 306     for src in target_tools {
 307         builder.copy_to_folder(&src, &target_bin_dir);
 308     }
 309
 310     // Warn windows-gnu users that the bundled GCC cannot compile C files
 311     builder.create(
 312         &target_bin_dir.join("GCC-WARNING.txt"),
 313         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 314          used as a linker. In order to be able to compile projects containing C code use\
 315          the GCC provided by MinGW or Cygwin."
 316     );
 317
 318     //Copy platform libs to platform-specific lib directory
 319     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 320     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 321     for src in target_libs {
 322         builder.copy_to_folder(&src, &target_lib_dir);
 323     }
 324 }
 325
 326 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 327 pub struct Mingw {
 328     pub host: Interned<String>,
 329 }
 330
 331 impl Step for Mingw {
 332     type Output = Option<PathBuf>;
 333     const DEFAULT: bool = true;
 334
 335     fn should_run(run: ShouldRun) -> ShouldRun {
 336         run.never()
 337     }
 338
 339     fn make_run(run: RunConfig) {
 340         run.builder.ensure(Mingw { host: run.target });
 341     }
 342
 343     /// Build the `rust-mingw` installer component.
 344     ///
 345     /// This contains all the bits and pieces to run the MinGW Windows targets
 346     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 347     fn run(self, builder: &Builder) -> Option<PathBuf> {
 348         let host = self.host;
 349
 350         if !host.contains("pc-windows-gnu") {
 351             return None;
 352         }
 353
 354         builder.info(&format!("Dist mingw ({})", host));
 355         let name = pkgname(builder, "rust-mingw");
 356         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 357         let _ = fs::remove_dir_all(&image);
 358         t!(fs::create_dir_all(&image));
 359
 360         // The first argument is a "temporary directory" which is just
 361         // thrown away (this contains the runtime DLLs included in the rustc package
 362         // above) and the second argument is where to place all the MinGW components
 363         // (which is what we want).
 364         make_win_dist(&tmpdir(builder), &image, host, &builder);
 365
 366         let mut cmd = rust_installer(builder);
 367         cmd.arg("generate")
 368            .arg("--product-name=Rust-MinGW")
 369            .arg("--rel-manifest-dir=rustlib")
 370            .arg("--success-message=Rust-MinGW-is-installed.")
 371            .arg("--image-dir").arg(&image)
 372            .arg("--work-dir").arg(&tmpdir(builder))
 373            .arg("--output-dir").arg(&distdir(builder))
 374            .arg(format!("--package-name={}-{}", name, host))
 375            .arg("--component-name=rust-mingw")
 376            .arg("--legacy-manifest-dirs=rustlib,cargo");
 377         builder.run(&mut cmd);
 378         t!(fs::remove_dir_all(&image));
 379         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 380     }
 381 }
 382
 383 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 384 pub struct Rustc {
 385     pub compiler: Compiler,
 386 }
 387
 388 impl Step for Rustc {
 389     type Output = PathBuf;
 390     const DEFAULT: bool = true;
 391     const ONLY_HOSTS: bool = true;
 392
 393     fn should_run(run: ShouldRun) -> ShouldRun {
 394         run.path("src/librustc")
 395     }
 396
 397     fn make_run(run: RunConfig) {
 398         run.builder.ensure(Rustc {
 399             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 400         });
 401     }
 402
 403     /// Creates the `rustc` installer component.
 404     fn run(self, builder: &Builder) -> PathBuf {
 405         let compiler = self.compiler;
 406         let host = self.compiler.host;
 407
 408         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 409         let name = pkgname(builder, "rustc");
 410         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 411         let _ = fs::remove_dir_all(&image);
 412         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 413         let _ = fs::remove_dir_all(&overlay);
 414
 415         // Prepare the rustc "image", what will actually end up getting installed
 416         prepare_image(builder, compiler, &image);
 417
 418         // Prepare the overlay which is part of the tarball but won't actually be
 419         // installed
 420         let cp = |file: &str| {
 421             builder.install(&builder.src.join(file), &overlay, 0o644);
 422         };
 423         cp("COPYRIGHT");
 424         cp("LICENSE-APACHE");
 425         cp("LICENSE-MIT");
 426         cp("README.md");
 427         // tiny morsel of metadata is used by rust-packaging
 428         let version = builder.rust_version();
 429         builder.create(&overlay.join("version"), &version);
 430         if let Some(sha) = builder.rust_sha() {
 431             builder.create(&overlay.join("git-commit-hash"), &sha);
 432         }
 433
 434         // On MinGW we've got a few runtime DLL dependencies that we need to
 435         // include. The first argument to this script is where to put these DLLs
 436         // (the image we're creating), and the second argument is a junk directory
 437         // to ignore all other MinGW stuff the script creates.
 438         //
 439         // On 32-bit MinGW we're always including a DLL which needs some extra
 440         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 441         // anything requiring us to distribute a license, but it's likely the
 442         // install will *also* include the rust-mingw package, which also needs
 443         // licenses, so to be safe we just include it here in all MinGW packages.
 444         if host.contains("pc-windows-gnu") {
 445             make_win_dist(&image, &tmpdir(builder), host, builder);
 446
 447             let dst = image.join("share/doc");
 448             t!(fs::create_dir_all(&dst));
 449             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 450         }
 451
 452         // Finally, wrap everything up in a nice tarball!
 453         let mut cmd = rust_installer(builder);
 454         cmd.arg("generate")
 455            .arg("--product-name=Rust")
 456            .arg("--rel-manifest-dir=rustlib")
 457            .arg("--success-message=Rust-is-ready-to-roll.")
 458            .arg("--image-dir").arg(&image)
 459            .arg("--work-dir").arg(&tmpdir(builder))
 460            .arg("--output-dir").arg(&distdir(builder))
 461            .arg("--non-installed-overlay").arg(&overlay)
 462            .arg(format!("--package-name={}-{}", name, host))
 463            .arg("--component-name=rustc")
 464            .arg("--legacy-manifest-dirs=rustlib,cargo");
 465         builder.run(&mut cmd);
 466         builder.remove_dir(&image);
 467         builder.remove_dir(&overlay);
 468
 469         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 470
 471         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 472             let host = compiler.host;
 473             let src = builder.sysroot(compiler);
 474             let libdir = libdir(&host);
 475
 476             // Copy rustc/rustdoc binaries
 477             t!(fs::create_dir_all(image.join("bin")));
 478             builder.cp_r(&src.join("bin"), &image.join("bin"));
 479
 480             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 481
 482             // Copy runtime DLLs needed by the compiler
 483             if libdir != "bin" {
 484                 for entry in builder.read_dir(&src.join(libdir)) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 489                         }
 490                     }
 491                 }
 492             }
 493
 494             // Copy over the codegen backends
 495             let backends_src = builder.sysroot_codegen_backends(compiler);
 496             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 497             let backends_dst = image.join(&backends_rel);
 498             t!(fs::create_dir_all(&backends_dst));
 499             builder.cp_r(&backends_src, &backends_dst);
 500
 501             // Copy libLLVM.so to the lib dir as well, if needed. While not
 502             // technically needed by rustc itself it's needed by lots of other
 503             // components like the llvm tools and LLD. LLD is included below and
 504             // tools/LLDB come later, so let's just throw it in the rustc
 505             // component for now.
 506             maybe_install_llvm_dylib(builder, host, image);
 507
 508             // Copy over lld if it's there
 509             if builder.config.lld_enabled {
 510                 let exe = exe("rust-lld", &compiler.host);
 511                 let src = builder.sysroot_libdir(compiler, host)
 512                     .parent()
 513                     .unwrap()
 514                     .join("bin")
 515                     .join(&exe);
 516                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 517                 let dst = image.join("lib/rustlib")
 518                     .join(&*host)
 519                     .join("bin")
 520                     .join(&exe);
 521                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 522                 builder.copy(&src, &dst);
 523             }
 524
 525             // Man pages
 526             t!(fs::create_dir_all(image.join("share/man/man1")));
 527             let man_src = builder.src.join("src/doc/man");
 528             let man_dst = image.join("share/man/man1");
 529             let month_year = t!(time::strftime("%B %Y", &time::now()));
 530             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 531             // to hardlink, and we don't want to edit the source templates
 532             for file_entry in builder.read_dir(&man_src) {
 533                 let page_src = file_entry.path();
 534                 let page_dst = man_dst.join(file_entry.file_name());
 535                 t!(fs::copy(&page_src, &page_dst));
 536                 // template in month/year and version number
 537                 builder.replace_in_file(&page_dst,
 538                                 &[("<INSERT DATE HERE>", &month_year),
 539                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 540             }
 541
 542             // Debugger scripts
 543             builder.ensure(DebuggerScripts {
 544                 sysroot: INTERNER.intern_path(image.to_owned()),
 545                 host,
 546             });
 547
 548             // Misc license info
 549             let cp = |file: &str| {
 550                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 551             };
 552             cp("COPYRIGHT");
 553             cp("LICENSE-APACHE");
 554             cp("LICENSE-MIT");
 555             cp("README.md");
 556         }
 557     }
 558 }
 559
 560 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 561 pub struct DebuggerScripts {
 562     pub sysroot: Interned<PathBuf>,
 563     pub host: Interned<String>,
 564 }
 565
 566 impl Step for DebuggerScripts {
 567     type Output = ();
 568
 569     fn should_run(run: ShouldRun) -> ShouldRun {
 570         run.path("src/lldb_batchmode.py")
 571     }
 572
 573     fn make_run(run: RunConfig) {
 574         run.builder.ensure(DebuggerScripts {
 575             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 576             host: run.target,
 577         });
 578     }
 579
 580     /// Copies debugger scripts for `target` into the `sysroot` specified.
 581     fn run(self, builder: &Builder) {
 582         let host = self.host;
 583         let sysroot = self.sysroot;
 584         let dst = sysroot.join("lib/rustlib/etc");
 585         t!(fs::create_dir_all(&dst));
 586         let cp_debugger_script = |file: &str| {
 587             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 588         };
 589         if host.contains("windows-msvc") {
 590             // windbg debugger scripts
 591             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 592                 0o755);
 593
 594             cp_debugger_script("natvis/intrinsic.natvis");
 595             cp_debugger_script("natvis/liballoc.natvis");
 596             cp_debugger_script("natvis/libcore.natvis");
 597         } else {
 598             cp_debugger_script("debugger_pretty_printers_common.py");
 599
 600             // gdb debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 602                     0o755);
 603
 604             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 605             cp_debugger_script("gdb_rust_pretty_printing.py");
 606
 607             // lldb debugger scripts
 608             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 609                     0o755);
 610
 611             cp_debugger_script("lldb_rust_formatters.py");
 612         }
 613     }
 614 }
 615
 616 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 617 pub struct Std {
 618     pub compiler: Compiler,
 619     pub target: Interned<String>,
 620 }
 621
 622 impl Step for Std {
 623     type Output = PathBuf;
 624     const DEFAULT: bool = true;
 625
 626     fn should_run(run: ShouldRun) -> ShouldRun {
 627         run.path("src/libstd")
 628     }
 629
 630     fn make_run(run: RunConfig) {
 631         run.builder.ensure(Std {
 632             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 633             target: run.target,
 634         });
 635     }
 636
 637     fn run(self, builder: &Builder) -> PathBuf {
 638         let compiler = self.compiler;
 639         let target = self.target;
 640
 641         let name = pkgname(builder, "rust-std");
 642         builder.info(&format!("Dist std stage{} ({} -> {})",
 643             compiler.stage, &compiler.host, target));
 644
 645         // The only true set of target libraries came from the build triple, so
 646         // let's reduce redundant work by only producing archives from that host.
 647         if compiler.host != builder.config.build {
 648             builder.info("\tskipping, not a build host");
 649             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 650         }
 651
 652         // We want to package up as many target libraries as possible
 653         // for the `rust-std` package, so if this is a host target we
 654         // depend on librustc and otherwise we just depend on libtest.
 655         if builder.hosts.iter().any(|t| t == target) {
 656             builder.ensure(compile::Rustc { compiler, target });
 657         } else {
 658             if builder.no_std(target) == Some(true) {
 659                 // the `test` doesn't compile for no-std targets
 660                 builder.ensure(compile::Std { compiler, target });
 661             } else {
 662                 builder.ensure(compile::Test { compiler, target });
 663             }
 664         }
 665
 666         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 667         let _ = fs::remove_dir_all(&image);
 668
 669         let dst = image.join("lib/rustlib").join(target);
 670         t!(fs::create_dir_all(&dst));
 671         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 672         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 673         builder.cp_filtered(&src, &dst, &|path| {
 674             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 675                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 676                     return false
 677                 }
 678                 if name == "bin" {
 679                     return false
 680                 }
 681                 if name.contains("LLVM") {
 682                     return false
 683                 }
 684             }
 685             true
 686         });
 687
 688         let mut cmd = rust_installer(builder);
 689         cmd.arg("generate")
 690            .arg("--product-name=Rust")
 691            .arg("--rel-manifest-dir=rustlib")
 692            .arg("--success-message=std-is-standing-at-the-ready.")
 693            .arg("--image-dir").arg(&image)
 694            .arg("--work-dir").arg(&tmpdir(builder))
 695            .arg("--output-dir").arg(&distdir(builder))
 696            .arg(format!("--package-name={}-{}", name, target))
 697            .arg(format!("--component-name=rust-std-{}", target))
 698            .arg("--legacy-manifest-dirs=rustlib,cargo");
 699         builder.run(&mut cmd);
 700         builder.remove_dir(&image);
 701         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 702     }
 703 }
 704
 705 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 706 pub struct Analysis {
 707     pub compiler: Compiler,
 708     pub target: Interned<String>,
 709 }
 710
 711 impl Step for Analysis {
 712     type Output = PathBuf;
 713     const DEFAULT: bool = true;
 714
 715     fn should_run(run: ShouldRun) -> ShouldRun {
 716         let builder = run.builder;
 717         run.path("analysis").default_condition(builder.config.extended)
 718     }
 719
 720     fn make_run(run: RunConfig) {
 721         run.builder.ensure(Analysis {
 722             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 723             target: run.target,
 724         });
 725     }
 726
 727     /// Creates a tarball of save-analysis metadata, if available.
 728     fn run(self, builder: &Builder) -> PathBuf {
 729         let compiler = self.compiler;
 730         let target = self.target;
 731         assert!(builder.config.extended);
 732         builder.info("Dist analysis");
 733         let name = pkgname(builder, "rust-analysis");
 734
 735         if &compiler.host != builder.config.build {
 736             builder.info("\tskipping, not a build host");
 737             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 738         }
 739
 740         builder.ensure(Std { compiler, target });
 741
 742         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 743         // stage2 artifacts is simply copied from stage1 in that case.
 744         let compiler = if builder.force_use_stage1(compiler, target) {
 745             builder.compiler(1, compiler.host)
 746         } else {
 747             compiler.clone()
 748         };
 749
 750         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 751
 752         let src = builder.stage_out(compiler, Mode::Std)
 753             .join(target).join(builder.cargo_dir()).join("deps");
 754
 755         let image_src = src.join("save-analysis");
 756         let dst = image.join("lib/rustlib").join(target).join("analysis");
 757         t!(fs::create_dir_all(&dst));
 758         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 759         builder.cp_r(&image_src, &dst);
 760
 761         let mut cmd = rust_installer(builder);
 762         cmd.arg("generate")
 763            .arg("--product-name=Rust")
 764            .arg("--rel-manifest-dir=rustlib")
 765            .arg("--success-message=save-analysis-saved.")
 766            .arg("--image-dir").arg(&image)
 767            .arg("--work-dir").arg(&tmpdir(builder))
 768            .arg("--output-dir").arg(&distdir(builder))
 769            .arg(format!("--package-name={}-{}", name, target))
 770            .arg(format!("--component-name=rust-analysis-{}", target))
 771            .arg("--legacy-manifest-dirs=rustlib,cargo");
 772         builder.run(&mut cmd);
 773         builder.remove_dir(&image);
 774         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 775     }
 776 }
 777
 778 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 779     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 780         let spath = match path.to_str() {
 781             Some(path) => path,
 782             None => return false,
 783         };
 784         if spath.ends_with("~") || spath.ends_with(".pyc") {
 785             return false
 786         }
 787         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 788             (spath.ends_with(".ll") ||
 789              spath.ends_with(".td") ||
 790              spath.ends_with(".s")) {
 791             return false
 792         }
 793         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 794             return false
 795         }
 796
 797         let full_path = Path::new(dir).join(path);
 798         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 799             return false;
 800         }
 801
 802         let excludes = [
 803             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 804             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 805             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 806             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 807         ];
 808         !path.iter()
 809              .map(|s| s.to_str().unwrap())
 810              .any(|s| excludes.contains(&s))
 811     }
 812
 813     // Copy the directories using our filter
 814     for item in src_dirs {
 815         let dst = &dst_dir.join(item);
 816         t!(fs::create_dir_all(dst));
 817         builder.cp_filtered(
 818             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 819     }
 820 }
 821
 822 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 823 pub struct Src;
 824
 825 impl Step for Src {
 826     /// The output path of the src installer tarball
 827     type Output = PathBuf;
 828     const DEFAULT: bool = true;
 829     const ONLY_HOSTS: bool = true;
 830
 831     fn should_run(run: ShouldRun) -> ShouldRun {
 832         run.path("src")
 833     }
 834
 835     fn make_run(run: RunConfig) {
 836         run.builder.ensure(Src);
 837     }
 838
 839     /// Creates the `rust-src` installer component
 840     fn run(self, builder: &Builder) -> PathBuf {
 841         builder.info("Dist src");
 842
 843         let name = pkgname(builder, "rust-src");
 844         let image = tmpdir(builder).join(format!("{}-image", name));
 845         let _ = fs::remove_dir_all(&image);
 846
 847         let dst = image.join("lib/rustlib/src");
 848         let dst_src = dst.join("rust");
 849         t!(fs::create_dir_all(&dst_src));
 850
 851         let src_files = [
 852             "Cargo.lock",
 853         ];
 854         // This is the reduced set of paths which will become the rust-src component
 855         // (essentially libstd and all of its path dependencies)
 856         let std_src_dirs = [
 857             "src/build_helper",
 858             "src/liballoc",
 859             "src/libcore",
 860             "src/libpanic_abort",
 861             "src/libpanic_unwind",
 862             "src/librustc_asan",
 863             "src/librustc_lsan",
 864             "src/librustc_msan",
 865             "src/librustc_tsan",
 866             "src/libstd",
 867             "src/libunwind",
 868             "src/libtest",
 869             "src/libterm",
 870             "src/libprofiler_builtins",
 871             "src/stdsimd",
 872             "src/libproc_macro",
 873             "src/tools/rustc-std-workspace-core",
 874         ];
 875
 876         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 877         for file in src_files.iter() {
 878             builder.copy(&builder.src.join(file), &dst_src.join(file));
 879         }
 880
 881         // Create source tarball in rust-installer format
 882         let mut cmd = rust_installer(builder);
 883         cmd.arg("generate")
 884            .arg("--product-name=Rust")
 885            .arg("--rel-manifest-dir=rustlib")
 886            .arg("--success-message=Awesome-Source.")
 887            .arg("--image-dir").arg(&image)
 888            .arg("--work-dir").arg(&tmpdir(builder))
 889            .arg("--output-dir").arg(&distdir(builder))
 890            .arg(format!("--package-name={}", name))
 891            .arg("--component-name=rust-src")
 892            .arg("--legacy-manifest-dirs=rustlib,cargo");
 893         builder.run(&mut cmd);
 894
 895         builder.remove_dir(&image);
 896         distdir(builder).join(&format!("{}.tar.gz", name))
 897     }
 898 }
 899
 900 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 901
 902 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 903 pub struct PlainSourceTarball;
 904
 905 impl Step for PlainSourceTarball {
 906     /// Produces the location of the tarball generated
 907     type Output = PathBuf;
 908     const DEFAULT: bool = true;
 909     const ONLY_HOSTS: bool = true;
 910
 911     fn should_run(run: ShouldRun) -> ShouldRun {
 912         let builder = run.builder;
 913         run.path("src").default_condition(builder.config.rust_dist_src)
 914     }
 915
 916     fn make_run(run: RunConfig) {
 917         run.builder.ensure(PlainSourceTarball);
 918     }
 919
 920     /// Creates the plain source tarball
 921     fn run(self, builder: &Builder) -> PathBuf {
 922         builder.info("Create plain source tarball");
 923
 924         // Make sure that the root folder of tarball has the correct name
 925         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 926         let plain_dst_src = tmpdir(builder).join(&plain_name);
 927         let _ = fs::remove_dir_all(&plain_dst_src);
 928         t!(fs::create_dir_all(&plain_dst_src));
 929
 930         // This is the set of root paths which will become part of the source package
 931         let src_files = [
 932             "COPYRIGHT",
 933             "LICENSE-APACHE",
 934             "LICENSE-MIT",
 935             "CONTRIBUTING.md",
 936             "README.md",
 937             "RELEASES.md",
 938             "configure",
 939             "x.py",
 940             "config.toml.example",
 941             "Cargo.toml",
 942             "Cargo.lock",
 943         ];
 944         let src_dirs = [
 945             "src",
 946         ];
 947
 948         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 949
 950         // Copy the files normally
 951         for item in &src_files {
 952             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 953         }
 954
 955         // Create the version file
 956         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 957         if let Some(sha) = builder.rust_sha() {
 958             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 959         }
 960
 961         // If we're building from git sources, we need to vendor a complete distribution.
 962         if builder.rust_info.is_git() {
 963             // Get cargo-vendor installed, if it isn't already.
 964             let mut has_cargo_vendor = false;
 965             let mut cmd = Command::new(&builder.initial_cargo);
 966             for line in output(cmd.arg("install").arg("--list")).lines() {
 967                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 968             }
 969             if !has_cargo_vendor {
 970                 let mut cmd = builder.cargo(
 971                     builder.compiler(0, builder.config.build),
 972                     Mode::ToolBootstrap,
 973                     builder.config.build,
 974                     "install"
 975                 );
 976                 cmd.arg("--force")
 977                    .arg("--debug")
 978                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 979                    .arg("cargo-vendor");
 980                 builder.run(&mut cmd);
 981             }
 982
 983             // Vendor all Cargo dependencies
 984             let mut cmd = Command::new(&builder.initial_cargo);
 985             cmd.arg("vendor")
 986                .current_dir(&plain_dst_src);
 987             builder.run(&mut cmd);
 988         }
 989
 990         // Create plain source tarball
 991         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
 992         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
 993         tarball.set_extension(""); // strip .gz
 994         tarball.set_extension(""); // strip .tar
 995         if let Some(dir) = tarball.parent() {
 996             builder.create_dir(&dir);
 997         }
 998         builder.info("running installer");
 999         let mut cmd = rust_installer(builder);
1000         cmd.arg("tarball")
1001            .arg("--input").arg(&plain_name)
1002            .arg("--output").arg(&tarball)
1003            .arg("--work-dir=.")
1004            .current_dir(tmpdir(builder));
1005         builder.run(&mut cmd);
1006         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1007     }
1008 }
1009
1010 // We have to run a few shell scripts, which choke quite a bit on both `\`
1011 // characters and on `C:\` paths, so normalize both of them away.
1012 pub fn sanitize_sh(path: &Path) -> String {
1013     let path = path.to_str().unwrap().replace("\\", "/");
1014     return change_drive(&path).unwrap_or(path);
1015
1016     fn change_drive(s: &str) -> Option<String> {
1017         let mut ch = s.chars();
1018         let drive = ch.next().unwrap_or('C');
1019         if ch.next() != Some(':') {
1020             return None
1021         }
1022         if ch.next() != Some('/') {
1023             return None
1024         }
1025         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1026     }
1027 }
1028
1029 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1030 pub struct Cargo {
1031     pub stage: u32,
1032     pub target: Interned<String>,
1033 }
1034
1035 impl Step for Cargo {
1036     type Output = PathBuf;
1037     const ONLY_HOSTS: bool = true;
1038
1039     fn should_run(run: ShouldRun) -> ShouldRun {
1040         run.path("cargo")
1041     }
1042
1043     fn make_run(run: RunConfig) {
1044         run.builder.ensure(Cargo {
1045             stage: run.builder.top_stage,
1046             target: run.target,
1047         });
1048     }
1049
1050     fn run(self, builder: &Builder) -> PathBuf {
1051         let stage = self.stage;
1052         let target = self.target;
1053
1054         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1055         let src = builder.src.join("src/tools/cargo");
1056         let etc = src.join("src/etc");
1057         let release_num = builder.release_num("cargo");
1058         let name = pkgname(builder, "cargo");
1059         let version = builder.cargo_info.version(builder, &release_num);
1060
1061         let tmp = tmpdir(builder);
1062         let image = tmp.join("cargo-image");
1063         drop(fs::remove_dir_all(&image));
1064         builder.create_dir(&image);
1065
1066         // Prepare the image directory
1067         builder.create_dir(&image.join("share/zsh/site-functions"));
1068         builder.create_dir(&image.join("etc/bash_completion.d"));
1069         let cargo = builder.ensure(tool::Cargo {
1070             compiler: builder.compiler(stage, builder.config.build),
1071             target
1072         });
1073         builder.install(&cargo, &image.join("bin"), 0o755);
1074         for man in t!(etc.join("man").read_dir()) {
1075             let man = t!(man);
1076             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1077         }
1078         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1079         builder.copy(&etc.join("cargo.bashcomp.sh"),
1080              &image.join("etc/bash_completion.d/cargo"));
1081         let doc = image.join("share/doc/cargo");
1082         builder.install(&src.join("README.md"), &doc, 0o644);
1083         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1084         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1085         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1086
1087         // Prepare the overlay
1088         let overlay = tmp.join("cargo-overlay");
1089         drop(fs::remove_dir_all(&overlay));
1090         builder.create_dir(&overlay);
1091         builder.install(&src.join("README.md"), &overlay, 0o644);
1092         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1093         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1094         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1095         builder.create(&overlay.join("version"), &version);
1096
1097         // Generate the installer tarball
1098         let mut cmd = rust_installer(builder);
1099         cmd.arg("generate")
1100            .arg("--product-name=Rust")
1101            .arg("--rel-manifest-dir=rustlib")
1102            .arg("--success-message=Rust-is-ready-to-roll.")
1103            .arg("--image-dir").arg(&image)
1104            .arg("--work-dir").arg(&tmpdir(builder))
1105            .arg("--output-dir").arg(&distdir(builder))
1106            .arg("--non-installed-overlay").arg(&overlay)
1107            .arg(format!("--package-name={}-{}", name, target))
1108            .arg("--component-name=cargo")
1109            .arg("--legacy-manifest-dirs=rustlib,cargo");
1110         builder.run(&mut cmd);
1111         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1112     }
1113 }
1114
1115 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1116 pub struct Rls {
1117     pub stage: u32,
1118     pub target: Interned<String>,
1119 }
1120
1121 impl Step for Rls {
1122     type Output = Option<PathBuf>;
1123     const ONLY_HOSTS: bool = true;
1124
1125     fn should_run(run: ShouldRun) -> ShouldRun {
1126         run.path("rls")
1127     }
1128
1129     fn make_run(run: RunConfig) {
1130         run.builder.ensure(Rls {
1131             stage: run.builder.top_stage,
1132             target: run.target,
1133         });
1134     }
1135
1136     fn run(self, builder: &Builder) -> Option<PathBuf> {
1137         let stage = self.stage;
1138         let target = self.target;
1139         assert!(builder.config.extended);
1140
1141         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1142         let src = builder.src.join("src/tools/rls");
1143         let release_num = builder.release_num("rls");
1144         let name = pkgname(builder, "rls");
1145         let version = builder.rls_info.version(builder, &release_num);
1146
1147         let tmp = tmpdir(builder);
1148         let image = tmp.join("rls-image");
1149         drop(fs::remove_dir_all(&image));
1150         t!(fs::create_dir_all(&image));
1151
1152         // Prepare the image directory
1153         // We expect RLS to build, because we've exited this step above if tool
1154         // state for RLS isn't testing.
1155         let rls = builder.ensure(tool::Rls {
1156             compiler: builder.compiler(stage, builder.config.build),
1157             target, extra_features: Vec::new()
1158         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1159
1160         builder.install(&rls, &image.join("bin"), 0o755);
1161         let doc = image.join("share/doc/rls");
1162         builder.install(&src.join("README.md"), &doc, 0o644);
1163         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1164         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1165
1166         // Prepare the overlay
1167         let overlay = tmp.join("rls-overlay");
1168         drop(fs::remove_dir_all(&overlay));
1169         t!(fs::create_dir_all(&overlay));
1170         builder.install(&src.join("README.md"), &overlay, 0o644);
1171         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1172         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1173         builder.create(&overlay.join("version"), &version);
1174
1175         // Generate the installer tarball
1176         let mut cmd = rust_installer(builder);
1177         cmd.arg("generate")
1178            .arg("--product-name=Rust")
1179            .arg("--rel-manifest-dir=rustlib")
1180            .arg("--success-message=RLS-ready-to-serve.")
1181            .arg("--image-dir").arg(&image)
1182            .arg("--work-dir").arg(&tmpdir(builder))
1183            .arg("--output-dir").arg(&distdir(builder))
1184            .arg("--non-installed-overlay").arg(&overlay)
1185            .arg(format!("--package-name={}-{}", name, target))
1186            .arg("--legacy-manifest-dirs=rustlib,cargo")
1187            .arg("--component-name=rls-preview");
1188
1189         builder.run(&mut cmd);
1190         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1191     }
1192 }
1193
1194 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1195 pub struct Clippy {
1196     pub stage: u32,
1197     pub target: Interned<String>,
1198 }
1199
1200 impl Step for Clippy {
1201     type Output = Option<PathBuf>;
1202     const ONLY_HOSTS: bool = true;
1203
1204     fn should_run(run: ShouldRun) -> ShouldRun {
1205         run.path("clippy")
1206     }
1207
1208     fn make_run(run: RunConfig) {
1209         run.builder.ensure(Clippy {
1210             stage: run.builder.top_stage,
1211             target: run.target,
1212         });
1213     }
1214
1215     fn run(self, builder: &Builder) -> Option<PathBuf> {
1216         let stage = self.stage;
1217         let target = self.target;
1218         assert!(builder.config.extended);
1219
1220         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1221         let src = builder.src.join("src/tools/clippy");
1222         let release_num = builder.release_num("clippy");
1223         let name = pkgname(builder, "clippy");
1224         let version = builder.clippy_info.version(builder, &release_num);
1225
1226         let tmp = tmpdir(builder);
1227         let image = tmp.join("clippy-image");
1228         drop(fs::remove_dir_all(&image));
1229         builder.create_dir(&image);
1230
1231         // Prepare the image directory
1232         // We expect clippy to build, because we've exited this step above if tool
1233         // state for clippy isn't testing.
1234         let clippy = builder.ensure(tool::Clippy {
1235             compiler: builder.compiler(stage, builder.config.build),
1236             target, extra_features: Vec::new()
1237         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1238         let cargoclippy = builder.ensure(tool::CargoClippy {
1239             compiler: builder.compiler(stage, builder.config.build),
1240             target, extra_features: Vec::new()
1241         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1242
1243         builder.install(&clippy, &image.join("bin"), 0o755);
1244         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1245         let doc = image.join("share/doc/clippy");
1246         builder.install(&src.join("README.md"), &doc, 0o644);
1247         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1248         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1249
1250         // Prepare the overlay
1251         let overlay = tmp.join("clippy-overlay");
1252         drop(fs::remove_dir_all(&overlay));
1253         t!(fs::create_dir_all(&overlay));
1254         builder.install(&src.join("README.md"), &overlay, 0o644);
1255         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1256         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1257         builder.create(&overlay.join("version"), &version);
1258
1259         // Generate the installer tarball
1260         let mut cmd = rust_installer(builder);
1261         cmd.arg("generate")
1262            .arg("--product-name=Rust")
1263            .arg("--rel-manifest-dir=rustlib")
1264            .arg("--success-message=clippy-ready-to-serve.")
1265            .arg("--image-dir").arg(&image)
1266            .arg("--work-dir").arg(&tmpdir(builder))
1267            .arg("--output-dir").arg(&distdir(builder))
1268            .arg("--non-installed-overlay").arg(&overlay)
1269            .arg(format!("--package-name={}-{}", name, target))
1270            .arg("--legacy-manifest-dirs=rustlib,cargo")
1271            .arg("--component-name=clippy-preview");
1272
1273         builder.run(&mut cmd);
1274         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1275     }
1276 }
1277
1278 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1279 pub struct Miri {
1280     pub stage: u32,
1281     pub target: Interned<String>,
1282 }
1283
1284 impl Step for Miri {
1285     type Output = Option<PathBuf>;
1286     const ONLY_HOSTS: bool = true;
1287
1288     fn should_run(run: ShouldRun) -> ShouldRun {
1289         run.path("miri")
1290     }
1291
1292     fn make_run(run: RunConfig) {
1293         run.builder.ensure(Miri {
1294             stage: run.builder.top_stage,
1295             target: run.target,
1296         });
1297     }
1298
1299     fn run(self, builder: &Builder) -> Option<PathBuf> {
1300         let stage = self.stage;
1301         let target = self.target;
1302         assert!(builder.config.extended);
1303
1304         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1305         let src = builder.src.join("src/tools/miri");
1306         let release_num = builder.release_num("miri");
1307         let name = pkgname(builder, "miri");
1308         let version = builder.miri_info.version(builder, &release_num);
1309
1310         let tmp = tmpdir(builder);
1311         let image = tmp.join("miri-image");
1312         drop(fs::remove_dir_all(&image));
1313         builder.create_dir(&image);
1314
1315         // Prepare the image directory
1316         // We expect miri to build, because we've exited this step above if tool
1317         // state for miri isn't testing.
1318         let miri = builder.ensure(tool::Miri {
1319             compiler: builder.compiler(stage, builder.config.build),
1320             target, extra_features: Vec::new()
1321         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1322         let cargomiri = builder.ensure(tool::CargoMiri {
1323             compiler: builder.compiler(stage, builder.config.build),
1324             target, extra_features: Vec::new()
1325         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1326
1327         builder.install(&miri, &image.join("bin"), 0o755);
1328         builder.install(&cargomiri, &image.join("bin"), 0o755);
1329         let doc = image.join("share/doc/miri");
1330         builder.install(&src.join("README.md"), &doc, 0o644);
1331         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1332         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1333
1334         // Prepare the overlay
1335         let overlay = tmp.join("miri-overlay");
1336         drop(fs::remove_dir_all(&overlay));
1337         t!(fs::create_dir_all(&overlay));
1338         builder.install(&src.join("README.md"), &overlay, 0o644);
1339         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1340         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1341         builder.create(&overlay.join("version"), &version);
1342
1343         // Generate the installer tarball
1344         let mut cmd = rust_installer(builder);
1345         cmd.arg("generate")
1346            .arg("--product-name=Rust")
1347            .arg("--rel-manifest-dir=rustlib")
1348            .arg("--success-message=miri-ready-to-serve.")
1349            .arg("--image-dir").arg(&image)
1350            .arg("--work-dir").arg(&tmpdir(builder))
1351            .arg("--output-dir").arg(&distdir(builder))
1352            .arg("--non-installed-overlay").arg(&overlay)
1353            .arg(format!("--package-name={}-{}", name, target))
1354            .arg("--legacy-manifest-dirs=rustlib,cargo")
1355            .arg("--component-name=miri-preview");
1356
1357         builder.run(&mut cmd);
1358         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1359     }
1360 }
1361
1362 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1363 pub struct Rustfmt {
1364     pub stage: u32,
1365     pub target: Interned<String>,
1366 }
1367
1368 impl Step for Rustfmt {
1369     type Output = Option<PathBuf>;
1370     const ONLY_HOSTS: bool = true;
1371
1372     fn should_run(run: ShouldRun) -> ShouldRun {
1373         run.path("rustfmt")
1374     }
1375
1376     fn make_run(run: RunConfig) {
1377         run.builder.ensure(Rustfmt {
1378             stage: run.builder.top_stage,
1379             target: run.target,
1380         });
1381     }
1382
1383     fn run(self, builder: &Builder) -> Option<PathBuf> {
1384         let stage = self.stage;
1385         let target = self.target;
1386
1387         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1388         let src = builder.src.join("src/tools/rustfmt");
1389         let release_num = builder.release_num("rustfmt");
1390         let name = pkgname(builder, "rustfmt");
1391         let version = builder.rustfmt_info.version(builder, &release_num);
1392
1393         let tmp = tmpdir(builder);
1394         let image = tmp.join("rustfmt-image");
1395         drop(fs::remove_dir_all(&image));
1396         builder.create_dir(&image);
1397
1398         // Prepare the image directory
1399         let rustfmt = builder.ensure(tool::Rustfmt {
1400             compiler: builder.compiler(stage, builder.config.build),
1401             target, extra_features: Vec::new()
1402         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1403         let cargofmt = builder.ensure(tool::Cargofmt {
1404             compiler: builder.compiler(stage, builder.config.build),
1405             target, extra_features: Vec::new()
1406         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1407
1408         builder.install(&rustfmt, &image.join("bin"), 0o755);
1409         builder.install(&cargofmt, &image.join("bin"), 0o755);
1410         let doc = image.join("share/doc/rustfmt");
1411         builder.install(&src.join("README.md"), &doc, 0o644);
1412         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1413         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1414
1415         // Prepare the overlay
1416         let overlay = tmp.join("rustfmt-overlay");
1417         drop(fs::remove_dir_all(&overlay));
1418         builder.create_dir(&overlay);
1419         builder.install(&src.join("README.md"), &overlay, 0o644);
1420         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1421         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1422         builder.create(&overlay.join("version"), &version);
1423
1424         // Generate the installer tarball
1425         let mut cmd = rust_installer(builder);
1426         cmd.arg("generate")
1427            .arg("--product-name=Rust")
1428            .arg("--rel-manifest-dir=rustlib")
1429            .arg("--success-message=rustfmt-ready-to-fmt.")
1430            .arg("--image-dir").arg(&image)
1431            .arg("--work-dir").arg(&tmpdir(builder))
1432            .arg("--output-dir").arg(&distdir(builder))
1433            .arg("--non-installed-overlay").arg(&overlay)
1434            .arg(format!("--package-name={}-{}", name, target))
1435            .arg("--legacy-manifest-dirs=rustlib,cargo")
1436            .arg("--component-name=rustfmt-preview");
1437
1438         builder.run(&mut cmd);
1439         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1440     }
1441 }
1442
1443 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1444 pub struct Extended {
1445     stage: u32,
1446     host: Interned<String>,
1447     target: Interned<String>,
1448 }
1449
1450 impl Step for Extended {
1451     type Output = ();
1452     const DEFAULT: bool = true;
1453     const ONLY_HOSTS: bool = true;
1454
1455     fn should_run(run: ShouldRun) -> ShouldRun {
1456         let builder = run.builder;
1457         run.path("extended").default_condition(builder.config.extended)
1458     }
1459
1460     fn make_run(run: RunConfig) {
1461         run.builder.ensure(Extended {
1462             stage: run.builder.top_stage,
1463             host: run.builder.config.build,
1464             target: run.target,
1465         });
1466     }
1467
1468     /// Creates a combined installer for the specified target in the provided stage.
1469     fn run(self, builder: &Builder) {
1470         let stage = self.stage;
1471         let target = self.target;
1472
1473         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1474
1475         let rustc_installer = builder.ensure(Rustc {
1476             compiler: builder.compiler(stage, target),
1477         });
1478         let cargo_installer = builder.ensure(Cargo { stage, target });
1479         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1480         let rls_installer = builder.ensure(Rls { stage, target });
1481         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1482         let clippy_installer = builder.ensure(Clippy { stage, target });
1483         let lldb_installer = builder.ensure(Lldb { target });
1484         let mingw_installer = builder.ensure(Mingw { host: target });
1485         let analysis_installer = builder.ensure(Analysis {
1486             compiler: builder.compiler(stage, self.host),
1487             target
1488         });
1489
1490         let docs_installer = builder.ensure(Docs { stage, host: target, });
1491         let std_installer = builder.ensure(Std {
1492             compiler: builder.compiler(stage, self.host),
1493             target,
1494         });
1495
1496         let tmp = tmpdir(builder);
1497         let overlay = tmp.join("extended-overlay");
1498         let etc = builder.src.join("src/etc/installer");
1499         let work = tmp.join("work");
1500
1501         let _ = fs::remove_dir_all(&overlay);
1502         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1503         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1504         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1505         let version = builder.rust_version();
1506         builder.create(&overlay.join("version"), &version);
1507         if let Some(sha) = builder.rust_sha() {
1508             builder.create(&overlay.join("git-commit-hash"), &sha);
1509         }
1510         builder.install(&etc.join("README.md"), &overlay, 0o644);
1511
1512         // When rust-std package split from rustc, we needed to ensure that during
1513         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1514         // the std files during uninstall. To do this ensure that rustc comes
1515         // before rust-std in the list below.
1516         let mut tarballs = Vec::new();
1517         tarballs.push(rustc_installer);
1518         tarballs.push(cargo_installer);
1519         tarballs.extend(rls_installer.clone());
1520         tarballs.extend(clippy_installer.clone());
1521         tarballs.extend(rustfmt_installer.clone());
1522         tarballs.extend(llvm_tools_installer);
1523         tarballs.extend(lldb_installer);
1524         tarballs.push(analysis_installer);
1525         tarballs.push(std_installer);
1526         if builder.config.docs {
1527             tarballs.push(docs_installer);
1528         }
1529         if target.contains("pc-windows-gnu") {
1530             tarballs.push(mingw_installer.unwrap());
1531         }
1532         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1533         for tarball in &tarballs[1..] {
1534             input_tarballs.push(",");
1535             input_tarballs.push(tarball);
1536         }
1537
1538         let mut cmd = rust_installer(builder);
1539         cmd.arg("combine")
1540             .arg("--product-name=Rust")
1541             .arg("--rel-manifest-dir=rustlib")
1542             .arg("--success-message=Rust-is-ready-to-roll.")
1543             .arg("--work-dir").arg(&work)
1544             .arg("--output-dir").arg(&distdir(builder))
1545             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1546             .arg("--legacy-manifest-dirs=rustlib,cargo")
1547             .arg("--input-tarballs").arg(input_tarballs)
1548             .arg("--non-installed-overlay").arg(&overlay);
1549         builder.run(&mut cmd);
1550
1551         let mut license = String::new();
1552         license += &builder.read(&builder.src.join("COPYRIGHT"));
1553         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1554         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1555         license.push_str("\n");
1556         license.push_str("\n");
1557
1558         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1559         let mut rtf = rtf.to_string();
1560         rtf.push_str("\n");
1561         for line in license.lines() {
1562             rtf.push_str(line);
1563             rtf.push_str("\\line ");
1564         }
1565         rtf.push_str("}");
1566
1567         fn filter(contents: &str, marker: &str) -> String {
1568             let start = format!("tool-{}-start", marker);
1569             let end = format!("tool-{}-end", marker);
1570             let mut lines = Vec::new();
1571             let mut omitted = false;
1572             for line in contents.lines() {
1573                 if line.contains(&start) {
1574                     omitted = true;
1575                 } else if line.contains(&end) {
1576                     omitted = false;
1577                 } else if !omitted {
1578                     lines.push(line);
1579                 }
1580             }
1581
1582             lines.join("\n")
1583         }
1584
1585         let xform = |p: &Path| {
1586             let mut contents = t!(fs::read_to_string(p));
1587             if rls_installer.is_none() {
1588                 contents = filter(&contents, "rls");
1589             }
1590             if clippy_installer.is_none() {
1591                 contents = filter(&contents, "clippy");
1592             }
1593             if rustfmt_installer.is_none() {
1594                 contents = filter(&contents, "rustfmt");
1595             }
1596             let ret = tmp.join(p.file_name().unwrap());
1597             t!(fs::write(&ret, &contents));
1598             ret
1599         };
1600
1601         if target.contains("apple-darwin") {
1602             let pkg = tmp.join("pkg");
1603             let _ = fs::remove_dir_all(&pkg);
1604
1605             let pkgbuild = |component: &str| {
1606                 let mut cmd = Command::new("pkgbuild");
1607                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1608                     .arg("--scripts").arg(pkg.join(component))
1609                     .arg("--nopayload")
1610                     .arg(pkg.join(component).with_extension("pkg"));
1611                 builder.run(&mut cmd);
1612             };
1613
1614             let prepare = |name: &str| {
1615                 builder.create_dir(&pkg.join(name));
1616                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1617                         &pkg.join(name));
1618                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1619                 pkgbuild(name);
1620             };
1621             prepare("rustc");
1622             prepare("cargo");
1623             prepare("rust-docs");
1624             prepare("rust-std");
1625             prepare("rust-analysis");
1626
1627             if rls_installer.is_some() {
1628                 prepare("rls");
1629             }
1630             if clippy_installer.is_some() {
1631                 prepare("clippy");
1632             }
1633
1634             // create an 'uninstall' package
1635             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1636             pkgbuild("uninstall");
1637
1638             builder.create_dir(&pkg.join("res"));
1639             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1640             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1641             let mut cmd = Command::new("productbuild");
1642             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1643                 .arg("--resources").arg(pkg.join("res"))
1644                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1645                                                     pkgname(builder, "rust"),
1646                                                     target)))
1647                 .arg("--package-path").arg(&pkg);
1648             builder.run(&mut cmd);
1649         }
1650
1651         if target.contains("windows") {
1652             let exe = tmp.join("exe");
1653             let _ = fs::remove_dir_all(&exe);
1654
1655             let prepare = |name: &str| {
1656                 builder.create_dir(&exe.join(name));
1657                 let dir = if name == "rust-std" || name == "rust-analysis" {
1658                     format!("{}-{}", name, target)
1659                 } else if name == "rls" {
1660                     "rls-preview".to_string()
1661                 } else if name == "clippy" {
1662                     "clippy-preview".to_string()
1663                 } else {
1664                     name.to_string()
1665                 };
1666                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1667                             .join(dir),
1668                         &exe.join(name));
1669                 builder.remove(&exe.join(name).join("manifest.in"));
1670             };
1671             prepare("rustc");
1672             prepare("cargo");
1673             prepare("rust-analysis");
1674             prepare("rust-docs");
1675             prepare("rust-std");
1676             if rls_installer.is_some() {
1677                 prepare("rls");
1678             }
1679             if clippy_installer.is_some() {
1680                 prepare("clippy");
1681             }
1682             if target.contains("windows-gnu") {
1683                 prepare("rust-mingw");
1684             }
1685
1686             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1687             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1688             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1689             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1690             builder.create(&exe.join("LICENSE.txt"), &license);
1691
1692             // Generate exe installer
1693             let mut cmd = Command::new("iscc");
1694             cmd.arg("rust.iss")
1695                 .current_dir(&exe);
1696             if target.contains("windows-gnu") {
1697                 cmd.arg("/dMINGW");
1698             }
1699             add_env(builder, &mut cmd, target);
1700             builder.run(&mut cmd);
1701             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1702                     &distdir(builder),
1703                     0o755);
1704
1705             // Generate msi installer
1706             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1707             let heat = wix.join("bin/heat.exe");
1708             let candle = wix.join("bin/candle.exe");
1709             let light = wix.join("bin/light.exe");
1710
1711             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1712             builder.run(Command::new(&heat)
1713                             .current_dir(&exe)
1714                             .arg("dir")
1715                             .arg("rustc")
1716                             .args(&heat_flags)
1717                             .arg("-cg").arg("RustcGroup")
1718                             .arg("-dr").arg("Rustc")
1719                             .arg("-var").arg("var.RustcDir")
1720                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1721             builder.run(Command::new(&heat)
1722                             .current_dir(&exe)
1723                             .arg("dir")
1724                             .arg("rust-docs")
1725                             .args(&heat_flags)
1726                             .arg("-cg").arg("DocsGroup")
1727                             .arg("-dr").arg("Docs")
1728                             .arg("-var").arg("var.DocsDir")
1729                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1730                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1731             builder.run(Command::new(&heat)
1732                             .current_dir(&exe)
1733                             .arg("dir")
1734                             .arg("cargo")
1735                             .args(&heat_flags)
1736                             .arg("-cg").arg("CargoGroup")
1737                             .arg("-dr").arg("Cargo")
1738                             .arg("-var").arg("var.CargoDir")
1739                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1740                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1741             builder.run(Command::new(&heat)
1742                             .current_dir(&exe)
1743                             .arg("dir")
1744                             .arg("rust-std")
1745                             .args(&heat_flags)
1746                             .arg("-cg").arg("StdGroup")
1747                             .arg("-dr").arg("Std")
1748                             .arg("-var").arg("var.StdDir")
1749                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1750             if rls_installer.is_some() {
1751                 builder.run(Command::new(&heat)
1752                                 .current_dir(&exe)
1753                                 .arg("dir")
1754                                 .arg("rls")
1755                                 .args(&heat_flags)
1756                                 .arg("-cg").arg("RlsGroup")
1757                                 .arg("-dr").arg("Rls")
1758                                 .arg("-var").arg("var.RlsDir")
1759                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1760                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1761             }
1762             if clippy_installer.is_some() {
1763                 builder.run(Command::new(&heat)
1764                                 .current_dir(&exe)
1765                                 .arg("dir")
1766                                 .arg("clippy")
1767                                 .args(&heat_flags)
1768                                 .arg("-cg").arg("ClippyGroup")
1769                                 .arg("-dr").arg("Clippy")
1770                                 .arg("-var").arg("var.ClippyDir")
1771                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1772                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1773             }
1774             builder.run(Command::new(&heat)
1775                             .current_dir(&exe)
1776                             .arg("dir")
1777                             .arg("rust-analysis")
1778                             .args(&heat_flags)
1779                             .arg("-cg").arg("AnalysisGroup")
1780                             .arg("-dr").arg("Analysis")
1781                             .arg("-var").arg("var.AnalysisDir")
1782                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1783                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1784             if target.contains("windows-gnu") {
1785                 builder.run(Command::new(&heat)
1786                                 .current_dir(&exe)
1787                                 .arg("dir")
1788                                 .arg("rust-mingw")
1789                                 .args(&heat_flags)
1790                                 .arg("-cg").arg("GccGroup")
1791                                 .arg("-dr").arg("Gcc")
1792                                 .arg("-var").arg("var.GccDir")
1793                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1794             }
1795
1796             let candle = |input: &Path| {
1797                 let output = exe.join(input.file_stem().unwrap())
1798                                 .with_extension("wixobj");
1799                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1800                 let mut cmd = Command::new(&candle);
1801                 cmd.current_dir(&exe)
1802                     .arg("-nologo")
1803                     .arg("-dRustcDir=rustc")
1804                     .arg("-dDocsDir=rust-docs")
1805                     .arg("-dCargoDir=cargo")
1806                     .arg("-dStdDir=rust-std")
1807                     .arg("-dAnalysisDir=rust-analysis")
1808                     .arg("-arch").arg(&arch)
1809                     .arg("-out").arg(&output)
1810                     .arg(&input);
1811                 add_env(builder, &mut cmd, target);
1812
1813                 if rls_installer.is_some() {
1814                     cmd.arg("-dRlsDir=rls");
1815                 }
1816                 if clippy_installer.is_some() {
1817                     cmd.arg("-dClippyDir=clippy");
1818                 }
1819                 if target.contains("windows-gnu") {
1820                     cmd.arg("-dGccDir=rust-mingw");
1821                 }
1822                 builder.run(&mut cmd);
1823             };
1824             candle(&xform(&etc.join("msi/rust.wxs")));
1825             candle(&etc.join("msi/ui.wxs"));
1826             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1827             candle("RustcGroup.wxs".as_ref());
1828             candle("DocsGroup.wxs".as_ref());
1829             candle("CargoGroup.wxs".as_ref());
1830             candle("StdGroup.wxs".as_ref());
1831             if rls_installer.is_some() {
1832                 candle("RlsGroup.wxs".as_ref());
1833             }
1834             if clippy_installer.is_some() {
1835                 candle("ClippyGroup.wxs".as_ref());
1836             }
1837             candle("AnalysisGroup.wxs".as_ref());
1838
1839             if target.contains("windows-gnu") {
1840                 candle("GccGroup.wxs".as_ref());
1841             }
1842
1843             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1844             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1845             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1846
1847             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1848             let mut cmd = Command::new(&light);
1849             cmd.arg("-nologo")
1850                 .arg("-ext").arg("WixUIExtension")
1851                 .arg("-ext").arg("WixUtilExtension")
1852                 .arg("-out").arg(exe.join(&filename))
1853                 .arg("rust.wixobj")
1854                 .arg("ui.wixobj")
1855                 .arg("rustwelcomedlg.wixobj")
1856                 .arg("RustcGroup.wixobj")
1857                 .arg("DocsGroup.wixobj")
1858                 .arg("CargoGroup.wixobj")
1859                 .arg("StdGroup.wixobj")
1860                 .arg("AnalysisGroup.wixobj")
1861                 .current_dir(&exe);
1862
1863             if rls_installer.is_some() {
1864                 cmd.arg("RlsGroup.wixobj");
1865             }
1866             if clippy_installer.is_some() {
1867                 cmd.arg("ClippyGroup.wixobj");
1868             }
1869
1870             if target.contains("windows-gnu") {
1871                 cmd.arg("GccGroup.wixobj");
1872             }
1873             // ICE57 wrongly complains about the shortcuts
1874             cmd.arg("-sice:ICE57");
1875
1876             builder.run(&mut cmd);
1877
1878             if !builder.config.dry_run {
1879                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1880             }
1881         }
1882     }
1883 }
1884
1885 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1886     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1887     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1888        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1889        .env("CFG_RELEASE", builder.rust_release())
1890        .env("CFG_VER_MAJOR", parts.next().unwrap())
1891        .env("CFG_VER_MINOR", parts.next().unwrap())
1892        .env("CFG_VER_PATCH", parts.next().unwrap())
1893        .env("CFG_VER_BUILD", "0") // just needed to build
1894        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1895        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1896        .env("CFG_BUILD", target)
1897        .env("CFG_CHANNEL", &builder.config.channel);
1898
1899     if target.contains("windows-gnu") {
1900        cmd.env("CFG_MINGW", "1")
1901           .env("CFG_ABI", "GNU");
1902     } else {
1903        cmd.env("CFG_MINGW", "0")
1904           .env("CFG_ABI", "MSVC");
1905     }
1906
1907     if target.contains("x86_64") {
1908        cmd.env("CFG_PLATFORM", "x64");
1909     } else {
1910        cmd.env("CFG_PLATFORM", "x86");
1911     }
1912 }
1913
1914 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1915 pub struct HashSign;
1916
1917 impl Step for HashSign {
1918     type Output = ();
1919     const ONLY_HOSTS: bool = true;
1920
1921     fn should_run(run: ShouldRun) -> ShouldRun {
1922         run.path("hash-and-sign")
1923     }
1924
1925     fn make_run(run: RunConfig) {
1926         run.builder.ensure(HashSign);
1927     }
1928
1929     fn run(self, builder: &Builder) {
1930         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1931         if builder.config.dry_run {
1932             return;
1933         }
1934         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1935             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1936         });
1937         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1938             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1939         });
1940         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1941             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1942         });
1943         let pass = t!(fs::read_to_string(&file));
1944
1945         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1946
1947         cmd.arg(sign);
1948         cmd.arg(distdir(builder));
1949         cmd.arg(today.trim());
1950         cmd.arg(builder.rust_package_vers());
1951         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1952         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1953         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1954         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1955         cmd.arg(builder.llvm_tools_package_vers());
1956         cmd.arg(builder.lldb_package_vers());
1957         cmd.arg(addr);
1958
1959         builder.create_dir(&distdir(builder));
1960
1961         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1962         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1963         let status = t!(child.wait());
1964         assert!(status.success());
1965     }
1966 }
1967
1968 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1969 // LLVM tools are linked dynamically.
1970 // Note: This function does no yet support Windows but we also don't support
1971 //       linking LLVM tools dynamically on Windows yet.
1972 pub fn maybe_install_llvm_dylib(builder: &Builder,
1973                                 target: Interned<String>,
1974                                 sysroot: &Path) {
1975     let src_libdir = builder
1976         .llvm_out(target)
1977         .join("lib");
1978     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
1979     t!(fs::create_dir_all(&dst_libdir));
1980
1981     if target.contains("apple-darwin") {
1982         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1983         if llvm_dylib_path.exists() {
1984             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1985         }
1986         return
1987     }
1988
1989     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1990     // Since tools link to the latter rather than the former, we have to
1991     // follow the symlink to find out what to distribute.
1992     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1993     if llvm_dylib_path.exists() {
1994         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1995             panic!("dist: Error calling canonicalize path `{}`: {}",
1996                    llvm_dylib_path.display(), e);
1997         });
1998
1999
2000         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2001     }
2002 }
2003
2004 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2005 pub struct LlvmTools {
2006     pub stage: u32,
2007     pub target: Interned<String>,
2008 }
2009
2010 impl Step for LlvmTools {
2011     type Output = Option<PathBuf>;
2012     const ONLY_HOSTS: bool = true;
2013
2014     fn should_run(run: ShouldRun) -> ShouldRun {
2015         run.path("llvm-tools")
2016     }
2017
2018     fn make_run(run: RunConfig) {
2019         run.builder.ensure(LlvmTools {
2020             stage: run.builder.top_stage,
2021             target: run.target,
2022         });
2023     }
2024
2025     fn run(self, builder: &Builder) -> Option<PathBuf> {
2026         let stage = self.stage;
2027         let target = self.target;
2028         assert!(builder.config.extended);
2029
2030         /* run only if llvm-config isn't used */
2031         if let Some(config) = builder.config.target_config.get(&target) {
2032             if let Some(ref _s) = config.llvm_config {
2033                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2034                     stage, target));
2035                 return None;
2036             }
2037         }
2038
2039         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2040         let src = builder.src.join("src/llvm");
2041         let name = pkgname(builder, "llvm-tools");
2042
2043         let tmp = tmpdir(builder);
2044         let image = tmp.join("llvm-tools-image");
2045         drop(fs::remove_dir_all(&image));
2046
2047         // Prepare the image directory
2048         let src_bindir = builder
2049             .llvm_out(target)
2050             .join("bin");
2051         let dst_bindir = image.join("lib/rustlib")
2052             .join(&*target)
2053             .join("bin");
2054         t!(fs::create_dir_all(&dst_bindir));
2055         for tool in LLVM_TOOLS {
2056             let exe = src_bindir.join(exe(tool, &target));
2057             builder.install(&exe, &dst_bindir, 0o755);
2058         }
2059
2060         // Prepare the overlay
2061         let overlay = tmp.join("llvm-tools-overlay");
2062         drop(fs::remove_dir_all(&overlay));
2063         builder.create_dir(&overlay);
2064         builder.install(&src.join("README.txt"), &overlay, 0o644);
2065         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2066         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2067
2068         // Generate the installer tarball
2069         let mut cmd = rust_installer(builder);
2070         cmd.arg("generate")
2071             .arg("--product-name=Rust")
2072             .arg("--rel-manifest-dir=rustlib")
2073             .arg("--success-message=llvm-tools-installed.")
2074             .arg("--image-dir").arg(&image)
2075             .arg("--work-dir").arg(&tmpdir(builder))
2076             .arg("--output-dir").arg(&distdir(builder))
2077             .arg("--non-installed-overlay").arg(&overlay)
2078             .arg(format!("--package-name={}-{}", name, target))
2079             .arg("--legacy-manifest-dirs=rustlib,cargo")
2080             .arg("--component-name=llvm-tools-preview");
2081
2082
2083         builder.run(&mut cmd);
2084         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2085     }
2086 }
2087
2088 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2089 pub struct Lldb {
2090     pub target: Interned<String>,
2091 }
2092
2093 impl Step for Lldb {
2094     type Output = Option<PathBuf>;
2095     const ONLY_HOSTS: bool = true;
2096     const DEFAULT: bool = true;
2097
2098     fn should_run(run: ShouldRun) -> ShouldRun {
2099         run.path("src/tools/lldb")
2100     }
2101
2102     fn make_run(run: RunConfig) {
2103         run.builder.ensure(Lldb {
2104             target: run.target,
2105         });
2106     }
2107
2108     fn run(self, builder: &Builder) -> Option<PathBuf> {
2109         let target = self.target;
2110
2111         if builder.config.dry_run {
2112             return None;
2113         }
2114
2115         let bindir = builder
2116             .llvm_out(target)
2117             .join("bin");
2118         let lldb_exe = bindir.join(exe("lldb", &target));
2119         if !lldb_exe.exists() {
2120             return None;
2121         }
2122
2123         builder.info(&format!("Dist Lldb ({})", target));
2124         let src = builder.src.join("src/tools/lldb");
2125         let name = pkgname(builder, "lldb");
2126
2127         let tmp = tmpdir(builder);
2128         let image = tmp.join("lldb-image");
2129         drop(fs::remove_dir_all(&image));
2130
2131         // Prepare the image directory
2132         let root = image.join("lib/rustlib").join(&*target);
2133         let dst = root.join("bin");
2134         t!(fs::create_dir_all(&dst));
2135         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2136             let exe = bindir.join(exe(program, &target));
2137             builder.install(&exe, &dst, 0o755);
2138         }
2139
2140         // The libraries.
2141         let libdir = builder.llvm_out(target).join("lib");
2142         let dst = root.join("lib");
2143         t!(fs::create_dir_all(&dst));
2144         for entry in t!(fs::read_dir(&libdir)) {
2145             let entry = entry.unwrap();
2146             if let Ok(name) = entry.file_name().into_string() {
2147                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2148                     if t!(entry.file_type()).is_symlink() {
2149                         builder.copy_to_folder(&entry.path(), &dst);
2150                     } else {
2151                        builder.install(&entry.path(), &dst, 0o755);
2152                     }
2153                 }
2154             }
2155         }
2156
2157         // The lldb scripts might be installed in lib/python$version
2158         // or in lib64/python$version.  If lib64 exists, use it;
2159         // otherwise lib.
2160         let libdir = builder.llvm_out(target).join("lib64");
2161         let (libdir, libdir_name) = if libdir.exists() {
2162             (libdir, "lib64")
2163         } else {
2164             (builder.llvm_out(target).join("lib"), "lib")
2165         };
2166         for entry in t!(fs::read_dir(&libdir)) {
2167             let entry = t!(entry);
2168             if let Ok(name) = entry.file_name().into_string() {
2169                 if name.starts_with("python") {
2170                     let dst = root.join(libdir_name)
2171                         .join(entry.file_name());
2172                     t!(fs::create_dir_all(&dst));
2173                     builder.cp_r(&entry.path(), &dst);
2174                     break;
2175                 }
2176             }
2177         }
2178
2179         // Prepare the overlay
2180         let overlay = tmp.join("lldb-overlay");
2181         drop(fs::remove_dir_all(&overlay));
2182         builder.create_dir(&overlay);
2183         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2184         builder.create(&overlay.join("version"), &builder.lldb_vers());
2185
2186         // Generate the installer tarball
2187         let mut cmd = rust_installer(builder);
2188         cmd.arg("generate")
2189             .arg("--product-name=Rust")
2190             .arg("--rel-manifest-dir=rustlib")
2191             .arg("--success-message=lldb-installed.")
2192             .arg("--image-dir").arg(&image)
2193             .arg("--work-dir").arg(&tmpdir(builder))
2194             .arg("--output-dir").arg(&distdir(builder))
2195             .arg("--non-installed-overlay").arg(&overlay)
2196             .arg(format!("--package-name={}-{}", name, target))
2197             .arg("--legacy-manifest-dirs=rustlib,cargo")
2198             .arg("--component-name=lldb-preview");
2199
2200
2201         builder.run(&mut cmd);
2202         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2203     }
2204 }