src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Compiler, Mode, LLVM_TOOLS};
  30 use channel;
  31 use util::{libdir, is_dylib, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/dlmalloc",
 861             "src/liballoc",
 862             "src/libbacktrace",
 863             "src/libcompiler_builtins",
 864             "src/libcore",
 865             "src/liblibc",
 866             "src/libpanic_abort",
 867             "src/libpanic_unwind",
 868             "src/librustc_asan",
 869             "src/librustc_lsan",
 870             "src/librustc_msan",
 871             "src/librustc_tsan",
 872             "src/libstd",
 873             "src/libunwind",
 874             "src/rustc/compiler_builtins_shim",
 875             "src/rustc/libc_shim",
 876             "src/rustc/dlmalloc_shim",
 877             "src/libtest",
 878             "src/libterm",
 879             "src/libprofiler_builtins",
 880             "src/stdsimd",
 881             "src/libproc_macro",
 882         ];
 883         let std_src_dirs_exclude = [
 884             "src/libcompiler_builtins/compiler-rt/test",
 885         ];
 886
 887         copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 888         for file in src_files.iter() {
 889             builder.copy(&builder.src.join(file), &dst_src.join(file));
 890         }
 891
 892         // Create source tarball in rust-installer format
 893         let mut cmd = rust_installer(builder);
 894         cmd.arg("generate")
 895            .arg("--product-name=Rust")
 896            .arg("--rel-manifest-dir=rustlib")
 897            .arg("--success-message=Awesome-Source.")
 898            .arg("--image-dir").arg(&image)
 899            .arg("--work-dir").arg(&tmpdir(builder))
 900            .arg("--output-dir").arg(&distdir(builder))
 901            .arg(format!("--package-name={}", name))
 902            .arg("--component-name=rust-src")
 903            .arg("--legacy-manifest-dirs=rustlib,cargo");
 904         builder.run(&mut cmd);
 905
 906         builder.remove_dir(&image);
 907         distdir(builder).join(&format!("{}.tar.gz", name))
 908     }
 909 }
 910
 911 const CARGO_VENDOR_VERSION: &str = "0.1.19";
 912
 913 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 914 pub struct PlainSourceTarball;
 915
 916 impl Step for PlainSourceTarball {
 917     /// Produces the location of the tarball generated
 918     type Output = PathBuf;
 919     const DEFAULT: bool = true;
 920     const ONLY_HOSTS: bool = true;
 921
 922     fn should_run(run: ShouldRun) -> ShouldRun {
 923         let builder = run.builder;
 924         run.path("src").default_condition(builder.config.rust_dist_src)
 925     }
 926
 927     fn make_run(run: RunConfig) {
 928         run.builder.ensure(PlainSourceTarball);
 929     }
 930
 931     /// Creates the plain source tarball
 932     fn run(self, builder: &Builder) -> PathBuf {
 933         builder.info("Create plain source tarball");
 934
 935         // Make sure that the root folder of tarball has the correct name
 936         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 937         let plain_dst_src = tmpdir(builder).join(&plain_name);
 938         let _ = fs::remove_dir_all(&plain_dst_src);
 939         t!(fs::create_dir_all(&plain_dst_src));
 940
 941         // This is the set of root paths which will become part of the source package
 942         let src_files = [
 943             "COPYRIGHT",
 944             "LICENSE-APACHE",
 945             "LICENSE-MIT",
 946             "CONTRIBUTING.md",
 947             "README.md",
 948             "RELEASES.md",
 949             "configure",
 950             "x.py",
 951             "config.toml.example",
 952             "Cargo.toml",
 953             "Cargo.lock",
 954         ];
 955         let src_dirs = [
 956             "src",
 957         ];
 958
 959         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 960
 961         // Copy the files normally
 962         for item in &src_files {
 963             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 964         }
 965
 966         // Create the version file
 967         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 968         if let Some(sha) = builder.rust_sha() {
 969             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 970         }
 971
 972         // If we're building from git sources, we need to vendor a complete distribution.
 973         if builder.rust_info.is_git() {
 974             // Get cargo-vendor installed, if it isn't already.
 975             let mut has_cargo_vendor = false;
 976             let mut cmd = Command::new(&builder.initial_cargo);
 977             for line in output(cmd.arg("install").arg("--list")).lines() {
 978                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 979             }
 980             if !has_cargo_vendor {
 981                 let mut cmd = builder.cargo(
 982                     builder.compiler(0, builder.config.build),
 983                     Mode::ToolBootstrap,
 984                     builder.config.build,
 985                     "install"
 986                 );
 987                 cmd.arg("--force")
 988                    .arg("--debug")
 989                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 990                    .arg("cargo-vendor");
 991                 builder.run(&mut cmd);
 992             }
 993
 994             // Vendor all Cargo dependencies
 995             let mut cmd = Command::new(&builder.initial_cargo);
 996             cmd.arg("vendor")
 997                .current_dir(&plain_dst_src);
 998             builder.run(&mut cmd);
 999         }
1000
1001         // Create plain source tarball
1002         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1003         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1004         tarball.set_extension(""); // strip .gz
1005         tarball.set_extension(""); // strip .tar
1006         if let Some(dir) = tarball.parent() {
1007             builder.create_dir(&dir);
1008         }
1009         builder.info("running installer");
1010         let mut cmd = rust_installer(builder);
1011         cmd.arg("tarball")
1012            .arg("--input").arg(&plain_name)
1013            .arg("--output").arg(&tarball)
1014            .arg("--work-dir=.")
1015            .current_dir(tmpdir(builder));
1016         builder.run(&mut cmd);
1017         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1018     }
1019 }
1020
1021 // We have to run a few shell scripts, which choke quite a bit on both `\`
1022 // characters and on `C:\` paths, so normalize both of them away.
1023 pub fn sanitize_sh(path: &Path) -> String {
1024     let path = path.to_str().unwrap().replace("\\", "/");
1025     return change_drive(&path).unwrap_or(path);
1026
1027     fn change_drive(s: &str) -> Option<String> {
1028         let mut ch = s.chars();
1029         let drive = ch.next().unwrap_or('C');
1030         if ch.next() != Some(':') {
1031             return None
1032         }
1033         if ch.next() != Some('/') {
1034             return None
1035         }
1036         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1037     }
1038 }
1039
1040 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1041 pub struct Cargo {
1042     pub stage: u32,
1043     pub target: Interned<String>,
1044 }
1045
1046 impl Step for Cargo {
1047     type Output = PathBuf;
1048     const ONLY_HOSTS: bool = true;
1049
1050     fn should_run(run: ShouldRun) -> ShouldRun {
1051         run.path("cargo")
1052     }
1053
1054     fn make_run(run: RunConfig) {
1055         run.builder.ensure(Cargo {
1056             stage: run.builder.top_stage,
1057             target: run.target,
1058         });
1059     }
1060
1061     fn run(self, builder: &Builder) -> PathBuf {
1062         let stage = self.stage;
1063         let target = self.target;
1064
1065         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1066         let src = builder.src.join("src/tools/cargo");
1067         let etc = src.join("src/etc");
1068         let release_num = builder.release_num("cargo");
1069         let name = pkgname(builder, "cargo");
1070         let version = builder.cargo_info.version(builder, &release_num);
1071
1072         let tmp = tmpdir(builder);
1073         let image = tmp.join("cargo-image");
1074         drop(fs::remove_dir_all(&image));
1075         builder.create_dir(&image);
1076
1077         // Prepare the image directory
1078         builder.create_dir(&image.join("share/zsh/site-functions"));
1079         builder.create_dir(&image.join("etc/bash_completion.d"));
1080         let cargo = builder.ensure(tool::Cargo {
1081             compiler: builder.compiler(stage, builder.config.build),
1082             target
1083         });
1084         builder.install(&cargo, &image.join("bin"), 0o755);
1085         for man in t!(etc.join("man").read_dir()) {
1086             let man = t!(man);
1087             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1088         }
1089         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1090         builder.copy(&etc.join("cargo.bashcomp.sh"),
1091              &image.join("etc/bash_completion.d/cargo"));
1092         let doc = image.join("share/doc/cargo");
1093         builder.install(&src.join("README.md"), &doc, 0o644);
1094         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1095         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1096         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1097
1098         // Prepare the overlay
1099         let overlay = tmp.join("cargo-overlay");
1100         drop(fs::remove_dir_all(&overlay));
1101         builder.create_dir(&overlay);
1102         builder.install(&src.join("README.md"), &overlay, 0o644);
1103         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1104         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1105         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1106         builder.create(&overlay.join("version"), &version);
1107
1108         // Generate the installer tarball
1109         let mut cmd = rust_installer(builder);
1110         cmd.arg("generate")
1111            .arg("--product-name=Rust")
1112            .arg("--rel-manifest-dir=rustlib")
1113            .arg("--success-message=Rust-is-ready-to-roll.")
1114            .arg("--image-dir").arg(&image)
1115            .arg("--work-dir").arg(&tmpdir(builder))
1116            .arg("--output-dir").arg(&distdir(builder))
1117            .arg("--non-installed-overlay").arg(&overlay)
1118            .arg(format!("--package-name={}-{}", name, target))
1119            .arg("--component-name=cargo")
1120            .arg("--legacy-manifest-dirs=rustlib,cargo");
1121         builder.run(&mut cmd);
1122         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1123     }
1124 }
1125
1126 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1127 pub struct Rls {
1128     pub stage: u32,
1129     pub target: Interned<String>,
1130 }
1131
1132 impl Step for Rls {
1133     type Output = Option<PathBuf>;
1134     const ONLY_HOSTS: bool = true;
1135
1136     fn should_run(run: ShouldRun) -> ShouldRun {
1137         run.path("rls")
1138     }
1139
1140     fn make_run(run: RunConfig) {
1141         run.builder.ensure(Rls {
1142             stage: run.builder.top_stage,
1143             target: run.target,
1144         });
1145     }
1146
1147     fn run(self, builder: &Builder) -> Option<PathBuf> {
1148         let stage = self.stage;
1149         let target = self.target;
1150         assert!(builder.config.extended);
1151
1152         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1153         let src = builder.src.join("src/tools/rls");
1154         let release_num = builder.release_num("rls");
1155         let name = pkgname(builder, "rls");
1156         let version = builder.rls_info.version(builder, &release_num);
1157
1158         let tmp = tmpdir(builder);
1159         let image = tmp.join("rls-image");
1160         drop(fs::remove_dir_all(&image));
1161         t!(fs::create_dir_all(&image));
1162
1163         // Prepare the image directory
1164         // We expect RLS to build, because we've exited this step above if tool
1165         // state for RLS isn't testing.
1166         let rls = builder.ensure(tool::Rls {
1167             compiler: builder.compiler(stage, builder.config.build),
1168             target, extra_features: Vec::new()
1169         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1170
1171         builder.install(&rls, &image.join("bin"), 0o755);
1172         let doc = image.join("share/doc/rls");
1173         builder.install(&src.join("README.md"), &doc, 0o644);
1174         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1175         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1176
1177         // Prepare the overlay
1178         let overlay = tmp.join("rls-overlay");
1179         drop(fs::remove_dir_all(&overlay));
1180         t!(fs::create_dir_all(&overlay));
1181         builder.install(&src.join("README.md"), &overlay, 0o644);
1182         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1183         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1184         builder.create(&overlay.join("version"), &version);
1185
1186         // Generate the installer tarball
1187         let mut cmd = rust_installer(builder);
1188         cmd.arg("generate")
1189            .arg("--product-name=Rust")
1190            .arg("--rel-manifest-dir=rustlib")
1191            .arg("--success-message=RLS-ready-to-serve.")
1192            .arg("--image-dir").arg(&image)
1193            .arg("--work-dir").arg(&tmpdir(builder))
1194            .arg("--output-dir").arg(&distdir(builder))
1195            .arg("--non-installed-overlay").arg(&overlay)
1196            .arg(format!("--package-name={}-{}", name, target))
1197            .arg("--legacy-manifest-dirs=rustlib,cargo")
1198            .arg("--component-name=rls-preview");
1199
1200         builder.run(&mut cmd);
1201         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1202     }
1203 }
1204
1205 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1206 pub struct Clippy {
1207     pub stage: u32,
1208     pub target: Interned<String>,
1209 }
1210
1211 impl Step for Clippy {
1212     type Output = Option<PathBuf>;
1213     const ONLY_HOSTS: bool = true;
1214
1215     fn should_run(run: ShouldRun) -> ShouldRun {
1216         run.path("clippy")
1217     }
1218
1219     fn make_run(run: RunConfig) {
1220         run.builder.ensure(Clippy {
1221             stage: run.builder.top_stage,
1222             target: run.target,
1223         });
1224     }
1225
1226     fn run(self, builder: &Builder) -> Option<PathBuf> {
1227         let stage = self.stage;
1228         let target = self.target;
1229         assert!(builder.config.extended);
1230
1231         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1232         let src = builder.src.join("src/tools/clippy");
1233         let release_num = builder.release_num("clippy");
1234         let name = pkgname(builder, "clippy");
1235         let version = builder.clippy_info.version(builder, &release_num);
1236
1237         let tmp = tmpdir(builder);
1238         let image = tmp.join("clippy-image");
1239         drop(fs::remove_dir_all(&image));
1240         builder.create_dir(&image);
1241
1242         // Prepare the image directory
1243         // We expect clippy to build, because we've exited this step above if tool
1244         // state for clippy isn't testing.
1245         let clippy = builder.ensure(tool::Clippy {
1246             compiler: builder.compiler(stage, builder.config.build),
1247             target, extra_features: Vec::new()
1248         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1249         let cargoclippy = builder.ensure(tool::CargoClippy {
1250             compiler: builder.compiler(stage, builder.config.build),
1251             target, extra_features: Vec::new()
1252         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1253
1254         builder.install(&clippy, &image.join("bin"), 0o755);
1255         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1256         let doc = image.join("share/doc/clippy");
1257         builder.install(&src.join("README.md"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1259         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1260
1261         // Prepare the overlay
1262         let overlay = tmp.join("clippy-overlay");
1263         drop(fs::remove_dir_all(&overlay));
1264         t!(fs::create_dir_all(&overlay));
1265         builder.install(&src.join("README.md"), &overlay, 0o644);
1266         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1267         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1268         builder.create(&overlay.join("version"), &version);
1269
1270         // Generate the installer tarball
1271         let mut cmd = rust_installer(builder);
1272         cmd.arg("generate")
1273            .arg("--product-name=Rust")
1274            .arg("--rel-manifest-dir=rustlib")
1275            .arg("--success-message=clippy-ready-to-serve.")
1276            .arg("--image-dir").arg(&image)
1277            .arg("--work-dir").arg(&tmpdir(builder))
1278            .arg("--output-dir").arg(&distdir(builder))
1279            .arg("--non-installed-overlay").arg(&overlay)
1280            .arg(format!("--package-name={}-{}", name, target))
1281            .arg("--legacy-manifest-dirs=rustlib,cargo")
1282            .arg("--component-name=clippy-preview");
1283
1284         builder.run(&mut cmd);
1285         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1286     }
1287 }
1288
1289 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1290 pub struct Rustfmt {
1291     pub stage: u32,
1292     pub target: Interned<String>,
1293 }
1294
1295 impl Step for Rustfmt {
1296     type Output = Option<PathBuf>;
1297     const ONLY_HOSTS: bool = true;
1298
1299     fn should_run(run: ShouldRun) -> ShouldRun {
1300         run.path("rustfmt")
1301     }
1302
1303     fn make_run(run: RunConfig) {
1304         run.builder.ensure(Rustfmt {
1305             stage: run.builder.top_stage,
1306             target: run.target,
1307         });
1308     }
1309
1310     fn run(self, builder: &Builder) -> Option<PathBuf> {
1311         let stage = self.stage;
1312         let target = self.target;
1313
1314         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1315         let src = builder.src.join("src/tools/rustfmt");
1316         let release_num = builder.release_num("rustfmt");
1317         let name = pkgname(builder, "rustfmt");
1318         let version = builder.rustfmt_info.version(builder, &release_num);
1319
1320         let tmp = tmpdir(builder);
1321         let image = tmp.join("rustfmt-image");
1322         drop(fs::remove_dir_all(&image));
1323         builder.create_dir(&image);
1324
1325         // Prepare the image directory
1326         let rustfmt = builder.ensure(tool::Rustfmt {
1327             compiler: builder.compiler(stage, builder.config.build),
1328             target, extra_features: Vec::new()
1329         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1330         let cargofmt = builder.ensure(tool::Cargofmt {
1331             compiler: builder.compiler(stage, builder.config.build),
1332             target, extra_features: Vec::new()
1333         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1334
1335         builder.install(&rustfmt, &image.join("bin"), 0o755);
1336         builder.install(&cargofmt, &image.join("bin"), 0o755);
1337         let doc = image.join("share/doc/rustfmt");
1338         builder.install(&src.join("README.md"), &doc, 0o644);
1339         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1340         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1341
1342         // Prepare the overlay
1343         let overlay = tmp.join("rustfmt-overlay");
1344         drop(fs::remove_dir_all(&overlay));
1345         builder.create_dir(&overlay);
1346         builder.install(&src.join("README.md"), &overlay, 0o644);
1347         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1348         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1349         builder.create(&overlay.join("version"), &version);
1350
1351         // Generate the installer tarball
1352         let mut cmd = rust_installer(builder);
1353         cmd.arg("generate")
1354            .arg("--product-name=Rust")
1355            .arg("--rel-manifest-dir=rustlib")
1356            .arg("--success-message=rustfmt-ready-to-fmt.")
1357            .arg("--image-dir").arg(&image)
1358            .arg("--work-dir").arg(&tmpdir(builder))
1359            .arg("--output-dir").arg(&distdir(builder))
1360            .arg("--non-installed-overlay").arg(&overlay)
1361            .arg(format!("--package-name={}-{}", name, target))
1362            .arg("--legacy-manifest-dirs=rustlib,cargo")
1363            .arg("--component-name=rustfmt-preview");
1364
1365         builder.run(&mut cmd);
1366         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1367     }
1368 }
1369
1370 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1371 pub struct Extended {
1372     stage: u32,
1373     host: Interned<String>,
1374     target: Interned<String>,
1375 }
1376
1377 impl Step for Extended {
1378     type Output = ();
1379     const DEFAULT: bool = true;
1380     const ONLY_HOSTS: bool = true;
1381
1382     fn should_run(run: ShouldRun) -> ShouldRun {
1383         let builder = run.builder;
1384         run.path("extended").default_condition(builder.config.extended)
1385     }
1386
1387     fn make_run(run: RunConfig) {
1388         run.builder.ensure(Extended {
1389             stage: run.builder.top_stage,
1390             host: run.builder.config.build,
1391             target: run.target,
1392         });
1393     }
1394
1395     /// Creates a combined installer for the specified target in the provided stage.
1396     fn run(self, builder: &Builder) {
1397         let stage = self.stage;
1398         let target = self.target;
1399
1400         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1401
1402         let rustc_installer = builder.ensure(Rustc {
1403             compiler: builder.compiler(stage, target),
1404         });
1405         let cargo_installer = builder.ensure(Cargo { stage, target });
1406         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1407         let rls_installer = builder.ensure(Rls { stage, target });
1408         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1409         let clippy_installer = builder.ensure(Clippy { stage, target });
1410         let lldb_installer = builder.ensure(Lldb { target });
1411         let mingw_installer = builder.ensure(Mingw { host: target });
1412         let analysis_installer = builder.ensure(Analysis {
1413             compiler: builder.compiler(stage, self.host),
1414             target
1415         });
1416
1417         let docs_installer = builder.ensure(Docs { stage, host: target, });
1418         let std_installer = builder.ensure(Std {
1419             compiler: builder.compiler(stage, self.host),
1420             target,
1421         });
1422
1423         let tmp = tmpdir(builder);
1424         let overlay = tmp.join("extended-overlay");
1425         let etc = builder.src.join("src/etc/installer");
1426         let work = tmp.join("work");
1427
1428         let _ = fs::remove_dir_all(&overlay);
1429         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1430         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1431         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1432         let version = builder.rust_version();
1433         builder.create(&overlay.join("version"), &version);
1434         if let Some(sha) = builder.rust_sha() {
1435             builder.create(&overlay.join("git-commit-hash"), &sha);
1436         }
1437         builder.install(&etc.join("README.md"), &overlay, 0o644);
1438
1439         // When rust-std package split from rustc, we needed to ensure that during
1440         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1441         // the std files during uninstall. To do this ensure that rustc comes
1442         // before rust-std in the list below.
1443         let mut tarballs = Vec::new();
1444         tarballs.push(rustc_installer);
1445         tarballs.push(cargo_installer);
1446         tarballs.extend(rls_installer.clone());
1447         tarballs.extend(clippy_installer.clone());
1448         tarballs.extend(rustfmt_installer.clone());
1449         tarballs.extend(llvm_tools_installer);
1450         tarballs.extend(lldb_installer);
1451         tarballs.push(analysis_installer);
1452         tarballs.push(std_installer);
1453         if builder.config.docs {
1454             tarballs.push(docs_installer);
1455         }
1456         if target.contains("pc-windows-gnu") {
1457             tarballs.push(mingw_installer.unwrap());
1458         }
1459         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1460         for tarball in &tarballs[1..] {
1461             input_tarballs.push(",");
1462             input_tarballs.push(tarball);
1463         }
1464
1465         let mut cmd = rust_installer(builder);
1466         cmd.arg("combine")
1467             .arg("--product-name=Rust")
1468             .arg("--rel-manifest-dir=rustlib")
1469             .arg("--success-message=Rust-is-ready-to-roll.")
1470             .arg("--work-dir").arg(&work)
1471             .arg("--output-dir").arg(&distdir(builder))
1472             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1473             .arg("--legacy-manifest-dirs=rustlib,cargo")
1474             .arg("--input-tarballs").arg(input_tarballs)
1475             .arg("--non-installed-overlay").arg(&overlay);
1476         builder.run(&mut cmd);
1477
1478         let mut license = String::new();
1479         license += &builder.read(&builder.src.join("COPYRIGHT"));
1480         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1481         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1482         license.push_str("\n");
1483         license.push_str("\n");
1484
1485         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1486         let mut rtf = rtf.to_string();
1487         rtf.push_str("\n");
1488         for line in license.lines() {
1489             rtf.push_str(line);
1490             rtf.push_str("\\line ");
1491         }
1492         rtf.push_str("}");
1493
1494         fn filter(contents: &str, marker: &str) -> String {
1495             let start = format!("tool-{}-start", marker);
1496             let end = format!("tool-{}-end", marker);
1497             let mut lines = Vec::new();
1498             let mut omitted = false;
1499             for line in contents.lines() {
1500                 if line.contains(&start) {
1501                     omitted = true;
1502                 } else if line.contains(&end) {
1503                     omitted = false;
1504                 } else if !omitted {
1505                     lines.push(line);
1506                 }
1507             }
1508
1509             lines.join("\n")
1510         }
1511
1512         let xform = |p: &Path| {
1513             let mut contents = String::new();
1514             t!(t!(File::open(p)).read_to_string(&mut contents));
1515             if rls_installer.is_none() {
1516                 contents = filter(&contents, "rls");
1517             }
1518             if clippy_installer.is_none() {
1519                 contents = filter(&contents, "clippy");
1520             }
1521             if rustfmt_installer.is_none() {
1522                 contents = filter(&contents, "rustfmt");
1523             }
1524             let ret = tmp.join(p.file_name().unwrap());
1525             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1526             return ret
1527         };
1528
1529         if target.contains("apple-darwin") {
1530             let pkg = tmp.join("pkg");
1531             let _ = fs::remove_dir_all(&pkg);
1532
1533             let pkgbuild = |component: &str| {
1534                 let mut cmd = Command::new("pkgbuild");
1535                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1536                     .arg("--scripts").arg(pkg.join(component))
1537                     .arg("--nopayload")
1538                     .arg(pkg.join(component).with_extension("pkg"));
1539                 builder.run(&mut cmd);
1540             };
1541
1542             let prepare = |name: &str| {
1543                 builder.create_dir(&pkg.join(name));
1544                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1545                         &pkg.join(name));
1546                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1547                 pkgbuild(name);
1548             };
1549             prepare("rustc");
1550             prepare("cargo");
1551             prepare("rust-docs");
1552             prepare("rust-std");
1553             prepare("rust-analysis");
1554
1555             if rls_installer.is_some() {
1556                 prepare("rls");
1557             }
1558             if clippy_installer.is_some() {
1559                 prepare("clippy");
1560             }
1561
1562             // create an 'uninstall' package
1563             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1564             pkgbuild("uninstall");
1565
1566             builder.create_dir(&pkg.join("res"));
1567             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1568             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1569             let mut cmd = Command::new("productbuild");
1570             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1571                 .arg("--resources").arg(pkg.join("res"))
1572                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1573                                                     pkgname(builder, "rust"),
1574                                                     target)))
1575                 .arg("--package-path").arg(&pkg);
1576             builder.run(&mut cmd);
1577         }
1578
1579         if target.contains("windows") {
1580             let exe = tmp.join("exe");
1581             let _ = fs::remove_dir_all(&exe);
1582
1583             let prepare = |name: &str| {
1584                 builder.create_dir(&exe.join(name));
1585                 let dir = if name == "rust-std" || name == "rust-analysis" {
1586                     format!("{}-{}", name, target)
1587                 } else if name == "rls" {
1588                     "rls-preview".to_string()
1589                 } else if name == "clippy" {
1590                     "clippy-preview".to_string()
1591                 } else {
1592                     name.to_string()
1593                 };
1594                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1595                             .join(dir),
1596                         &exe.join(name));
1597                 builder.remove(&exe.join(name).join("manifest.in"));
1598             };
1599             prepare("rustc");
1600             prepare("cargo");
1601             prepare("rust-analysis");
1602             prepare("rust-docs");
1603             prepare("rust-std");
1604             if rls_installer.is_some() {
1605                 prepare("rls");
1606             }
1607             if clippy_installer.is_some() {
1608                 prepare("clippy");
1609             }
1610             if target.contains("windows-gnu") {
1611                 prepare("rust-mingw");
1612             }
1613
1614             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1615             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1616             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1617             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1618             builder.create(&exe.join("LICENSE.txt"), &license);
1619
1620             // Generate exe installer
1621             let mut cmd = Command::new("iscc");
1622             cmd.arg("rust.iss")
1623                 .current_dir(&exe);
1624             if target.contains("windows-gnu") {
1625                 cmd.arg("/dMINGW");
1626             }
1627             add_env(builder, &mut cmd, target);
1628             builder.run(&mut cmd);
1629             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1630                     &distdir(builder),
1631                     0o755);
1632
1633             // Generate msi installer
1634             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1635             let heat = wix.join("bin/heat.exe");
1636             let candle = wix.join("bin/candle.exe");
1637             let light = wix.join("bin/light.exe");
1638
1639             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1640             builder.run(Command::new(&heat)
1641                             .current_dir(&exe)
1642                             .arg("dir")
1643                             .arg("rustc")
1644                             .args(&heat_flags)
1645                             .arg("-cg").arg("RustcGroup")
1646                             .arg("-dr").arg("Rustc")
1647                             .arg("-var").arg("var.RustcDir")
1648                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1649             builder.run(Command::new(&heat)
1650                             .current_dir(&exe)
1651                             .arg("dir")
1652                             .arg("rust-docs")
1653                             .args(&heat_flags)
1654                             .arg("-cg").arg("DocsGroup")
1655                             .arg("-dr").arg("Docs")
1656                             .arg("-var").arg("var.DocsDir")
1657                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1658                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1659             builder.run(Command::new(&heat)
1660                             .current_dir(&exe)
1661                             .arg("dir")
1662                             .arg("cargo")
1663                             .args(&heat_flags)
1664                             .arg("-cg").arg("CargoGroup")
1665                             .arg("-dr").arg("Cargo")
1666                             .arg("-var").arg("var.CargoDir")
1667                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1668                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1669             builder.run(Command::new(&heat)
1670                             .current_dir(&exe)
1671                             .arg("dir")
1672                             .arg("rust-std")
1673                             .args(&heat_flags)
1674                             .arg("-cg").arg("StdGroup")
1675                             .arg("-dr").arg("Std")
1676                             .arg("-var").arg("var.StdDir")
1677                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1678             if rls_installer.is_some() {
1679                 builder.run(Command::new(&heat)
1680                                 .current_dir(&exe)
1681                                 .arg("dir")
1682                                 .arg("rls")
1683                                 .args(&heat_flags)
1684                                 .arg("-cg").arg("RlsGroup")
1685                                 .arg("-dr").arg("Rls")
1686                                 .arg("-var").arg("var.RlsDir")
1687                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1688                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1689             }
1690             if clippy_installer.is_some() {
1691                 builder.run(Command::new(&heat)
1692                                 .current_dir(&exe)
1693                                 .arg("dir")
1694                                 .arg("clippy")
1695                                 .args(&heat_flags)
1696                                 .arg("-cg").arg("ClippyGroup")
1697                                 .arg("-dr").arg("Clippy")
1698                                 .arg("-var").arg("var.ClippyDir")
1699                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1700                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1701             }
1702             builder.run(Command::new(&heat)
1703                             .current_dir(&exe)
1704                             .arg("dir")
1705                             .arg("rust-analysis")
1706                             .args(&heat_flags)
1707                             .arg("-cg").arg("AnalysisGroup")
1708                             .arg("-dr").arg("Analysis")
1709                             .arg("-var").arg("var.AnalysisDir")
1710                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1711                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1712             if target.contains("windows-gnu") {
1713                 builder.run(Command::new(&heat)
1714                                 .current_dir(&exe)
1715                                 .arg("dir")
1716                                 .arg("rust-mingw")
1717                                 .args(&heat_flags)
1718                                 .arg("-cg").arg("GccGroup")
1719                                 .arg("-dr").arg("Gcc")
1720                                 .arg("-var").arg("var.GccDir")
1721                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1722             }
1723
1724             let candle = |input: &Path| {
1725                 let output = exe.join(input.file_stem().unwrap())
1726                                 .with_extension("wixobj");
1727                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1728                 let mut cmd = Command::new(&candle);
1729                 cmd.current_dir(&exe)
1730                     .arg("-nologo")
1731                     .arg("-dRustcDir=rustc")
1732                     .arg("-dDocsDir=rust-docs")
1733                     .arg("-dCargoDir=cargo")
1734                     .arg("-dStdDir=rust-std")
1735                     .arg("-dAnalysisDir=rust-analysis")
1736                     .arg("-arch").arg(&arch)
1737                     .arg("-out").arg(&output)
1738                     .arg(&input);
1739                 add_env(builder, &mut cmd, target);
1740
1741                 if rls_installer.is_some() {
1742                     cmd.arg("-dRlsDir=rls");
1743                 }
1744                 if clippy_installer.is_some() {
1745                     cmd.arg("-dClippyDir=clippy");
1746                 }
1747                 if target.contains("windows-gnu") {
1748                     cmd.arg("-dGccDir=rust-mingw");
1749                 }
1750                 builder.run(&mut cmd);
1751             };
1752             candle(&xform(&etc.join("msi/rust.wxs")));
1753             candle(&etc.join("msi/ui.wxs"));
1754             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1755             candle("RustcGroup.wxs".as_ref());
1756             candle("DocsGroup.wxs".as_ref());
1757             candle("CargoGroup.wxs".as_ref());
1758             candle("StdGroup.wxs".as_ref());
1759             if rls_installer.is_some() {
1760                 candle("RlsGroup.wxs".as_ref());
1761             }
1762             if clippy_installer.is_some() {
1763                 candle("ClippyGroup.wxs".as_ref());
1764             }
1765             candle("AnalysisGroup.wxs".as_ref());
1766
1767             if target.contains("windows-gnu") {
1768                 candle("GccGroup.wxs".as_ref());
1769             }
1770
1771             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1772             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1773             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1774
1775             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1776             let mut cmd = Command::new(&light);
1777             cmd.arg("-nologo")
1778                 .arg("-ext").arg("WixUIExtension")
1779                 .arg("-ext").arg("WixUtilExtension")
1780                 .arg("-out").arg(exe.join(&filename))
1781                 .arg("rust.wixobj")
1782                 .arg("ui.wixobj")
1783                 .arg("rustwelcomedlg.wixobj")
1784                 .arg("RustcGroup.wixobj")
1785                 .arg("DocsGroup.wixobj")
1786                 .arg("CargoGroup.wixobj")
1787                 .arg("StdGroup.wixobj")
1788                 .arg("AnalysisGroup.wixobj")
1789                 .current_dir(&exe);
1790
1791             if rls_installer.is_some() {
1792                 cmd.arg("RlsGroup.wixobj");
1793             }
1794             if clippy_installer.is_some() {
1795                 cmd.arg("ClippyGroup.wixobj");
1796             }
1797
1798             if target.contains("windows-gnu") {
1799                 cmd.arg("GccGroup.wixobj");
1800             }
1801             // ICE57 wrongly complains about the shortcuts
1802             cmd.arg("-sice:ICE57");
1803
1804             builder.run(&mut cmd);
1805
1806             if !builder.config.dry_run {
1807                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1808             }
1809         }
1810     }
1811 }
1812
1813 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1814     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1815     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1816        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1817        .env("CFG_RELEASE", builder.rust_release())
1818        .env("CFG_VER_MAJOR", parts.next().unwrap())
1819        .env("CFG_VER_MINOR", parts.next().unwrap())
1820        .env("CFG_VER_PATCH", parts.next().unwrap())
1821        .env("CFG_VER_BUILD", "0") // just needed to build
1822        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1823        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1824        .env("CFG_BUILD", target)
1825        .env("CFG_CHANNEL", &builder.config.channel);
1826
1827     if target.contains("windows-gnu") {
1828        cmd.env("CFG_MINGW", "1")
1829           .env("CFG_ABI", "GNU");
1830     } else {
1831        cmd.env("CFG_MINGW", "0")
1832           .env("CFG_ABI", "MSVC");
1833     }
1834
1835     if target.contains("x86_64") {
1836        cmd.env("CFG_PLATFORM", "x64");
1837     } else {
1838        cmd.env("CFG_PLATFORM", "x86");
1839     }
1840 }
1841
1842 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1843 pub struct HashSign;
1844
1845 impl Step for HashSign {
1846     type Output = ();
1847     const ONLY_HOSTS: bool = true;
1848
1849     fn should_run(run: ShouldRun) -> ShouldRun {
1850         run.path("hash-and-sign")
1851     }
1852
1853     fn make_run(run: RunConfig) {
1854         run.builder.ensure(HashSign);
1855     }
1856
1857     fn run(self, builder: &Builder) {
1858         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1859         if builder.config.dry_run {
1860             return;
1861         }
1862         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1863             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1864         });
1865         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1866             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1867         });
1868         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1869             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1870         });
1871         let mut pass = String::new();
1872         t!(t!(File::open(&file)).read_to_string(&mut pass));
1873
1874         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1875
1876         cmd.arg(sign);
1877         cmd.arg(distdir(builder));
1878         cmd.arg(today.trim());
1879         cmd.arg(builder.rust_package_vers());
1880         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1881         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1882         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1883         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1884         cmd.arg(builder.llvm_tools_package_vers());
1885         cmd.arg(builder.lldb_package_vers());
1886         cmd.arg(addr);
1887
1888         builder.create_dir(&distdir(builder));
1889
1890         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1891         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1892         let status = t!(child.wait());
1893         assert!(status.success());
1894     }
1895 }
1896
1897 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1898 // LLVM tools are linked dynamically.
1899 // Note: This function does no yet support Windows but we also don't support
1900 //       linking LLVM tools dynamically on Windows yet.
1901 fn maybe_install_llvm_dylib(builder: &Builder,
1902                             target: Interned<String>,
1903                             image: &Path) {
1904     let src_libdir = builder
1905         .llvm_out(target)
1906         .join("lib");
1907     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1908     t!(fs::create_dir_all(&dst_libdir));
1909
1910     if target.contains("apple-darwin") {
1911         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1912         if llvm_dylib_path.exists() {
1913             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1914         }
1915         return
1916     }
1917
1918     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1919     // Since tools link to the latter rather than the former, we have to
1920     // follow the symlink to find out what to distribute.
1921     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1922     if llvm_dylib_path.exists() {
1923         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1924             panic!("dist: Error calling canonicalize path `{}`: {}",
1925                    llvm_dylib_path.display(), e);
1926         });
1927
1928
1929         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1930     }
1931 }
1932
1933 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1934 pub struct LlvmTools {
1935     pub stage: u32,
1936     pub target: Interned<String>,
1937 }
1938
1939 impl Step for LlvmTools {
1940     type Output = Option<PathBuf>;
1941     const ONLY_HOSTS: bool = true;
1942
1943     fn should_run(run: ShouldRun) -> ShouldRun {
1944         run.path("llvm-tools")
1945     }
1946
1947     fn make_run(run: RunConfig) {
1948         run.builder.ensure(LlvmTools {
1949             stage: run.builder.top_stage,
1950             target: run.target,
1951         });
1952     }
1953
1954     fn run(self, builder: &Builder) -> Option<PathBuf> {
1955         let stage = self.stage;
1956         let target = self.target;
1957         assert!(builder.config.extended);
1958
1959         /* run only if llvm-config isn't used */
1960         if let Some(config) = builder.config.target_config.get(&target) {
1961             if let Some(ref _s) = config.llvm_config {
1962                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1963                     stage, target));
1964                 return None;
1965             }
1966         }
1967
1968         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1969         let src = builder.src.join("src/llvm");
1970         let name = pkgname(builder, "llvm-tools");
1971
1972         let tmp = tmpdir(builder);
1973         let image = tmp.join("llvm-tools-image");
1974         drop(fs::remove_dir_all(&image));
1975
1976         // Prepare the image directory
1977         let src_bindir = builder
1978             .llvm_out(target)
1979             .join("bin");
1980         let dst_bindir = image.join("lib/rustlib")
1981             .join(&*target)
1982             .join("bin");
1983         t!(fs::create_dir_all(&dst_bindir));
1984         for tool in LLVM_TOOLS {
1985             let exe = src_bindir.join(exe(tool, &target));
1986             builder.install(&exe, &dst_bindir, 0o755);
1987         }
1988
1989         // Prepare the overlay
1990         let overlay = tmp.join("llvm-tools-overlay");
1991         drop(fs::remove_dir_all(&overlay));
1992         builder.create_dir(&overlay);
1993         builder.install(&src.join("README.txt"), &overlay, 0o644);
1994         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1995         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1996
1997         // Generate the installer tarball
1998         let mut cmd = rust_installer(builder);
1999         cmd.arg("generate")
2000             .arg("--product-name=Rust")
2001             .arg("--rel-manifest-dir=rustlib")
2002             .arg("--success-message=llvm-tools-installed.")
2003             .arg("--image-dir").arg(&image)
2004             .arg("--work-dir").arg(&tmpdir(builder))
2005             .arg("--output-dir").arg(&distdir(builder))
2006             .arg("--non-installed-overlay").arg(&overlay)
2007             .arg(format!("--package-name={}-{}", name, target))
2008             .arg("--legacy-manifest-dirs=rustlib,cargo")
2009             .arg("--component-name=llvm-tools-preview");
2010
2011
2012         builder.run(&mut cmd);
2013         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2014     }
2015 }
2016
2017 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2018 pub struct Lldb {
2019     pub target: Interned<String>,
2020 }
2021
2022 impl Step for Lldb {
2023     type Output = Option<PathBuf>;
2024     const ONLY_HOSTS: bool = true;
2025     const DEFAULT: bool = true;
2026
2027     fn should_run(run: ShouldRun) -> ShouldRun {
2028         run.path("src/tools/lldb")
2029     }
2030
2031     fn make_run(run: RunConfig) {
2032         run.builder.ensure(Lldb {
2033             target: run.target,
2034         });
2035     }
2036
2037     fn run(self, builder: &Builder) -> Option<PathBuf> {
2038         let target = self.target;
2039
2040         if builder.config.dry_run {
2041             return None;
2042         }
2043
2044         let bindir = builder
2045             .llvm_out(target)
2046             .join("bin");
2047         let lldb_exe = bindir.join(exe("lldb", &target));
2048         if !lldb_exe.exists() {
2049             return None;
2050         }
2051
2052         builder.info(&format!("Dist Lldb ({})", target));
2053         let src = builder.src.join("src/tools/lldb");
2054         let name = pkgname(builder, "lldb");
2055
2056         let tmp = tmpdir(builder);
2057         let image = tmp.join("lldb-image");
2058         drop(fs::remove_dir_all(&image));
2059
2060         // Prepare the image directory
2061         let root = image.join("lib/rustlib").join(&*target);
2062         let dst = root.join("bin");
2063         t!(fs::create_dir_all(&dst));
2064         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2065             let exe = bindir.join(exe(program, &target));
2066             builder.install(&exe, &dst, 0o755);
2067         }
2068
2069         // The libraries.
2070         let libdir = builder.llvm_out(target).join("lib");
2071         let dst = root.join("lib");
2072         t!(fs::create_dir_all(&dst));
2073         for entry in t!(fs::read_dir(&libdir)) {
2074             let entry = entry.unwrap();
2075             if let Ok(name) = entry.file_name().into_string() {
2076                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2077                     if t!(entry.file_type()).is_symlink() {
2078                         builder.copy_to_folder(&entry.path(), &dst);
2079                     } else {
2080                        builder.install(&entry.path(), &dst, 0o755);
2081                     }
2082                 }
2083             }
2084         }
2085
2086         // The lldb scripts might be installed in lib/python$version
2087         // or in lib64/python$version.  If lib64 exists, use it;
2088         // otherwise lib.
2089         let libdir = builder.llvm_out(target).join("lib64");
2090         let (libdir, libdir_name) = if libdir.exists() {
2091             (libdir, "lib64")
2092         } else {
2093             (builder.llvm_out(target).join("lib"), "lib")
2094         };
2095         for entry in t!(fs::read_dir(&libdir)) {
2096             let entry = t!(entry);
2097             if let Ok(name) = entry.file_name().into_string() {
2098                 if name.starts_with("python") {
2099                     let dst = root.join(libdir_name)
2100                         .join(entry.file_name());
2101                     t!(fs::create_dir_all(&dst));
2102                     builder.cp_r(&entry.path(), &dst);
2103                     break;
2104                 }
2105             }
2106         }
2107
2108         // Prepare the overlay
2109         let overlay = tmp.join("lldb-overlay");
2110         drop(fs::remove_dir_all(&overlay));
2111         builder.create_dir(&overlay);
2112         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2113         builder.create(&overlay.join("version"), &builder.lldb_vers());
2114
2115         // Generate the installer tarball
2116         let mut cmd = rust_installer(builder);
2117         cmd.arg("generate")
2118             .arg("--product-name=Rust")
2119             .arg("--rel-manifest-dir=rustlib")
2120             .arg("--success-message=lldb-installed.")
2121             .arg("--image-dir").arg(&image)
2122             .arg("--work-dir").arg(&tmpdir(builder))
2123             .arg("--output-dir").arg(&distdir(builder))
2124             .arg("--non-installed-overlay").arg(&overlay)
2125             .arg(format!("--package-name={}-{}", name, target))
2126             .arg("--legacy-manifest-dirs=rustlib,cargo")
2127             .arg("--component-name=lldb-preview");
2128
2129
2130         builder.run(&mut cmd);
2131         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2132     }
2133 }