src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Compiler, Mode, LLVM_TOOLS};
  30 use channel;
  31 use util::{libdir, is_dylib, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "src/Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/dlmalloc",
 861             "src/liballoc",
 862             "src/liballoc_system",
 863             "src/libbacktrace",
 864             "src/libcompiler_builtins",
 865             "src/libcore",
 866             "src/liblibc",
 867             "src/libpanic_abort",
 868             "src/libpanic_unwind",
 869             "src/librustc_asan",
 870             "src/librustc_lsan",
 871             "src/librustc_msan",
 872             "src/librustc_tsan",
 873             "src/libstd",
 874             "src/libunwind",
 875             "src/rustc/compiler_builtins_shim",
 876             "src/rustc/libc_shim",
 877             "src/rustc/dlmalloc_shim",
 878             "src/libtest",
 879             "src/libterm",
 880             "src/libprofiler_builtins",
 881             "src/stdsimd",
 882             "src/libproc_macro",
 883         ];
 884         let std_src_dirs_exclude = [
 885             "src/libcompiler_builtins/compiler-rt/test",
 886         ];
 887
 888         copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 889         for file in src_files.iter() {
 890             builder.copy(&builder.src.join(file), &dst_src.join(file));
 891         }
 892
 893         // Create source tarball in rust-installer format
 894         let mut cmd = rust_installer(builder);
 895         cmd.arg("generate")
 896            .arg("--product-name=Rust")
 897            .arg("--rel-manifest-dir=rustlib")
 898            .arg("--success-message=Awesome-Source.")
 899            .arg("--image-dir").arg(&image)
 900            .arg("--work-dir").arg(&tmpdir(builder))
 901            .arg("--output-dir").arg(&distdir(builder))
 902            .arg(format!("--package-name={}", name))
 903            .arg("--component-name=rust-src")
 904            .arg("--legacy-manifest-dirs=rustlib,cargo");
 905         builder.run(&mut cmd);
 906
 907         builder.remove_dir(&image);
 908         distdir(builder).join(&format!("{}.tar.gz", name))
 909     }
 910 }
 911
 912 const CARGO_VENDOR_VERSION: &str = "0.1.19";
 913
 914 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 915 pub struct PlainSourceTarball;
 916
 917 impl Step for PlainSourceTarball {
 918     /// Produces the location of the tarball generated
 919     type Output = PathBuf;
 920     const DEFAULT: bool = true;
 921     const ONLY_HOSTS: bool = true;
 922
 923     fn should_run(run: ShouldRun) -> ShouldRun {
 924         let builder = run.builder;
 925         run.path("src").default_condition(builder.config.rust_dist_src)
 926     }
 927
 928     fn make_run(run: RunConfig) {
 929         run.builder.ensure(PlainSourceTarball);
 930     }
 931
 932     /// Creates the plain source tarball
 933     fn run(self, builder: &Builder) -> PathBuf {
 934         builder.info("Create plain source tarball");
 935
 936         // Make sure that the root folder of tarball has the correct name
 937         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 938         let plain_dst_src = tmpdir(builder).join(&plain_name);
 939         let _ = fs::remove_dir_all(&plain_dst_src);
 940         t!(fs::create_dir_all(&plain_dst_src));
 941
 942         // This is the set of root paths which will become part of the source package
 943         let src_files = [
 944             "COPYRIGHT",
 945             "LICENSE-APACHE",
 946             "LICENSE-MIT",
 947             "CONTRIBUTING.md",
 948             "README.md",
 949             "RELEASES.md",
 950             "configure",
 951             "x.py",
 952             "config.toml.example",
 953         ];
 954         let src_dirs = [
 955             "src",
 956         ];
 957
 958         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 959
 960         // Copy the files normally
 961         for item in &src_files {
 962             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 963         }
 964
 965         // Create the version file
 966         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 967         if let Some(sha) = builder.rust_sha() {
 968             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 969         }
 970
 971         // If we're building from git sources, we need to vendor a complete distribution.
 972         if builder.rust_info.is_git() {
 973             // Get cargo-vendor installed, if it isn't already.
 974             let mut has_cargo_vendor = false;
 975             let mut cmd = Command::new(&builder.initial_cargo);
 976             for line in output(cmd.arg("install").arg("--list")).lines() {
 977                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 978             }
 979             if !has_cargo_vendor {
 980                 let mut cmd = builder.cargo(
 981                     builder.compiler(0, builder.config.build),
 982                     Mode::ToolBootstrap,
 983                     builder.config.build,
 984                     "install"
 985                 );
 986                 cmd.arg("--force")
 987                    .arg("--debug")
 988                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 989                    .arg("cargo-vendor");
 990                 builder.run(&mut cmd);
 991             }
 992
 993             // Vendor all Cargo dependencies
 994             let mut cmd = Command::new(&builder.initial_cargo);
 995             cmd.arg("vendor")
 996                .current_dir(&plain_dst_src.join("src"));
 997             builder.run(&mut cmd);
 998         }
 999
1000         // Create plain source tarball
1001         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1002         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1003         tarball.set_extension(""); // strip .gz
1004         tarball.set_extension(""); // strip .tar
1005         if let Some(dir) = tarball.parent() {
1006             builder.create_dir(&dir);
1007         }
1008         builder.info("running installer");
1009         let mut cmd = rust_installer(builder);
1010         cmd.arg("tarball")
1011            .arg("--input").arg(&plain_name)
1012            .arg("--output").arg(&tarball)
1013            .arg("--work-dir=.")
1014            .current_dir(tmpdir(builder));
1015         builder.run(&mut cmd);
1016         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1017     }
1018 }
1019
1020 // We have to run a few shell scripts, which choke quite a bit on both `\`
1021 // characters and on `C:\` paths, so normalize both of them away.
1022 pub fn sanitize_sh(path: &Path) -> String {
1023     let path = path.to_str().unwrap().replace("\\", "/");
1024     return change_drive(&path).unwrap_or(path);
1025
1026     fn change_drive(s: &str) -> Option<String> {
1027         let mut ch = s.chars();
1028         let drive = ch.next().unwrap_or('C');
1029         if ch.next() != Some(':') {
1030             return None
1031         }
1032         if ch.next() != Some('/') {
1033             return None
1034         }
1035         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1036     }
1037 }
1038
1039 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1040 pub struct Cargo {
1041     pub stage: u32,
1042     pub target: Interned<String>,
1043 }
1044
1045 impl Step for Cargo {
1046     type Output = PathBuf;
1047     const ONLY_HOSTS: bool = true;
1048
1049     fn should_run(run: ShouldRun) -> ShouldRun {
1050         run.path("cargo")
1051     }
1052
1053     fn make_run(run: RunConfig) {
1054         run.builder.ensure(Cargo {
1055             stage: run.builder.top_stage,
1056             target: run.target,
1057         });
1058     }
1059
1060     fn run(self, builder: &Builder) -> PathBuf {
1061         let stage = self.stage;
1062         let target = self.target;
1063
1064         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1065         let src = builder.src.join("src/tools/cargo");
1066         let etc = src.join("src/etc");
1067         let release_num = builder.release_num("cargo");
1068         let name = pkgname(builder, "cargo");
1069         let version = builder.cargo_info.version(builder, &release_num);
1070
1071         let tmp = tmpdir(builder);
1072         let image = tmp.join("cargo-image");
1073         drop(fs::remove_dir_all(&image));
1074         builder.create_dir(&image);
1075
1076         // Prepare the image directory
1077         builder.create_dir(&image.join("share/zsh/site-functions"));
1078         builder.create_dir(&image.join("etc/bash_completion.d"));
1079         let cargo = builder.ensure(tool::Cargo {
1080             compiler: builder.compiler(stage, builder.config.build),
1081             target
1082         });
1083         builder.install(&cargo, &image.join("bin"), 0o755);
1084         for man in t!(etc.join("man").read_dir()) {
1085             let man = t!(man);
1086             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1087         }
1088         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1089         builder.copy(&etc.join("cargo.bashcomp.sh"),
1090              &image.join("etc/bash_completion.d/cargo"));
1091         let doc = image.join("share/doc/cargo");
1092         builder.install(&src.join("README.md"), &doc, 0o644);
1093         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1094         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1095         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1096
1097         // Prepare the overlay
1098         let overlay = tmp.join("cargo-overlay");
1099         drop(fs::remove_dir_all(&overlay));
1100         builder.create_dir(&overlay);
1101         builder.install(&src.join("README.md"), &overlay, 0o644);
1102         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1103         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1104         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1105         builder.create(&overlay.join("version"), &version);
1106
1107         // Generate the installer tarball
1108         let mut cmd = rust_installer(builder);
1109         cmd.arg("generate")
1110            .arg("--product-name=Rust")
1111            .arg("--rel-manifest-dir=rustlib")
1112            .arg("--success-message=Rust-is-ready-to-roll.")
1113            .arg("--image-dir").arg(&image)
1114            .arg("--work-dir").arg(&tmpdir(builder))
1115            .arg("--output-dir").arg(&distdir(builder))
1116            .arg("--non-installed-overlay").arg(&overlay)
1117            .arg(format!("--package-name={}-{}", name, target))
1118            .arg("--component-name=cargo")
1119            .arg("--legacy-manifest-dirs=rustlib,cargo");
1120         builder.run(&mut cmd);
1121         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1122     }
1123 }
1124
1125 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1126 pub struct Rls {
1127     pub stage: u32,
1128     pub target: Interned<String>,
1129 }
1130
1131 impl Step for Rls {
1132     type Output = Option<PathBuf>;
1133     const ONLY_HOSTS: bool = true;
1134
1135     fn should_run(run: ShouldRun) -> ShouldRun {
1136         run.path("rls")
1137     }
1138
1139     fn make_run(run: RunConfig) {
1140         run.builder.ensure(Rls {
1141             stage: run.builder.top_stage,
1142             target: run.target,
1143         });
1144     }
1145
1146     fn run(self, builder: &Builder) -> Option<PathBuf> {
1147         let stage = self.stage;
1148         let target = self.target;
1149         assert!(builder.config.extended);
1150
1151         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1152         let src = builder.src.join("src/tools/rls");
1153         let release_num = builder.release_num("rls");
1154         let name = pkgname(builder, "rls");
1155         let version = builder.rls_info.version(builder, &release_num);
1156
1157         let tmp = tmpdir(builder);
1158         let image = tmp.join("rls-image");
1159         drop(fs::remove_dir_all(&image));
1160         t!(fs::create_dir_all(&image));
1161
1162         // Prepare the image directory
1163         // We expect RLS to build, because we've exited this step above if tool
1164         // state for RLS isn't testing.
1165         let rls = builder.ensure(tool::Rls {
1166             compiler: builder.compiler(stage, builder.config.build),
1167             target, extra_features: Vec::new()
1168         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1169
1170         builder.install(&rls, &image.join("bin"), 0o755);
1171         let doc = image.join("share/doc/rls");
1172         builder.install(&src.join("README.md"), &doc, 0o644);
1173         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1174         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1175
1176         // Prepare the overlay
1177         let overlay = tmp.join("rls-overlay");
1178         drop(fs::remove_dir_all(&overlay));
1179         t!(fs::create_dir_all(&overlay));
1180         builder.install(&src.join("README.md"), &overlay, 0o644);
1181         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1182         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1183         builder.create(&overlay.join("version"), &version);
1184
1185         // Generate the installer tarball
1186         let mut cmd = rust_installer(builder);
1187         cmd.arg("generate")
1188            .arg("--product-name=Rust")
1189            .arg("--rel-manifest-dir=rustlib")
1190            .arg("--success-message=RLS-ready-to-serve.")
1191            .arg("--image-dir").arg(&image)
1192            .arg("--work-dir").arg(&tmpdir(builder))
1193            .arg("--output-dir").arg(&distdir(builder))
1194            .arg("--non-installed-overlay").arg(&overlay)
1195            .arg(format!("--package-name={}-{}", name, target))
1196            .arg("--legacy-manifest-dirs=rustlib,cargo")
1197            .arg("--component-name=rls-preview");
1198
1199         builder.run(&mut cmd);
1200         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1201     }
1202 }
1203
1204 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1205 pub struct Clippy {
1206     pub stage: u32,
1207     pub target: Interned<String>,
1208 }
1209
1210 impl Step for Clippy {
1211     type Output = Option<PathBuf>;
1212     const ONLY_HOSTS: bool = true;
1213
1214     fn should_run(run: ShouldRun) -> ShouldRun {
1215         run.path("clippy")
1216     }
1217
1218     fn make_run(run: RunConfig) {
1219         run.builder.ensure(Clippy {
1220             stage: run.builder.top_stage,
1221             target: run.target,
1222         });
1223     }
1224
1225     fn run(self, builder: &Builder) -> Option<PathBuf> {
1226         let stage = self.stage;
1227         let target = self.target;
1228         assert!(builder.config.extended);
1229
1230         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1231         let src = builder.src.join("src/tools/clippy");
1232         let release_num = builder.release_num("clippy");
1233         let name = pkgname(builder, "clippy");
1234         let version = builder.clippy_info.version(builder, &release_num);
1235
1236         let tmp = tmpdir(builder);
1237         let image = tmp.join("clippy-image");
1238         drop(fs::remove_dir_all(&image));
1239         builder.create_dir(&image);
1240
1241         // Prepare the image directory
1242         // We expect clippy to build, because we've exited this step above if tool
1243         // state for clippy isn't testing.
1244         let clippy = builder.ensure(tool::Clippy {
1245             compiler: builder.compiler(stage, builder.config.build),
1246             target, extra_features: Vec::new()
1247         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1248         let cargoclippy = builder.ensure(tool::CargoClippy {
1249             compiler: builder.compiler(stage, builder.config.build),
1250             target, extra_features: Vec::new()
1251         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1252
1253         builder.install(&clippy, &image.join("bin"), 0o755);
1254         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1255         let doc = image.join("share/doc/clippy");
1256         builder.install(&src.join("README.md"), &doc, 0o644);
1257         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1259
1260         // Prepare the overlay
1261         let overlay = tmp.join("clippy-overlay");
1262         drop(fs::remove_dir_all(&overlay));
1263         t!(fs::create_dir_all(&overlay));
1264         builder.install(&src.join("README.md"), &overlay, 0o644);
1265         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1266         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1267         builder.create(&overlay.join("version"), &version);
1268
1269         // Generate the installer tarball
1270         let mut cmd = rust_installer(builder);
1271         cmd.arg("generate")
1272            .arg("--product-name=Rust")
1273            .arg("--rel-manifest-dir=rustlib")
1274            .arg("--success-message=clippy-ready-to-serve.")
1275            .arg("--image-dir").arg(&image)
1276            .arg("--work-dir").arg(&tmpdir(builder))
1277            .arg("--output-dir").arg(&distdir(builder))
1278            .arg("--non-installed-overlay").arg(&overlay)
1279            .arg(format!("--package-name={}-{}", name, target))
1280            .arg("--legacy-manifest-dirs=rustlib,cargo")
1281            .arg("--component-name=clippy-preview");
1282
1283         builder.run(&mut cmd);
1284         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1285     }
1286 }
1287
1288 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1289 pub struct Rustfmt {
1290     pub stage: u32,
1291     pub target: Interned<String>,
1292 }
1293
1294 impl Step for Rustfmt {
1295     type Output = Option<PathBuf>;
1296     const ONLY_HOSTS: bool = true;
1297
1298     fn should_run(run: ShouldRun) -> ShouldRun {
1299         run.path("rustfmt")
1300     }
1301
1302     fn make_run(run: RunConfig) {
1303         run.builder.ensure(Rustfmt {
1304             stage: run.builder.top_stage,
1305             target: run.target,
1306         });
1307     }
1308
1309     fn run(self, builder: &Builder) -> Option<PathBuf> {
1310         let stage = self.stage;
1311         let target = self.target;
1312
1313         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1314         let src = builder.src.join("src/tools/rustfmt");
1315         let release_num = builder.release_num("rustfmt");
1316         let name = pkgname(builder, "rustfmt");
1317         let version = builder.rustfmt_info.version(builder, &release_num);
1318
1319         let tmp = tmpdir(builder);
1320         let image = tmp.join("rustfmt-image");
1321         drop(fs::remove_dir_all(&image));
1322         builder.create_dir(&image);
1323
1324         // Prepare the image directory
1325         let rustfmt = builder.ensure(tool::Rustfmt {
1326             compiler: builder.compiler(stage, builder.config.build),
1327             target, extra_features: Vec::new()
1328         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1329         let cargofmt = builder.ensure(tool::Cargofmt {
1330             compiler: builder.compiler(stage, builder.config.build),
1331             target, extra_features: Vec::new()
1332         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1333
1334         builder.install(&rustfmt, &image.join("bin"), 0o755);
1335         builder.install(&cargofmt, &image.join("bin"), 0o755);
1336         let doc = image.join("share/doc/rustfmt");
1337         builder.install(&src.join("README.md"), &doc, 0o644);
1338         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1339         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1340
1341         // Prepare the overlay
1342         let overlay = tmp.join("rustfmt-overlay");
1343         drop(fs::remove_dir_all(&overlay));
1344         builder.create_dir(&overlay);
1345         builder.install(&src.join("README.md"), &overlay, 0o644);
1346         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1347         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1348         builder.create(&overlay.join("version"), &version);
1349
1350         // Generate the installer tarball
1351         let mut cmd = rust_installer(builder);
1352         cmd.arg("generate")
1353            .arg("--product-name=Rust")
1354            .arg("--rel-manifest-dir=rustlib")
1355            .arg("--success-message=rustfmt-ready-to-fmt.")
1356            .arg("--image-dir").arg(&image)
1357            .arg("--work-dir").arg(&tmpdir(builder))
1358            .arg("--output-dir").arg(&distdir(builder))
1359            .arg("--non-installed-overlay").arg(&overlay)
1360            .arg(format!("--package-name={}-{}", name, target))
1361            .arg("--legacy-manifest-dirs=rustlib,cargo")
1362            .arg("--component-name=rustfmt-preview");
1363
1364         builder.run(&mut cmd);
1365         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1366     }
1367 }
1368
1369 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1370 pub struct Extended {
1371     stage: u32,
1372     host: Interned<String>,
1373     target: Interned<String>,
1374 }
1375
1376 impl Step for Extended {
1377     type Output = ();
1378     const DEFAULT: bool = true;
1379     const ONLY_HOSTS: bool = true;
1380
1381     fn should_run(run: ShouldRun) -> ShouldRun {
1382         let builder = run.builder;
1383         run.path("extended").default_condition(builder.config.extended)
1384     }
1385
1386     fn make_run(run: RunConfig) {
1387         run.builder.ensure(Extended {
1388             stage: run.builder.top_stage,
1389             host: run.builder.config.build,
1390             target: run.target,
1391         });
1392     }
1393
1394     /// Creates a combined installer for the specified target in the provided stage.
1395     fn run(self, builder: &Builder) {
1396         let stage = self.stage;
1397         let target = self.target;
1398
1399         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1400
1401         let rustc_installer = builder.ensure(Rustc {
1402             compiler: builder.compiler(stage, target),
1403         });
1404         let cargo_installer = builder.ensure(Cargo { stage, target });
1405         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1406         let rls_installer = builder.ensure(Rls { stage, target });
1407         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1408         let clippy_installer = builder.ensure(Clippy { stage, target });
1409         let lldb_installer = builder.ensure(Lldb { target });
1410         let mingw_installer = builder.ensure(Mingw { host: target });
1411         let analysis_installer = builder.ensure(Analysis {
1412             compiler: builder.compiler(stage, self.host),
1413             target
1414         });
1415
1416         let docs_installer = builder.ensure(Docs { stage, host: target, });
1417         let std_installer = builder.ensure(Std {
1418             compiler: builder.compiler(stage, self.host),
1419             target,
1420         });
1421
1422         let tmp = tmpdir(builder);
1423         let overlay = tmp.join("extended-overlay");
1424         let etc = builder.src.join("src/etc/installer");
1425         let work = tmp.join("work");
1426
1427         let _ = fs::remove_dir_all(&overlay);
1428         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1429         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1430         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1431         let version = builder.rust_version();
1432         builder.create(&overlay.join("version"), &version);
1433         if let Some(sha) = builder.rust_sha() {
1434             builder.create(&overlay.join("git-commit-hash"), &sha);
1435         }
1436         builder.install(&etc.join("README.md"), &overlay, 0o644);
1437
1438         // When rust-std package split from rustc, we needed to ensure that during
1439         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1440         // the std files during uninstall. To do this ensure that rustc comes
1441         // before rust-std in the list below.
1442         let mut tarballs = Vec::new();
1443         tarballs.push(rustc_installer);
1444         tarballs.push(cargo_installer);
1445         tarballs.extend(rls_installer.clone());
1446         tarballs.extend(clippy_installer.clone());
1447         tarballs.extend(rustfmt_installer.clone());
1448         tarballs.extend(llvm_tools_installer);
1449         tarballs.extend(lldb_installer);
1450         tarballs.push(analysis_installer);
1451         tarballs.push(std_installer);
1452         if builder.config.docs {
1453             tarballs.push(docs_installer);
1454         }
1455         if target.contains("pc-windows-gnu") {
1456             tarballs.push(mingw_installer.unwrap());
1457         }
1458         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1459         for tarball in &tarballs[1..] {
1460             input_tarballs.push(",");
1461             input_tarballs.push(tarball);
1462         }
1463
1464         let mut cmd = rust_installer(builder);
1465         cmd.arg("combine")
1466             .arg("--product-name=Rust")
1467             .arg("--rel-manifest-dir=rustlib")
1468             .arg("--success-message=Rust-is-ready-to-roll.")
1469             .arg("--work-dir").arg(&work)
1470             .arg("--output-dir").arg(&distdir(builder))
1471             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1472             .arg("--legacy-manifest-dirs=rustlib,cargo")
1473             .arg("--input-tarballs").arg(input_tarballs)
1474             .arg("--non-installed-overlay").arg(&overlay);
1475         builder.run(&mut cmd);
1476
1477         let mut license = String::new();
1478         license += &builder.read(&builder.src.join("COPYRIGHT"));
1479         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1480         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1481         license.push_str("\n");
1482         license.push_str("\n");
1483
1484         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1485         let mut rtf = rtf.to_string();
1486         rtf.push_str("\n");
1487         for line in license.lines() {
1488             rtf.push_str(line);
1489             rtf.push_str("\\line ");
1490         }
1491         rtf.push_str("}");
1492
1493         fn filter(contents: &str, marker: &str) -> String {
1494             let start = format!("tool-{}-start", marker);
1495             let end = format!("tool-{}-end", marker);
1496             let mut lines = Vec::new();
1497             let mut omitted = false;
1498             for line in contents.lines() {
1499                 if line.contains(&start) {
1500                     omitted = true;
1501                 } else if line.contains(&end) {
1502                     omitted = false;
1503                 } else if !omitted {
1504                     lines.push(line);
1505                 }
1506             }
1507
1508             lines.join("\n")
1509         }
1510
1511         let xform = |p: &Path| {
1512             let mut contents = String::new();
1513             t!(t!(File::open(p)).read_to_string(&mut contents));
1514             if rls_installer.is_none() {
1515                 contents = filter(&contents, "rls");
1516             }
1517             if clippy_installer.is_none() {
1518                 contents = filter(&contents, "clippy");
1519             }
1520             if rustfmt_installer.is_none() {
1521                 contents = filter(&contents, "rustfmt");
1522             }
1523             let ret = tmp.join(p.file_name().unwrap());
1524             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1525             return ret
1526         };
1527
1528         if target.contains("apple-darwin") {
1529             let pkg = tmp.join("pkg");
1530             let _ = fs::remove_dir_all(&pkg);
1531
1532             let pkgbuild = |component: &str| {
1533                 let mut cmd = Command::new("pkgbuild");
1534                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1535                     .arg("--scripts").arg(pkg.join(component))
1536                     .arg("--nopayload")
1537                     .arg(pkg.join(component).with_extension("pkg"));
1538                 builder.run(&mut cmd);
1539             };
1540
1541             let prepare = |name: &str| {
1542                 builder.create_dir(&pkg.join(name));
1543                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1544                         &pkg.join(name));
1545                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1546                 pkgbuild(name);
1547             };
1548             prepare("rustc");
1549             prepare("cargo");
1550             prepare("rust-docs");
1551             prepare("rust-std");
1552             prepare("rust-analysis");
1553
1554             if rls_installer.is_some() {
1555                 prepare("rls");
1556             }
1557             if clippy_installer.is_some() {
1558                 prepare("clippy");
1559             }
1560
1561             // create an 'uninstall' package
1562             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1563             pkgbuild("uninstall");
1564
1565             builder.create_dir(&pkg.join("res"));
1566             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1567             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1568             let mut cmd = Command::new("productbuild");
1569             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1570                 .arg("--resources").arg(pkg.join("res"))
1571                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1572                                                     pkgname(builder, "rust"),
1573                                                     target)))
1574                 .arg("--package-path").arg(&pkg);
1575             builder.run(&mut cmd);
1576         }
1577
1578         if target.contains("windows") {
1579             let exe = tmp.join("exe");
1580             let _ = fs::remove_dir_all(&exe);
1581
1582             let prepare = |name: &str| {
1583                 builder.create_dir(&exe.join(name));
1584                 let dir = if name == "rust-std" || name == "rust-analysis" {
1585                     format!("{}-{}", name, target)
1586                 } else if name == "rls" {
1587                     "rls-preview".to_string()
1588                 } else if name == "clippy" {
1589                     "clippy-preview".to_string()
1590                 } else {
1591                     name.to_string()
1592                 };
1593                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1594                             .join(dir),
1595                         &exe.join(name));
1596                 builder.remove(&exe.join(name).join("manifest.in"));
1597             };
1598             prepare("rustc");
1599             prepare("cargo");
1600             prepare("rust-analysis");
1601             prepare("rust-docs");
1602             prepare("rust-std");
1603             if rls_installer.is_some() {
1604                 prepare("rls");
1605             }
1606             if clippy_installer.is_some() {
1607                 prepare("clippy");
1608             }
1609             if target.contains("windows-gnu") {
1610                 prepare("rust-mingw");
1611             }
1612
1613             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1614             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1615             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1616             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1617             builder.create(&exe.join("LICENSE.txt"), &license);
1618
1619             // Generate exe installer
1620             let mut cmd = Command::new("iscc");
1621             cmd.arg("rust.iss")
1622                 .current_dir(&exe);
1623             if target.contains("windows-gnu") {
1624                 cmd.arg("/dMINGW");
1625             }
1626             add_env(builder, &mut cmd, target);
1627             builder.run(&mut cmd);
1628             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1629                     &distdir(builder),
1630                     0o755);
1631
1632             // Generate msi installer
1633             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1634             let heat = wix.join("bin/heat.exe");
1635             let candle = wix.join("bin/candle.exe");
1636             let light = wix.join("bin/light.exe");
1637
1638             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1639             builder.run(Command::new(&heat)
1640                             .current_dir(&exe)
1641                             .arg("dir")
1642                             .arg("rustc")
1643                             .args(&heat_flags)
1644                             .arg("-cg").arg("RustcGroup")
1645                             .arg("-dr").arg("Rustc")
1646                             .arg("-var").arg("var.RustcDir")
1647                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1648             builder.run(Command::new(&heat)
1649                             .current_dir(&exe)
1650                             .arg("dir")
1651                             .arg("rust-docs")
1652                             .args(&heat_flags)
1653                             .arg("-cg").arg("DocsGroup")
1654                             .arg("-dr").arg("Docs")
1655                             .arg("-var").arg("var.DocsDir")
1656                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1657                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1658             builder.run(Command::new(&heat)
1659                             .current_dir(&exe)
1660                             .arg("dir")
1661                             .arg("cargo")
1662                             .args(&heat_flags)
1663                             .arg("-cg").arg("CargoGroup")
1664                             .arg("-dr").arg("Cargo")
1665                             .arg("-var").arg("var.CargoDir")
1666                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1667                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1668             builder.run(Command::new(&heat)
1669                             .current_dir(&exe)
1670                             .arg("dir")
1671                             .arg("rust-std")
1672                             .args(&heat_flags)
1673                             .arg("-cg").arg("StdGroup")
1674                             .arg("-dr").arg("Std")
1675                             .arg("-var").arg("var.StdDir")
1676                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1677             if rls_installer.is_some() {
1678                 builder.run(Command::new(&heat)
1679                                 .current_dir(&exe)
1680                                 .arg("dir")
1681                                 .arg("rls")
1682                                 .args(&heat_flags)
1683                                 .arg("-cg").arg("RlsGroup")
1684                                 .arg("-dr").arg("Rls")
1685                                 .arg("-var").arg("var.RlsDir")
1686                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1687                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1688             }
1689             if clippy_installer.is_some() {
1690                 builder.run(Command::new(&heat)
1691                                 .current_dir(&exe)
1692                                 .arg("dir")
1693                                 .arg("clippy")
1694                                 .args(&heat_flags)
1695                                 .arg("-cg").arg("ClippyGroup")
1696                                 .arg("-dr").arg("Clippy")
1697                                 .arg("-var").arg("var.ClippyDir")
1698                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1699                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1700             }
1701             builder.run(Command::new(&heat)
1702                             .current_dir(&exe)
1703                             .arg("dir")
1704                             .arg("rust-analysis")
1705                             .args(&heat_flags)
1706                             .arg("-cg").arg("AnalysisGroup")
1707                             .arg("-dr").arg("Analysis")
1708                             .arg("-var").arg("var.AnalysisDir")
1709                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1710                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1711             if target.contains("windows-gnu") {
1712                 builder.run(Command::new(&heat)
1713                                 .current_dir(&exe)
1714                                 .arg("dir")
1715                                 .arg("rust-mingw")
1716                                 .args(&heat_flags)
1717                                 .arg("-cg").arg("GccGroup")
1718                                 .arg("-dr").arg("Gcc")
1719                                 .arg("-var").arg("var.GccDir")
1720                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1721             }
1722
1723             let candle = |input: &Path| {
1724                 let output = exe.join(input.file_stem().unwrap())
1725                                 .with_extension("wixobj");
1726                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1727                 let mut cmd = Command::new(&candle);
1728                 cmd.current_dir(&exe)
1729                     .arg("-nologo")
1730                     .arg("-dRustcDir=rustc")
1731                     .arg("-dDocsDir=rust-docs")
1732                     .arg("-dCargoDir=cargo")
1733                     .arg("-dStdDir=rust-std")
1734                     .arg("-dAnalysisDir=rust-analysis")
1735                     .arg("-arch").arg(&arch)
1736                     .arg("-out").arg(&output)
1737                     .arg(&input);
1738                 add_env(builder, &mut cmd, target);
1739
1740                 if rls_installer.is_some() {
1741                     cmd.arg("-dRlsDir=rls");
1742                 }
1743                 if clippy_installer.is_some() {
1744                     cmd.arg("-dClippyDir=clippy");
1745                 }
1746                 if target.contains("windows-gnu") {
1747                     cmd.arg("-dGccDir=rust-mingw");
1748                 }
1749                 builder.run(&mut cmd);
1750             };
1751             candle(&xform(&etc.join("msi/rust.wxs")));
1752             candle(&etc.join("msi/ui.wxs"));
1753             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1754             candle("RustcGroup.wxs".as_ref());
1755             candle("DocsGroup.wxs".as_ref());
1756             candle("CargoGroup.wxs".as_ref());
1757             candle("StdGroup.wxs".as_ref());
1758             if rls_installer.is_some() {
1759                 candle("RlsGroup.wxs".as_ref());
1760             }
1761             if clippy_installer.is_some() {
1762                 candle("ClippyGroup.wxs".as_ref());
1763             }
1764             candle("AnalysisGroup.wxs".as_ref());
1765
1766             if target.contains("windows-gnu") {
1767                 candle("GccGroup.wxs".as_ref());
1768             }
1769
1770             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1771             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1772             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1773
1774             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1775             let mut cmd = Command::new(&light);
1776             cmd.arg("-nologo")
1777                 .arg("-ext").arg("WixUIExtension")
1778                 .arg("-ext").arg("WixUtilExtension")
1779                 .arg("-out").arg(exe.join(&filename))
1780                 .arg("rust.wixobj")
1781                 .arg("ui.wixobj")
1782                 .arg("rustwelcomedlg.wixobj")
1783                 .arg("RustcGroup.wixobj")
1784                 .arg("DocsGroup.wixobj")
1785                 .arg("CargoGroup.wixobj")
1786                 .arg("StdGroup.wixobj")
1787                 .arg("AnalysisGroup.wixobj")
1788                 .current_dir(&exe);
1789
1790             if rls_installer.is_some() {
1791                 cmd.arg("RlsGroup.wixobj");
1792             }
1793             if clippy_installer.is_some() {
1794                 cmd.arg("ClippyGroup.wixobj");
1795             }
1796
1797             if target.contains("windows-gnu") {
1798                 cmd.arg("GccGroup.wixobj");
1799             }
1800             // ICE57 wrongly complains about the shortcuts
1801             cmd.arg("-sice:ICE57");
1802
1803             builder.run(&mut cmd);
1804
1805             if !builder.config.dry_run {
1806                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1807             }
1808         }
1809     }
1810 }
1811
1812 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1813     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1814     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1815        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1816        .env("CFG_RELEASE", builder.rust_release())
1817        .env("CFG_VER_MAJOR", parts.next().unwrap())
1818        .env("CFG_VER_MINOR", parts.next().unwrap())
1819        .env("CFG_VER_PATCH", parts.next().unwrap())
1820        .env("CFG_VER_BUILD", "0") // just needed to build
1821        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1822        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1823        .env("CFG_BUILD", target)
1824        .env("CFG_CHANNEL", &builder.config.channel);
1825
1826     if target.contains("windows-gnu") {
1827        cmd.env("CFG_MINGW", "1")
1828           .env("CFG_ABI", "GNU");
1829     } else {
1830        cmd.env("CFG_MINGW", "0")
1831           .env("CFG_ABI", "MSVC");
1832     }
1833
1834     if target.contains("x86_64") {
1835        cmd.env("CFG_PLATFORM", "x64");
1836     } else {
1837        cmd.env("CFG_PLATFORM", "x86");
1838     }
1839 }
1840
1841 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1842 pub struct HashSign;
1843
1844 impl Step for HashSign {
1845     type Output = ();
1846     const ONLY_HOSTS: bool = true;
1847
1848     fn should_run(run: ShouldRun) -> ShouldRun {
1849         run.path("hash-and-sign")
1850     }
1851
1852     fn make_run(run: RunConfig) {
1853         run.builder.ensure(HashSign);
1854     }
1855
1856     fn run(self, builder: &Builder) {
1857         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1858         if builder.config.dry_run {
1859             return;
1860         }
1861         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1862             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1863         });
1864         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1865             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1866         });
1867         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1868             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1869         });
1870         let mut pass = String::new();
1871         t!(t!(File::open(&file)).read_to_string(&mut pass));
1872
1873         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1874
1875         cmd.arg(sign);
1876         cmd.arg(distdir(builder));
1877         cmd.arg(today.trim());
1878         cmd.arg(builder.rust_package_vers());
1879         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1880         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1881         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1882         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1883         cmd.arg(builder.llvm_tools_package_vers());
1884         cmd.arg(builder.lldb_package_vers());
1885         cmd.arg(addr);
1886
1887         builder.create_dir(&distdir(builder));
1888
1889         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1890         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1891         let status = t!(child.wait());
1892         assert!(status.success());
1893     }
1894 }
1895
1896 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1897 // LLVM tools are linked dynamically.
1898 // Note: This function does no yet support Windows but we also don't support
1899 //       linking LLVM tools dynamically on Windows yet.
1900 fn maybe_install_llvm_dylib(builder: &Builder,
1901                             target: Interned<String>,
1902                             image: &Path) {
1903     let src_libdir = builder
1904         .llvm_out(target)
1905         .join("lib");
1906     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1907     t!(fs::create_dir_all(&dst_libdir));
1908
1909     if target.contains("apple-darwin") {
1910         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1911         if llvm_dylib_path.exists() {
1912             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1913         }
1914         return
1915     }
1916
1917     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1918     // Since tools link to the latter rather than the former, we have to
1919     // follow the symlink to find out what to distribute.
1920     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1921     if llvm_dylib_path.exists() {
1922         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1923             panic!("dist: Error calling canonicalize path `{}`: {}",
1924                    llvm_dylib_path.display(), e);
1925         });
1926
1927
1928         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1929     }
1930 }
1931
1932 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1933 pub struct LlvmTools {
1934     pub stage: u32,
1935     pub target: Interned<String>,
1936 }
1937
1938 impl Step for LlvmTools {
1939     type Output = Option<PathBuf>;
1940     const ONLY_HOSTS: bool = true;
1941
1942     fn should_run(run: ShouldRun) -> ShouldRun {
1943         run.path("llvm-tools")
1944     }
1945
1946     fn make_run(run: RunConfig) {
1947         run.builder.ensure(LlvmTools {
1948             stage: run.builder.top_stage,
1949             target: run.target,
1950         });
1951     }
1952
1953     fn run(self, builder: &Builder) -> Option<PathBuf> {
1954         let stage = self.stage;
1955         let target = self.target;
1956         assert!(builder.config.extended);
1957
1958         /* run only if llvm-config isn't used */
1959         if let Some(config) = builder.config.target_config.get(&target) {
1960             if let Some(ref _s) = config.llvm_config {
1961                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1962                     stage, target));
1963                 return None;
1964             }
1965         }
1966
1967         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1968         let src = builder.src.join("src/llvm");
1969         let name = pkgname(builder, "llvm-tools");
1970
1971         let tmp = tmpdir(builder);
1972         let image = tmp.join("llvm-tools-image");
1973         drop(fs::remove_dir_all(&image));
1974
1975         // Prepare the image directory
1976         let src_bindir = builder
1977             .llvm_out(target)
1978             .join("bin");
1979         let dst_bindir = image.join("lib/rustlib")
1980             .join(&*target)
1981             .join("bin");
1982         t!(fs::create_dir_all(&dst_bindir));
1983         for tool in LLVM_TOOLS {
1984             let exe = src_bindir.join(exe(tool, &target));
1985             builder.install(&exe, &dst_bindir, 0o755);
1986         }
1987
1988         // Prepare the overlay
1989         let overlay = tmp.join("llvm-tools-overlay");
1990         drop(fs::remove_dir_all(&overlay));
1991         builder.create_dir(&overlay);
1992         builder.install(&src.join("README.txt"), &overlay, 0o644);
1993         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1994         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1995
1996         // Generate the installer tarball
1997         let mut cmd = rust_installer(builder);
1998         cmd.arg("generate")
1999             .arg("--product-name=Rust")
2000             .arg("--rel-manifest-dir=rustlib")
2001             .arg("--success-message=llvm-tools-installed.")
2002             .arg("--image-dir").arg(&image)
2003             .arg("--work-dir").arg(&tmpdir(builder))
2004             .arg("--output-dir").arg(&distdir(builder))
2005             .arg("--non-installed-overlay").arg(&overlay)
2006             .arg(format!("--package-name={}-{}", name, target))
2007             .arg("--legacy-manifest-dirs=rustlib,cargo")
2008             .arg("--component-name=llvm-tools-preview");
2009
2010
2011         builder.run(&mut cmd);
2012         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2013     }
2014 }
2015
2016 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2017 pub struct Lldb {
2018     pub target: Interned<String>,
2019 }
2020
2021 impl Step for Lldb {
2022     type Output = Option<PathBuf>;
2023     const ONLY_HOSTS: bool = true;
2024     const DEFAULT: bool = true;
2025
2026     fn should_run(run: ShouldRun) -> ShouldRun {
2027         run.path("src/tools/lldb")
2028     }
2029
2030     fn make_run(run: RunConfig) {
2031         run.builder.ensure(Lldb {
2032             target: run.target,
2033         });
2034     }
2035
2036     fn run(self, builder: &Builder) -> Option<PathBuf> {
2037         let target = self.target;
2038
2039         if builder.config.dry_run {
2040             return None;
2041         }
2042
2043         let bindir = builder
2044             .llvm_out(target)
2045             .join("bin");
2046         let lldb_exe = bindir.join(exe("lldb", &target));
2047         if !lldb_exe.exists() {
2048             return None;
2049         }
2050
2051         builder.info(&format!("Dist Lldb ({})", target));
2052         let src = builder.src.join("src/tools/lldb");
2053         let name = pkgname(builder, "lldb");
2054
2055         let tmp = tmpdir(builder);
2056         let image = tmp.join("lldb-image");
2057         drop(fs::remove_dir_all(&image));
2058
2059         // Prepare the image directory
2060         let root = image.join("lib/rustlib").join(&*target);
2061         let dst = root.join("bin");
2062         t!(fs::create_dir_all(&dst));
2063         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2064             let exe = bindir.join(exe(program, &target));
2065             builder.install(&exe, &dst, 0o755);
2066         }
2067
2068         // The libraries.
2069         let libdir = builder.llvm_out(target).join("lib");
2070         let dst = root.join("lib");
2071         t!(fs::create_dir_all(&dst));
2072         for entry in t!(fs::read_dir(&libdir)) {
2073             let entry = entry.unwrap();
2074             if let Ok(name) = entry.file_name().into_string() {
2075                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2076                     if t!(entry.file_type()).is_symlink() {
2077                         builder.copy_to_folder(&entry.path(), &dst);
2078                     } else {
2079                        builder.install(&entry.path(), &dst, 0o755);
2080                     }
2081                 }
2082             }
2083         }
2084
2085         // The lldb scripts might be installed in lib/python$version
2086         // or in lib64/python$version.  If lib64 exists, use it;
2087         // otherwise lib.
2088         let libdir = builder.llvm_out(target).join("lib64");
2089         let (libdir, libdir_name) = if libdir.exists() {
2090             (libdir, "lib64")
2091         } else {
2092             (builder.llvm_out(target).join("lib"), "lib")
2093         };
2094         for entry in t!(fs::read_dir(&libdir)) {
2095             let entry = t!(entry);
2096             if let Ok(name) = entry.file_name().into_string() {
2097                 if name.starts_with("python") {
2098                     let dst = root.join(libdir_name)
2099                         .join(entry.file_name());
2100                     t!(fs::create_dir_all(&dst));
2101                     builder.cp_r(&entry.path(), &dst);
2102                     break;
2103                 }
2104             }
2105         }
2106
2107         // Prepare the overlay
2108         let overlay = tmp.join("lldb-overlay");
2109         drop(fs::remove_dir_all(&overlay));
2110         builder.create_dir(&overlay);
2111         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2112         builder.create(&overlay.join("version"), &builder.lldb_vers());
2113
2114         // Generate the installer tarball
2115         let mut cmd = rust_installer(builder);
2116         cmd.arg("generate")
2117             .arg("--product-name=Rust")
2118             .arg("--rel-manifest-dir=rustlib")
2119             .arg("--success-message=lldb-installed.")
2120             .arg("--image-dir").arg(&image)
2121             .arg("--work-dir").arg(&tmpdir(builder))
2122             .arg("--output-dir").arg(&distdir(builder))
2123             .arg("--non-installed-overlay").arg(&overlay)
2124             .arg(format!("--package-name={}-{}", name, target))
2125             .arg("--legacy-manifest-dirs=rustlib,cargo")
2126             .arg("--component-name=lldb-preview");
2127
2128
2129         builder.run(&mut cmd);
2130         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2131     }
2132 }