src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Compiler, Mode, LLVM_TOOLS};
  30 use channel;
  31 use util::{libdir, is_dylib, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "src/Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/dlmalloc",
 861             "src/liballoc",
 862             "src/libbacktrace",
 863             "src/libcompiler_builtins",
 864             "src/libcore",
 865             "src/liblibc",
 866             "src/libpanic_abort",
 867             "src/libpanic_unwind",
 868             "src/librustc_asan",
 869             "src/librustc_lsan",
 870             "src/librustc_msan",
 871             "src/librustc_tsan",
 872             "src/libstd",
 873             "src/libunwind",
 874             "src/rustc/compiler_builtins_shim",
 875             "src/rustc/libc_shim",
 876             "src/rustc/dlmalloc_shim",
 877             "src/libtest",
 878             "src/libterm",
 879             "src/libprofiler_builtins",
 880             "src/stdsimd",
 881             "src/libproc_macro",
 882         ];
 883         let std_src_dirs_exclude = [
 884             "src/libcompiler_builtins/compiler-rt/test",
 885         ];
 886
 887         copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 888         for file in src_files.iter() {
 889             builder.copy(&builder.src.join(file), &dst_src.join(file));
 890         }
 891
 892         // Create source tarball in rust-installer format
 893         let mut cmd = rust_installer(builder);
 894         cmd.arg("generate")
 895            .arg("--product-name=Rust")
 896            .arg("--rel-manifest-dir=rustlib")
 897            .arg("--success-message=Awesome-Source.")
 898            .arg("--image-dir").arg(&image)
 899            .arg("--work-dir").arg(&tmpdir(builder))
 900            .arg("--output-dir").arg(&distdir(builder))
 901            .arg(format!("--package-name={}", name))
 902            .arg("--component-name=rust-src")
 903            .arg("--legacy-manifest-dirs=rustlib,cargo");
 904         builder.run(&mut cmd);
 905
 906         builder.remove_dir(&image);
 907         distdir(builder).join(&format!("{}.tar.gz", name))
 908     }
 909 }
 910
 911 const CARGO_VENDOR_VERSION: &str = "0.1.19";
 912
 913 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 914 pub struct PlainSourceTarball;
 915
 916 impl Step for PlainSourceTarball {
 917     /// Produces the location of the tarball generated
 918     type Output = PathBuf;
 919     const DEFAULT: bool = true;
 920     const ONLY_HOSTS: bool = true;
 921
 922     fn should_run(run: ShouldRun) -> ShouldRun {
 923         let builder = run.builder;
 924         run.path("src").default_condition(builder.config.rust_dist_src)
 925     }
 926
 927     fn make_run(run: RunConfig) {
 928         run.builder.ensure(PlainSourceTarball);
 929     }
 930
 931     /// Creates the plain source tarball
 932     fn run(self, builder: &Builder) -> PathBuf {
 933         builder.info("Create plain source tarball");
 934
 935         // Make sure that the root folder of tarball has the correct name
 936         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 937         let plain_dst_src = tmpdir(builder).join(&plain_name);
 938         let _ = fs::remove_dir_all(&plain_dst_src);
 939         t!(fs::create_dir_all(&plain_dst_src));
 940
 941         // This is the set of root paths which will become part of the source package
 942         let src_files = [
 943             "COPYRIGHT",
 944             "LICENSE-APACHE",
 945             "LICENSE-MIT",
 946             "CONTRIBUTING.md",
 947             "README.md",
 948             "RELEASES.md",
 949             "configure",
 950             "x.py",
 951             "config.toml.example",
 952         ];
 953         let src_dirs = [
 954             "src",
 955         ];
 956
 957         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 958
 959         // Copy the files normally
 960         for item in &src_files {
 961             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 962         }
 963
 964         // Create the version file
 965         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 966         if let Some(sha) = builder.rust_sha() {
 967             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 968         }
 969
 970         // If we're building from git sources, we need to vendor a complete distribution.
 971         if builder.rust_info.is_git() {
 972             // Get cargo-vendor installed, if it isn't already.
 973             let mut has_cargo_vendor = false;
 974             let mut cmd = Command::new(&builder.initial_cargo);
 975             for line in output(cmd.arg("install").arg("--list")).lines() {
 976                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 977             }
 978             if !has_cargo_vendor {
 979                 let mut cmd = builder.cargo(
 980                     builder.compiler(0, builder.config.build),
 981                     Mode::ToolBootstrap,
 982                     builder.config.build,
 983                     "install"
 984                 );
 985                 cmd.arg("--force")
 986                    .arg("--debug")
 987                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 988                    .arg("cargo-vendor");
 989                 builder.run(&mut cmd);
 990             }
 991
 992             // Vendor all Cargo dependencies
 993             let mut cmd = Command::new(&builder.initial_cargo);
 994             cmd.arg("vendor")
 995                .current_dir(&plain_dst_src.join("src"));
 996             builder.run(&mut cmd);
 997         }
 998
 999         // Create plain source tarball
1000         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1001         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1002         tarball.set_extension(""); // strip .gz
1003         tarball.set_extension(""); // strip .tar
1004         if let Some(dir) = tarball.parent() {
1005             builder.create_dir(&dir);
1006         }
1007         builder.info("running installer");
1008         let mut cmd = rust_installer(builder);
1009         cmd.arg("tarball")
1010            .arg("--input").arg(&plain_name)
1011            .arg("--output").arg(&tarball)
1012            .arg("--work-dir=.")
1013            .current_dir(tmpdir(builder));
1014         builder.run(&mut cmd);
1015         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1016     }
1017 }
1018
1019 // We have to run a few shell scripts, which choke quite a bit on both `\`
1020 // characters and on `C:\` paths, so normalize both of them away.
1021 pub fn sanitize_sh(path: &Path) -> String {
1022     let path = path.to_str().unwrap().replace("\\", "/");
1023     return change_drive(&path).unwrap_or(path);
1024
1025     fn change_drive(s: &str) -> Option<String> {
1026         let mut ch = s.chars();
1027         let drive = ch.next().unwrap_or('C');
1028         if ch.next() != Some(':') {
1029             return None
1030         }
1031         if ch.next() != Some('/') {
1032             return None
1033         }
1034         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1035     }
1036 }
1037
1038 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1039 pub struct Cargo {
1040     pub stage: u32,
1041     pub target: Interned<String>,
1042 }
1043
1044 impl Step for Cargo {
1045     type Output = PathBuf;
1046     const ONLY_HOSTS: bool = true;
1047
1048     fn should_run(run: ShouldRun) -> ShouldRun {
1049         run.path("cargo")
1050     }
1051
1052     fn make_run(run: RunConfig) {
1053         run.builder.ensure(Cargo {
1054             stage: run.builder.top_stage,
1055             target: run.target,
1056         });
1057     }
1058
1059     fn run(self, builder: &Builder) -> PathBuf {
1060         let stage = self.stage;
1061         let target = self.target;
1062
1063         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1064         let src = builder.src.join("src/tools/cargo");
1065         let etc = src.join("src/etc");
1066         let release_num = builder.release_num("cargo");
1067         let name = pkgname(builder, "cargo");
1068         let version = builder.cargo_info.version(builder, &release_num);
1069
1070         let tmp = tmpdir(builder);
1071         let image = tmp.join("cargo-image");
1072         drop(fs::remove_dir_all(&image));
1073         builder.create_dir(&image);
1074
1075         // Prepare the image directory
1076         builder.create_dir(&image.join("share/zsh/site-functions"));
1077         builder.create_dir(&image.join("etc/bash_completion.d"));
1078         let cargo = builder.ensure(tool::Cargo {
1079             compiler: builder.compiler(stage, builder.config.build),
1080             target
1081         });
1082         builder.install(&cargo, &image.join("bin"), 0o755);
1083         for man in t!(etc.join("man").read_dir()) {
1084             let man = t!(man);
1085             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1086         }
1087         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1088         builder.copy(&etc.join("cargo.bashcomp.sh"),
1089              &image.join("etc/bash_completion.d/cargo"));
1090         let doc = image.join("share/doc/cargo");
1091         builder.install(&src.join("README.md"), &doc, 0o644);
1092         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1093         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1094         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1095
1096         // Prepare the overlay
1097         let overlay = tmp.join("cargo-overlay");
1098         drop(fs::remove_dir_all(&overlay));
1099         builder.create_dir(&overlay);
1100         builder.install(&src.join("README.md"), &overlay, 0o644);
1101         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1102         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1103         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1104         builder.create(&overlay.join("version"), &version);
1105
1106         // Generate the installer tarball
1107         let mut cmd = rust_installer(builder);
1108         cmd.arg("generate")
1109            .arg("--product-name=Rust")
1110            .arg("--rel-manifest-dir=rustlib")
1111            .arg("--success-message=Rust-is-ready-to-roll.")
1112            .arg("--image-dir").arg(&image)
1113            .arg("--work-dir").arg(&tmpdir(builder))
1114            .arg("--output-dir").arg(&distdir(builder))
1115            .arg("--non-installed-overlay").arg(&overlay)
1116            .arg(format!("--package-name={}-{}", name, target))
1117            .arg("--component-name=cargo")
1118            .arg("--legacy-manifest-dirs=rustlib,cargo");
1119         builder.run(&mut cmd);
1120         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1121     }
1122 }
1123
1124 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1125 pub struct Rls {
1126     pub stage: u32,
1127     pub target: Interned<String>,
1128 }
1129
1130 impl Step for Rls {
1131     type Output = Option<PathBuf>;
1132     const ONLY_HOSTS: bool = true;
1133
1134     fn should_run(run: ShouldRun) -> ShouldRun {
1135         run.path("rls")
1136     }
1137
1138     fn make_run(run: RunConfig) {
1139         run.builder.ensure(Rls {
1140             stage: run.builder.top_stage,
1141             target: run.target,
1142         });
1143     }
1144
1145     fn run(self, builder: &Builder) -> Option<PathBuf> {
1146         let stage = self.stage;
1147         let target = self.target;
1148         assert!(builder.config.extended);
1149
1150         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1151         let src = builder.src.join("src/tools/rls");
1152         let release_num = builder.release_num("rls");
1153         let name = pkgname(builder, "rls");
1154         let version = builder.rls_info.version(builder, &release_num);
1155
1156         let tmp = tmpdir(builder);
1157         let image = tmp.join("rls-image");
1158         drop(fs::remove_dir_all(&image));
1159         t!(fs::create_dir_all(&image));
1160
1161         // Prepare the image directory
1162         // We expect RLS to build, because we've exited this step above if tool
1163         // state for RLS isn't testing.
1164         let rls = builder.ensure(tool::Rls {
1165             compiler: builder.compiler(stage, builder.config.build),
1166             target, extra_features: Vec::new()
1167         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1168
1169         builder.install(&rls, &image.join("bin"), 0o755);
1170         let doc = image.join("share/doc/rls");
1171         builder.install(&src.join("README.md"), &doc, 0o644);
1172         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1173         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1174
1175         // Prepare the overlay
1176         let overlay = tmp.join("rls-overlay");
1177         drop(fs::remove_dir_all(&overlay));
1178         t!(fs::create_dir_all(&overlay));
1179         builder.install(&src.join("README.md"), &overlay, 0o644);
1180         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1181         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1182         builder.create(&overlay.join("version"), &version);
1183
1184         // Generate the installer tarball
1185         let mut cmd = rust_installer(builder);
1186         cmd.arg("generate")
1187            .arg("--product-name=Rust")
1188            .arg("--rel-manifest-dir=rustlib")
1189            .arg("--success-message=RLS-ready-to-serve.")
1190            .arg("--image-dir").arg(&image)
1191            .arg("--work-dir").arg(&tmpdir(builder))
1192            .arg("--output-dir").arg(&distdir(builder))
1193            .arg("--non-installed-overlay").arg(&overlay)
1194            .arg(format!("--package-name={}-{}", name, target))
1195            .arg("--legacy-manifest-dirs=rustlib,cargo")
1196            .arg("--component-name=rls-preview");
1197
1198         builder.run(&mut cmd);
1199         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1200     }
1201 }
1202
1203 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1204 pub struct Clippy {
1205     pub stage: u32,
1206     pub target: Interned<String>,
1207 }
1208
1209 impl Step for Clippy {
1210     type Output = Option<PathBuf>;
1211     const ONLY_HOSTS: bool = true;
1212
1213     fn should_run(run: ShouldRun) -> ShouldRun {
1214         run.path("clippy")
1215     }
1216
1217     fn make_run(run: RunConfig) {
1218         run.builder.ensure(Clippy {
1219             stage: run.builder.top_stage,
1220             target: run.target,
1221         });
1222     }
1223
1224     fn run(self, builder: &Builder) -> Option<PathBuf> {
1225         let stage = self.stage;
1226         let target = self.target;
1227         assert!(builder.config.extended);
1228
1229         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1230         let src = builder.src.join("src/tools/clippy");
1231         let release_num = builder.release_num("clippy");
1232         let name = pkgname(builder, "clippy");
1233         let version = builder.clippy_info.version(builder, &release_num);
1234
1235         let tmp = tmpdir(builder);
1236         let image = tmp.join("clippy-image");
1237         drop(fs::remove_dir_all(&image));
1238         builder.create_dir(&image);
1239
1240         // Prepare the image directory
1241         // We expect clippy to build, because we've exited this step above if tool
1242         // state for clippy isn't testing.
1243         let clippy = builder.ensure(tool::Clippy {
1244             compiler: builder.compiler(stage, builder.config.build),
1245             target, extra_features: Vec::new()
1246         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1247         let cargoclippy = builder.ensure(tool::CargoClippy {
1248             compiler: builder.compiler(stage, builder.config.build),
1249             target, extra_features: Vec::new()
1250         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1251
1252         builder.install(&clippy, &image.join("bin"), 0o755);
1253         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1254         let doc = image.join("share/doc/clippy");
1255         builder.install(&src.join("README.md"), &doc, 0o644);
1256         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1257         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1258
1259         // Prepare the overlay
1260         let overlay = tmp.join("clippy-overlay");
1261         drop(fs::remove_dir_all(&overlay));
1262         t!(fs::create_dir_all(&overlay));
1263         builder.install(&src.join("README.md"), &overlay, 0o644);
1264         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1265         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1266         builder.create(&overlay.join("version"), &version);
1267
1268         // Generate the installer tarball
1269         let mut cmd = rust_installer(builder);
1270         cmd.arg("generate")
1271            .arg("--product-name=Rust")
1272            .arg("--rel-manifest-dir=rustlib")
1273            .arg("--success-message=clippy-ready-to-serve.")
1274            .arg("--image-dir").arg(&image)
1275            .arg("--work-dir").arg(&tmpdir(builder))
1276            .arg("--output-dir").arg(&distdir(builder))
1277            .arg("--non-installed-overlay").arg(&overlay)
1278            .arg(format!("--package-name={}-{}", name, target))
1279            .arg("--legacy-manifest-dirs=rustlib,cargo")
1280            .arg("--component-name=clippy-preview");
1281
1282         builder.run(&mut cmd);
1283         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1284     }
1285 }
1286
1287 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1288 pub struct Rustfmt {
1289     pub stage: u32,
1290     pub target: Interned<String>,
1291 }
1292
1293 impl Step for Rustfmt {
1294     type Output = Option<PathBuf>;
1295     const ONLY_HOSTS: bool = true;
1296
1297     fn should_run(run: ShouldRun) -> ShouldRun {
1298         run.path("rustfmt")
1299     }
1300
1301     fn make_run(run: RunConfig) {
1302         run.builder.ensure(Rustfmt {
1303             stage: run.builder.top_stage,
1304             target: run.target,
1305         });
1306     }
1307
1308     fn run(self, builder: &Builder) -> Option<PathBuf> {
1309         let stage = self.stage;
1310         let target = self.target;
1311
1312         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1313         let src = builder.src.join("src/tools/rustfmt");
1314         let release_num = builder.release_num("rustfmt");
1315         let name = pkgname(builder, "rustfmt");
1316         let version = builder.rustfmt_info.version(builder, &release_num);
1317
1318         let tmp = tmpdir(builder);
1319         let image = tmp.join("rustfmt-image");
1320         drop(fs::remove_dir_all(&image));
1321         builder.create_dir(&image);
1322
1323         // Prepare the image directory
1324         let rustfmt = builder.ensure(tool::Rustfmt {
1325             compiler: builder.compiler(stage, builder.config.build),
1326             target, extra_features: Vec::new()
1327         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1328         let cargofmt = builder.ensure(tool::Cargofmt {
1329             compiler: builder.compiler(stage, builder.config.build),
1330             target, extra_features: Vec::new()
1331         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1332
1333         builder.install(&rustfmt, &image.join("bin"), 0o755);
1334         builder.install(&cargofmt, &image.join("bin"), 0o755);
1335         let doc = image.join("share/doc/rustfmt");
1336         builder.install(&src.join("README.md"), &doc, 0o644);
1337         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1338         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1339
1340         // Prepare the overlay
1341         let overlay = tmp.join("rustfmt-overlay");
1342         drop(fs::remove_dir_all(&overlay));
1343         builder.create_dir(&overlay);
1344         builder.install(&src.join("README.md"), &overlay, 0o644);
1345         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1346         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1347         builder.create(&overlay.join("version"), &version);
1348
1349         // Generate the installer tarball
1350         let mut cmd = rust_installer(builder);
1351         cmd.arg("generate")
1352            .arg("--product-name=Rust")
1353            .arg("--rel-manifest-dir=rustlib")
1354            .arg("--success-message=rustfmt-ready-to-fmt.")
1355            .arg("--image-dir").arg(&image)
1356            .arg("--work-dir").arg(&tmpdir(builder))
1357            .arg("--output-dir").arg(&distdir(builder))
1358            .arg("--non-installed-overlay").arg(&overlay)
1359            .arg(format!("--package-name={}-{}", name, target))
1360            .arg("--legacy-manifest-dirs=rustlib,cargo")
1361            .arg("--component-name=rustfmt-preview");
1362
1363         builder.run(&mut cmd);
1364         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1365     }
1366 }
1367
1368 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1369 pub struct Extended {
1370     stage: u32,
1371     host: Interned<String>,
1372     target: Interned<String>,
1373 }
1374
1375 impl Step for Extended {
1376     type Output = ();
1377     const DEFAULT: bool = true;
1378     const ONLY_HOSTS: bool = true;
1379
1380     fn should_run(run: ShouldRun) -> ShouldRun {
1381         let builder = run.builder;
1382         run.path("extended").default_condition(builder.config.extended)
1383     }
1384
1385     fn make_run(run: RunConfig) {
1386         run.builder.ensure(Extended {
1387             stage: run.builder.top_stage,
1388             host: run.builder.config.build,
1389             target: run.target,
1390         });
1391     }
1392
1393     /// Creates a combined installer for the specified target in the provided stage.
1394     fn run(self, builder: &Builder) {
1395         let stage = self.stage;
1396         let target = self.target;
1397
1398         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1399
1400         let rustc_installer = builder.ensure(Rustc {
1401             compiler: builder.compiler(stage, target),
1402         });
1403         let cargo_installer = builder.ensure(Cargo { stage, target });
1404         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1405         let rls_installer = builder.ensure(Rls { stage, target });
1406         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1407         let clippy_installer = builder.ensure(Clippy { stage, target });
1408         let lldb_installer = builder.ensure(Lldb { target });
1409         let mingw_installer = builder.ensure(Mingw { host: target });
1410         let analysis_installer = builder.ensure(Analysis {
1411             compiler: builder.compiler(stage, self.host),
1412             target
1413         });
1414
1415         let docs_installer = builder.ensure(Docs { stage, host: target, });
1416         let std_installer = builder.ensure(Std {
1417             compiler: builder.compiler(stage, self.host),
1418             target,
1419         });
1420
1421         let tmp = tmpdir(builder);
1422         let overlay = tmp.join("extended-overlay");
1423         let etc = builder.src.join("src/etc/installer");
1424         let work = tmp.join("work");
1425
1426         let _ = fs::remove_dir_all(&overlay);
1427         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1428         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1429         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1430         let version = builder.rust_version();
1431         builder.create(&overlay.join("version"), &version);
1432         if let Some(sha) = builder.rust_sha() {
1433             builder.create(&overlay.join("git-commit-hash"), &sha);
1434         }
1435         builder.install(&etc.join("README.md"), &overlay, 0o644);
1436
1437         // When rust-std package split from rustc, we needed to ensure that during
1438         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1439         // the std files during uninstall. To do this ensure that rustc comes
1440         // before rust-std in the list below.
1441         let mut tarballs = Vec::new();
1442         tarballs.push(rustc_installer);
1443         tarballs.push(cargo_installer);
1444         tarballs.extend(rls_installer.clone());
1445         tarballs.extend(clippy_installer.clone());
1446         tarballs.extend(rustfmt_installer.clone());
1447         tarballs.extend(llvm_tools_installer);
1448         tarballs.extend(lldb_installer);
1449         tarballs.push(analysis_installer);
1450         tarballs.push(std_installer);
1451         if builder.config.docs {
1452             tarballs.push(docs_installer);
1453         }
1454         if target.contains("pc-windows-gnu") {
1455             tarballs.push(mingw_installer.unwrap());
1456         }
1457         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1458         for tarball in &tarballs[1..] {
1459             input_tarballs.push(",");
1460             input_tarballs.push(tarball);
1461         }
1462
1463         let mut cmd = rust_installer(builder);
1464         cmd.arg("combine")
1465             .arg("--product-name=Rust")
1466             .arg("--rel-manifest-dir=rustlib")
1467             .arg("--success-message=Rust-is-ready-to-roll.")
1468             .arg("--work-dir").arg(&work)
1469             .arg("--output-dir").arg(&distdir(builder))
1470             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1471             .arg("--legacy-manifest-dirs=rustlib,cargo")
1472             .arg("--input-tarballs").arg(input_tarballs)
1473             .arg("--non-installed-overlay").arg(&overlay);
1474         builder.run(&mut cmd);
1475
1476         let mut license = String::new();
1477         license += &builder.read(&builder.src.join("COPYRIGHT"));
1478         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1479         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1480         license.push_str("\n");
1481         license.push_str("\n");
1482
1483         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1484         let mut rtf = rtf.to_string();
1485         rtf.push_str("\n");
1486         for line in license.lines() {
1487             rtf.push_str(line);
1488             rtf.push_str("\\line ");
1489         }
1490         rtf.push_str("}");
1491
1492         fn filter(contents: &str, marker: &str) -> String {
1493             let start = format!("tool-{}-start", marker);
1494             let end = format!("tool-{}-end", marker);
1495             let mut lines = Vec::new();
1496             let mut omitted = false;
1497             for line in contents.lines() {
1498                 if line.contains(&start) {
1499                     omitted = true;
1500                 } else if line.contains(&end) {
1501                     omitted = false;
1502                 } else if !omitted {
1503                     lines.push(line);
1504                 }
1505             }
1506
1507             lines.join("\n")
1508         }
1509
1510         let xform = |p: &Path| {
1511             let mut contents = String::new();
1512             t!(t!(File::open(p)).read_to_string(&mut contents));
1513             if rls_installer.is_none() {
1514                 contents = filter(&contents, "rls");
1515             }
1516             if clippy_installer.is_none() {
1517                 contents = filter(&contents, "clippy");
1518             }
1519             if rustfmt_installer.is_none() {
1520                 contents = filter(&contents, "rustfmt");
1521             }
1522             let ret = tmp.join(p.file_name().unwrap());
1523             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1524             return ret
1525         };
1526
1527         if target.contains("apple-darwin") {
1528             let pkg = tmp.join("pkg");
1529             let _ = fs::remove_dir_all(&pkg);
1530
1531             let pkgbuild = |component: &str| {
1532                 let mut cmd = Command::new("pkgbuild");
1533                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1534                     .arg("--scripts").arg(pkg.join(component))
1535                     .arg("--nopayload")
1536                     .arg(pkg.join(component).with_extension("pkg"));
1537                 builder.run(&mut cmd);
1538             };
1539
1540             let prepare = |name: &str| {
1541                 builder.create_dir(&pkg.join(name));
1542                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1543                         &pkg.join(name));
1544                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1545                 pkgbuild(name);
1546             };
1547             prepare("rustc");
1548             prepare("cargo");
1549             prepare("rust-docs");
1550             prepare("rust-std");
1551             prepare("rust-analysis");
1552
1553             if rls_installer.is_some() {
1554                 prepare("rls");
1555             }
1556             if clippy_installer.is_some() {
1557                 prepare("clippy");
1558             }
1559
1560             // create an 'uninstall' package
1561             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1562             pkgbuild("uninstall");
1563
1564             builder.create_dir(&pkg.join("res"));
1565             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1566             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1567             let mut cmd = Command::new("productbuild");
1568             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1569                 .arg("--resources").arg(pkg.join("res"))
1570                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1571                                                     pkgname(builder, "rust"),
1572                                                     target)))
1573                 .arg("--package-path").arg(&pkg);
1574             builder.run(&mut cmd);
1575         }
1576
1577         if target.contains("windows") {
1578             let exe = tmp.join("exe");
1579             let _ = fs::remove_dir_all(&exe);
1580
1581             let prepare = |name: &str| {
1582                 builder.create_dir(&exe.join(name));
1583                 let dir = if name == "rust-std" || name == "rust-analysis" {
1584                     format!("{}-{}", name, target)
1585                 } else if name == "rls" {
1586                     "rls-preview".to_string()
1587                 } else if name == "clippy" {
1588                     "clippy-preview".to_string()
1589                 } else {
1590                     name.to_string()
1591                 };
1592                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1593                             .join(dir),
1594                         &exe.join(name));
1595                 builder.remove(&exe.join(name).join("manifest.in"));
1596             };
1597             prepare("rustc");
1598             prepare("cargo");
1599             prepare("rust-analysis");
1600             prepare("rust-docs");
1601             prepare("rust-std");
1602             if rls_installer.is_some() {
1603                 prepare("rls");
1604             }
1605             if clippy_installer.is_some() {
1606                 prepare("clippy");
1607             }
1608             if target.contains("windows-gnu") {
1609                 prepare("rust-mingw");
1610             }
1611
1612             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1613             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1614             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1615             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1616             builder.create(&exe.join("LICENSE.txt"), &license);
1617
1618             // Generate exe installer
1619             let mut cmd = Command::new("iscc");
1620             cmd.arg("rust.iss")
1621                 .current_dir(&exe);
1622             if target.contains("windows-gnu") {
1623                 cmd.arg("/dMINGW");
1624             }
1625             add_env(builder, &mut cmd, target);
1626             builder.run(&mut cmd);
1627             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1628                     &distdir(builder),
1629                     0o755);
1630
1631             // Generate msi installer
1632             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1633             let heat = wix.join("bin/heat.exe");
1634             let candle = wix.join("bin/candle.exe");
1635             let light = wix.join("bin/light.exe");
1636
1637             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1638             builder.run(Command::new(&heat)
1639                             .current_dir(&exe)
1640                             .arg("dir")
1641                             .arg("rustc")
1642                             .args(&heat_flags)
1643                             .arg("-cg").arg("RustcGroup")
1644                             .arg("-dr").arg("Rustc")
1645                             .arg("-var").arg("var.RustcDir")
1646                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1647             builder.run(Command::new(&heat)
1648                             .current_dir(&exe)
1649                             .arg("dir")
1650                             .arg("rust-docs")
1651                             .args(&heat_flags)
1652                             .arg("-cg").arg("DocsGroup")
1653                             .arg("-dr").arg("Docs")
1654                             .arg("-var").arg("var.DocsDir")
1655                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1656                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1657             builder.run(Command::new(&heat)
1658                             .current_dir(&exe)
1659                             .arg("dir")
1660                             .arg("cargo")
1661                             .args(&heat_flags)
1662                             .arg("-cg").arg("CargoGroup")
1663                             .arg("-dr").arg("Cargo")
1664                             .arg("-var").arg("var.CargoDir")
1665                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1666                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1667             builder.run(Command::new(&heat)
1668                             .current_dir(&exe)
1669                             .arg("dir")
1670                             .arg("rust-std")
1671                             .args(&heat_flags)
1672                             .arg("-cg").arg("StdGroup")
1673                             .arg("-dr").arg("Std")
1674                             .arg("-var").arg("var.StdDir")
1675                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1676             if rls_installer.is_some() {
1677                 builder.run(Command::new(&heat)
1678                                 .current_dir(&exe)
1679                                 .arg("dir")
1680                                 .arg("rls")
1681                                 .args(&heat_flags)
1682                                 .arg("-cg").arg("RlsGroup")
1683                                 .arg("-dr").arg("Rls")
1684                                 .arg("-var").arg("var.RlsDir")
1685                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1686                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1687             }
1688             if clippy_installer.is_some() {
1689                 builder.run(Command::new(&heat)
1690                                 .current_dir(&exe)
1691                                 .arg("dir")
1692                                 .arg("clippy")
1693                                 .args(&heat_flags)
1694                                 .arg("-cg").arg("ClippyGroup")
1695                                 .arg("-dr").arg("Clippy")
1696                                 .arg("-var").arg("var.ClippyDir")
1697                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1698                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1699             }
1700             builder.run(Command::new(&heat)
1701                             .current_dir(&exe)
1702                             .arg("dir")
1703                             .arg("rust-analysis")
1704                             .args(&heat_flags)
1705                             .arg("-cg").arg("AnalysisGroup")
1706                             .arg("-dr").arg("Analysis")
1707                             .arg("-var").arg("var.AnalysisDir")
1708                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1709                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1710             if target.contains("windows-gnu") {
1711                 builder.run(Command::new(&heat)
1712                                 .current_dir(&exe)
1713                                 .arg("dir")
1714                                 .arg("rust-mingw")
1715                                 .args(&heat_flags)
1716                                 .arg("-cg").arg("GccGroup")
1717                                 .arg("-dr").arg("Gcc")
1718                                 .arg("-var").arg("var.GccDir")
1719                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1720             }
1721
1722             let candle = |input: &Path| {
1723                 let output = exe.join(input.file_stem().unwrap())
1724                                 .with_extension("wixobj");
1725                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1726                 let mut cmd = Command::new(&candle);
1727                 cmd.current_dir(&exe)
1728                     .arg("-nologo")
1729                     .arg("-dRustcDir=rustc")
1730                     .arg("-dDocsDir=rust-docs")
1731                     .arg("-dCargoDir=cargo")
1732                     .arg("-dStdDir=rust-std")
1733                     .arg("-dAnalysisDir=rust-analysis")
1734                     .arg("-arch").arg(&arch)
1735                     .arg("-out").arg(&output)
1736                     .arg(&input);
1737                 add_env(builder, &mut cmd, target);
1738
1739                 if rls_installer.is_some() {
1740                     cmd.arg("-dRlsDir=rls");
1741                 }
1742                 if clippy_installer.is_some() {
1743                     cmd.arg("-dClippyDir=clippy");
1744                 }
1745                 if target.contains("windows-gnu") {
1746                     cmd.arg("-dGccDir=rust-mingw");
1747                 }
1748                 builder.run(&mut cmd);
1749             };
1750             candle(&xform(&etc.join("msi/rust.wxs")));
1751             candle(&etc.join("msi/ui.wxs"));
1752             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1753             candle("RustcGroup.wxs".as_ref());
1754             candle("DocsGroup.wxs".as_ref());
1755             candle("CargoGroup.wxs".as_ref());
1756             candle("StdGroup.wxs".as_ref());
1757             if rls_installer.is_some() {
1758                 candle("RlsGroup.wxs".as_ref());
1759             }
1760             if clippy_installer.is_some() {
1761                 candle("ClippyGroup.wxs".as_ref());
1762             }
1763             candle("AnalysisGroup.wxs".as_ref());
1764
1765             if target.contains("windows-gnu") {
1766                 candle("GccGroup.wxs".as_ref());
1767             }
1768
1769             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1770             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1771             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1772
1773             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1774             let mut cmd = Command::new(&light);
1775             cmd.arg("-nologo")
1776                 .arg("-ext").arg("WixUIExtension")
1777                 .arg("-ext").arg("WixUtilExtension")
1778                 .arg("-out").arg(exe.join(&filename))
1779                 .arg("rust.wixobj")
1780                 .arg("ui.wixobj")
1781                 .arg("rustwelcomedlg.wixobj")
1782                 .arg("RustcGroup.wixobj")
1783                 .arg("DocsGroup.wixobj")
1784                 .arg("CargoGroup.wixobj")
1785                 .arg("StdGroup.wixobj")
1786                 .arg("AnalysisGroup.wixobj")
1787                 .current_dir(&exe);
1788
1789             if rls_installer.is_some() {
1790                 cmd.arg("RlsGroup.wixobj");
1791             }
1792             if clippy_installer.is_some() {
1793                 cmd.arg("ClippyGroup.wixobj");
1794             }
1795
1796             if target.contains("windows-gnu") {
1797                 cmd.arg("GccGroup.wixobj");
1798             }
1799             // ICE57 wrongly complains about the shortcuts
1800             cmd.arg("-sice:ICE57");
1801
1802             builder.run(&mut cmd);
1803
1804             if !builder.config.dry_run {
1805                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1806             }
1807         }
1808     }
1809 }
1810
1811 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1812     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1813     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1814        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1815        .env("CFG_RELEASE", builder.rust_release())
1816        .env("CFG_VER_MAJOR", parts.next().unwrap())
1817        .env("CFG_VER_MINOR", parts.next().unwrap())
1818        .env("CFG_VER_PATCH", parts.next().unwrap())
1819        .env("CFG_VER_BUILD", "0") // just needed to build
1820        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1821        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1822        .env("CFG_BUILD", target)
1823        .env("CFG_CHANNEL", &builder.config.channel);
1824
1825     if target.contains("windows-gnu") {
1826        cmd.env("CFG_MINGW", "1")
1827           .env("CFG_ABI", "GNU");
1828     } else {
1829        cmd.env("CFG_MINGW", "0")
1830           .env("CFG_ABI", "MSVC");
1831     }
1832
1833     if target.contains("x86_64") {
1834        cmd.env("CFG_PLATFORM", "x64");
1835     } else {
1836        cmd.env("CFG_PLATFORM", "x86");
1837     }
1838 }
1839
1840 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1841 pub struct HashSign;
1842
1843 impl Step for HashSign {
1844     type Output = ();
1845     const ONLY_HOSTS: bool = true;
1846
1847     fn should_run(run: ShouldRun) -> ShouldRun {
1848         run.path("hash-and-sign")
1849     }
1850
1851     fn make_run(run: RunConfig) {
1852         run.builder.ensure(HashSign);
1853     }
1854
1855     fn run(self, builder: &Builder) {
1856         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1857         if builder.config.dry_run {
1858             return;
1859         }
1860         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1861             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1862         });
1863         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1864             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1865         });
1866         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1867             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1868         });
1869         let mut pass = String::new();
1870         t!(t!(File::open(&file)).read_to_string(&mut pass));
1871
1872         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1873
1874         cmd.arg(sign);
1875         cmd.arg(distdir(builder));
1876         cmd.arg(today.trim());
1877         cmd.arg(builder.rust_package_vers());
1878         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1879         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1880         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1881         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1882         cmd.arg(builder.llvm_tools_package_vers());
1883         cmd.arg(builder.lldb_package_vers());
1884         cmd.arg(addr);
1885
1886         builder.create_dir(&distdir(builder));
1887
1888         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1889         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1890         let status = t!(child.wait());
1891         assert!(status.success());
1892     }
1893 }
1894
1895 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1896 // LLVM tools are linked dynamically.
1897 // Note: This function does no yet support Windows but we also don't support
1898 //       linking LLVM tools dynamically on Windows yet.
1899 fn maybe_install_llvm_dylib(builder: &Builder,
1900                             target: Interned<String>,
1901                             image: &Path) {
1902     let src_libdir = builder
1903         .llvm_out(target)
1904         .join("lib");
1905     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1906     t!(fs::create_dir_all(&dst_libdir));
1907
1908     if target.contains("apple-darwin") {
1909         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1910         if llvm_dylib_path.exists() {
1911             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1912         }
1913         return
1914     }
1915
1916     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1917     // Since tools link to the latter rather than the former, we have to
1918     // follow the symlink to find out what to distribute.
1919     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1920     if llvm_dylib_path.exists() {
1921         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1922             panic!("dist: Error calling canonicalize path `{}`: {}",
1923                    llvm_dylib_path.display(), e);
1924         });
1925
1926
1927         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1928     }
1929 }
1930
1931 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1932 pub struct LlvmTools {
1933     pub stage: u32,
1934     pub target: Interned<String>,
1935 }
1936
1937 impl Step for LlvmTools {
1938     type Output = Option<PathBuf>;
1939     const ONLY_HOSTS: bool = true;
1940
1941     fn should_run(run: ShouldRun) -> ShouldRun {
1942         run.path("llvm-tools")
1943     }
1944
1945     fn make_run(run: RunConfig) {
1946         run.builder.ensure(LlvmTools {
1947             stage: run.builder.top_stage,
1948             target: run.target,
1949         });
1950     }
1951
1952     fn run(self, builder: &Builder) -> Option<PathBuf> {
1953         let stage = self.stage;
1954         let target = self.target;
1955         assert!(builder.config.extended);
1956
1957         /* run only if llvm-config isn't used */
1958         if let Some(config) = builder.config.target_config.get(&target) {
1959             if let Some(ref _s) = config.llvm_config {
1960                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1961                     stage, target));
1962                 return None;
1963             }
1964         }
1965
1966         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1967         let src = builder.src.join("src/llvm");
1968         let name = pkgname(builder, "llvm-tools");
1969
1970         let tmp = tmpdir(builder);
1971         let image = tmp.join("llvm-tools-image");
1972         drop(fs::remove_dir_all(&image));
1973
1974         // Prepare the image directory
1975         let src_bindir = builder
1976             .llvm_out(target)
1977             .join("bin");
1978         let dst_bindir = image.join("lib/rustlib")
1979             .join(&*target)
1980             .join("bin");
1981         t!(fs::create_dir_all(&dst_bindir));
1982         for tool in LLVM_TOOLS {
1983             let exe = src_bindir.join(exe(tool, &target));
1984             builder.install(&exe, &dst_bindir, 0o755);
1985         }
1986
1987         // Prepare the overlay
1988         let overlay = tmp.join("llvm-tools-overlay");
1989         drop(fs::remove_dir_all(&overlay));
1990         builder.create_dir(&overlay);
1991         builder.install(&src.join("README.txt"), &overlay, 0o644);
1992         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1993         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1994
1995         // Generate the installer tarball
1996         let mut cmd = rust_installer(builder);
1997         cmd.arg("generate")
1998             .arg("--product-name=Rust")
1999             .arg("--rel-manifest-dir=rustlib")
2000             .arg("--success-message=llvm-tools-installed.")
2001             .arg("--image-dir").arg(&image)
2002             .arg("--work-dir").arg(&tmpdir(builder))
2003             .arg("--output-dir").arg(&distdir(builder))
2004             .arg("--non-installed-overlay").arg(&overlay)
2005             .arg(format!("--package-name={}-{}", name, target))
2006             .arg("--legacy-manifest-dirs=rustlib,cargo")
2007             .arg("--component-name=llvm-tools-preview");
2008
2009
2010         builder.run(&mut cmd);
2011         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2012     }
2013 }
2014
2015 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2016 pub struct Lldb {
2017     pub target: Interned<String>,
2018 }
2019
2020 impl Step for Lldb {
2021     type Output = Option<PathBuf>;
2022     const ONLY_HOSTS: bool = true;
2023     const DEFAULT: bool = true;
2024
2025     fn should_run(run: ShouldRun) -> ShouldRun {
2026         run.path("src/tools/lldb")
2027     }
2028
2029     fn make_run(run: RunConfig) {
2030         run.builder.ensure(Lldb {
2031             target: run.target,
2032         });
2033     }
2034
2035     fn run(self, builder: &Builder) -> Option<PathBuf> {
2036         let target = self.target;
2037
2038         if builder.config.dry_run {
2039             return None;
2040         }
2041
2042         let bindir = builder
2043             .llvm_out(target)
2044             .join("bin");
2045         let lldb_exe = bindir.join(exe("lldb", &target));
2046         if !lldb_exe.exists() {
2047             return None;
2048         }
2049
2050         builder.info(&format!("Dist Lldb ({})", target));
2051         let src = builder.src.join("src/tools/lldb");
2052         let name = pkgname(builder, "lldb");
2053
2054         let tmp = tmpdir(builder);
2055         let image = tmp.join("lldb-image");
2056         drop(fs::remove_dir_all(&image));
2057
2058         // Prepare the image directory
2059         let root = image.join("lib/rustlib").join(&*target);
2060         let dst = root.join("bin");
2061         t!(fs::create_dir_all(&dst));
2062         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2063             let exe = bindir.join(exe(program, &target));
2064             builder.install(&exe, &dst, 0o755);
2065         }
2066
2067         // The libraries.
2068         let libdir = builder.llvm_out(target).join("lib");
2069         let dst = root.join("lib");
2070         t!(fs::create_dir_all(&dst));
2071         for entry in t!(fs::read_dir(&libdir)) {
2072             let entry = entry.unwrap();
2073             if let Ok(name) = entry.file_name().into_string() {
2074                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2075                     if t!(entry.file_type()).is_symlink() {
2076                         builder.copy_to_folder(&entry.path(), &dst);
2077                     } else {
2078                        builder.install(&entry.path(), &dst, 0o755);
2079                     }
2080                 }
2081             }
2082         }
2083
2084         // The lldb scripts might be installed in lib/python$version
2085         // or in lib64/python$version.  If lib64 exists, use it;
2086         // otherwise lib.
2087         let libdir = builder.llvm_out(target).join("lib64");
2088         let (libdir, libdir_name) = if libdir.exists() {
2089             (libdir, "lib64")
2090         } else {
2091             (builder.llvm_out(target).join("lib"), "lib")
2092         };
2093         for entry in t!(fs::read_dir(&libdir)) {
2094             let entry = t!(entry);
2095             if let Ok(name) = entry.file_name().into_string() {
2096                 if name.starts_with("python") {
2097                     let dst = root.join(libdir_name)
2098                         .join(entry.file_name());
2099                     t!(fs::create_dir_all(&dst));
2100                     builder.cp_r(&entry.path(), &dst);
2101                     break;
2102                 }
2103             }
2104         }
2105
2106         // Prepare the overlay
2107         let overlay = tmp.join("lldb-overlay");
2108         drop(fs::remove_dir_all(&overlay));
2109         builder.create_dir(&overlay);
2110         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2111         builder.create(&overlay.join("version"), &builder.lldb_vers());
2112
2113         // Generate the installer tarball
2114         let mut cmd = rust_installer(builder);
2115         cmd.arg("generate")
2116             .arg("--product-name=Rust")
2117             .arg("--rel-manifest-dir=rustlib")
2118             .arg("--success-message=lldb-installed.")
2119             .arg("--image-dir").arg(&image)
2120             .arg("--work-dir").arg(&tmpdir(builder))
2121             .arg("--output-dir").arg(&distdir(builder))
2122             .arg("--non-installed-overlay").arg(&overlay)
2123             .arg(format!("--package-name={}-{}", name, target))
2124             .arg("--legacy-manifest-dirs=rustlib,cargo")
2125             .arg("--component-name=lldb-preview");
2126
2127
2128         builder.run(&mut cmd);
2129         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2130     }
2131 }