src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Compiler, Mode, LLVM_TOOLS};
  30 use channel;
  31 use util::{libdir, is_dylib, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/dlmalloc",
 861             "src/liballoc",
 862             "src/libbacktrace",
 863             "src/libcompiler_builtins",
 864             "src/libcore",
 865             "src/liblibc",
 866             "src/libpanic_abort",
 867             "src/libpanic_unwind",
 868             "src/librustc_asan",
 869             "src/librustc_lsan",
 870             "src/librustc_msan",
 871             "src/librustc_tsan",
 872             "src/libstd",
 873             "src/libunwind",
 874             "src/rustc/compiler_builtins_shim",
 875             "src/rustc/libc_shim",
 876             "src/rustc/dlmalloc_shim",
 877             "src/rustc/fortanix-sgx-abi_shim",
 878             "src/libtest",
 879             "src/libterm",
 880             "src/libprofiler_builtins",
 881             "src/stdsimd",
 882             "src/libproc_macro",
 883         ];
 884         let std_src_dirs_exclude = [
 885             "src/libcompiler_builtins/compiler-rt/test",
 886         ];
 887
 888         copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 889         for file in src_files.iter() {
 890             builder.copy(&builder.src.join(file), &dst_src.join(file));
 891         }
 892
 893         // Create source tarball in rust-installer format
 894         let mut cmd = rust_installer(builder);
 895         cmd.arg("generate")
 896            .arg("--product-name=Rust")
 897            .arg("--rel-manifest-dir=rustlib")
 898            .arg("--success-message=Awesome-Source.")
 899            .arg("--image-dir").arg(&image)
 900            .arg("--work-dir").arg(&tmpdir(builder))
 901            .arg("--output-dir").arg(&distdir(builder))
 902            .arg(format!("--package-name={}", name))
 903            .arg("--component-name=rust-src")
 904            .arg("--legacy-manifest-dirs=rustlib,cargo");
 905         builder.run(&mut cmd);
 906
 907         builder.remove_dir(&image);
 908         distdir(builder).join(&format!("{}.tar.gz", name))
 909     }
 910 }
 911
 912 const CARGO_VENDOR_VERSION: &str = "0.1.19";
 913
 914 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 915 pub struct PlainSourceTarball;
 916
 917 impl Step for PlainSourceTarball {
 918     /// Produces the location of the tarball generated
 919     type Output = PathBuf;
 920     const DEFAULT: bool = true;
 921     const ONLY_HOSTS: bool = true;
 922
 923     fn should_run(run: ShouldRun) -> ShouldRun {
 924         let builder = run.builder;
 925         run.path("src").default_condition(builder.config.rust_dist_src)
 926     }
 927
 928     fn make_run(run: RunConfig) {
 929         run.builder.ensure(PlainSourceTarball);
 930     }
 931
 932     /// Creates the plain source tarball
 933     fn run(self, builder: &Builder) -> PathBuf {
 934         builder.info("Create plain source tarball");
 935
 936         // Make sure that the root folder of tarball has the correct name
 937         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 938         let plain_dst_src = tmpdir(builder).join(&plain_name);
 939         let _ = fs::remove_dir_all(&plain_dst_src);
 940         t!(fs::create_dir_all(&plain_dst_src));
 941
 942         // This is the set of root paths which will become part of the source package
 943         let src_files = [
 944             "COPYRIGHT",
 945             "LICENSE-APACHE",
 946             "LICENSE-MIT",
 947             "CONTRIBUTING.md",
 948             "README.md",
 949             "RELEASES.md",
 950             "configure",
 951             "x.py",
 952             "config.toml.example",
 953             "Cargo.toml",
 954             "Cargo.lock",
 955         ];
 956         let src_dirs = [
 957             "src",
 958         ];
 959
 960         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 961
 962         // Copy the files normally
 963         for item in &src_files {
 964             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 965         }
 966
 967         // Create the version file
 968         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 969         if let Some(sha) = builder.rust_sha() {
 970             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 971         }
 972
 973         // If we're building from git sources, we need to vendor a complete distribution.
 974         if builder.rust_info.is_git() {
 975             // Get cargo-vendor installed, if it isn't already.
 976             let mut has_cargo_vendor = false;
 977             let mut cmd = Command::new(&builder.initial_cargo);
 978             for line in output(cmd.arg("install").arg("--list")).lines() {
 979                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 980             }
 981             if !has_cargo_vendor {
 982                 let mut cmd = builder.cargo(
 983                     builder.compiler(0, builder.config.build),
 984                     Mode::ToolBootstrap,
 985                     builder.config.build,
 986                     "install"
 987                 );
 988                 cmd.arg("--force")
 989                    .arg("--debug")
 990                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 991                    .arg("cargo-vendor");
 992                 builder.run(&mut cmd);
 993             }
 994
 995             // Vendor all Cargo dependencies
 996             let mut cmd = Command::new(&builder.initial_cargo);
 997             cmd.arg("vendor")
 998                .current_dir(&plain_dst_src);
 999             builder.run(&mut cmd);
1000         }
1001
1002         // Create plain source tarball
1003         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1004         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1005         tarball.set_extension(""); // strip .gz
1006         tarball.set_extension(""); // strip .tar
1007         if let Some(dir) = tarball.parent() {
1008             builder.create_dir(&dir);
1009         }
1010         builder.info("running installer");
1011         let mut cmd = rust_installer(builder);
1012         cmd.arg("tarball")
1013            .arg("--input").arg(&plain_name)
1014            .arg("--output").arg(&tarball)
1015            .arg("--work-dir=.")
1016            .current_dir(tmpdir(builder));
1017         builder.run(&mut cmd);
1018         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1019     }
1020 }
1021
1022 // We have to run a few shell scripts, which choke quite a bit on both `\`
1023 // characters and on `C:\` paths, so normalize both of them away.
1024 pub fn sanitize_sh(path: &Path) -> String {
1025     let path = path.to_str().unwrap().replace("\\", "/");
1026     return change_drive(&path).unwrap_or(path);
1027
1028     fn change_drive(s: &str) -> Option<String> {
1029         let mut ch = s.chars();
1030         let drive = ch.next().unwrap_or('C');
1031         if ch.next() != Some(':') {
1032             return None
1033         }
1034         if ch.next() != Some('/') {
1035             return None
1036         }
1037         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1038     }
1039 }
1040
1041 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1042 pub struct Cargo {
1043     pub stage: u32,
1044     pub target: Interned<String>,
1045 }
1046
1047 impl Step for Cargo {
1048     type Output = PathBuf;
1049     const ONLY_HOSTS: bool = true;
1050
1051     fn should_run(run: ShouldRun) -> ShouldRun {
1052         run.path("cargo")
1053     }
1054
1055     fn make_run(run: RunConfig) {
1056         run.builder.ensure(Cargo {
1057             stage: run.builder.top_stage,
1058             target: run.target,
1059         });
1060     }
1061
1062     fn run(self, builder: &Builder) -> PathBuf {
1063         let stage = self.stage;
1064         let target = self.target;
1065
1066         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1067         let src = builder.src.join("src/tools/cargo");
1068         let etc = src.join("src/etc");
1069         let release_num = builder.release_num("cargo");
1070         let name = pkgname(builder, "cargo");
1071         let version = builder.cargo_info.version(builder, &release_num);
1072
1073         let tmp = tmpdir(builder);
1074         let image = tmp.join("cargo-image");
1075         drop(fs::remove_dir_all(&image));
1076         builder.create_dir(&image);
1077
1078         // Prepare the image directory
1079         builder.create_dir(&image.join("share/zsh/site-functions"));
1080         builder.create_dir(&image.join("etc/bash_completion.d"));
1081         let cargo = builder.ensure(tool::Cargo {
1082             compiler: builder.compiler(stage, builder.config.build),
1083             target
1084         });
1085         builder.install(&cargo, &image.join("bin"), 0o755);
1086         for man in t!(etc.join("man").read_dir()) {
1087             let man = t!(man);
1088             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1089         }
1090         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1091         builder.copy(&etc.join("cargo.bashcomp.sh"),
1092              &image.join("etc/bash_completion.d/cargo"));
1093         let doc = image.join("share/doc/cargo");
1094         builder.install(&src.join("README.md"), &doc, 0o644);
1095         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1096         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1097         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1098
1099         // Prepare the overlay
1100         let overlay = tmp.join("cargo-overlay");
1101         drop(fs::remove_dir_all(&overlay));
1102         builder.create_dir(&overlay);
1103         builder.install(&src.join("README.md"), &overlay, 0o644);
1104         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1105         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1106         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1107         builder.create(&overlay.join("version"), &version);
1108
1109         // Generate the installer tarball
1110         let mut cmd = rust_installer(builder);
1111         cmd.arg("generate")
1112            .arg("--product-name=Rust")
1113            .arg("--rel-manifest-dir=rustlib")
1114            .arg("--success-message=Rust-is-ready-to-roll.")
1115            .arg("--image-dir").arg(&image)
1116            .arg("--work-dir").arg(&tmpdir(builder))
1117            .arg("--output-dir").arg(&distdir(builder))
1118            .arg("--non-installed-overlay").arg(&overlay)
1119            .arg(format!("--package-name={}-{}", name, target))
1120            .arg("--component-name=cargo")
1121            .arg("--legacy-manifest-dirs=rustlib,cargo");
1122         builder.run(&mut cmd);
1123         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1124     }
1125 }
1126
1127 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1128 pub struct Rls {
1129     pub stage: u32,
1130     pub target: Interned<String>,
1131 }
1132
1133 impl Step for Rls {
1134     type Output = Option<PathBuf>;
1135     const ONLY_HOSTS: bool = true;
1136
1137     fn should_run(run: ShouldRun) -> ShouldRun {
1138         run.path("rls")
1139     }
1140
1141     fn make_run(run: RunConfig) {
1142         run.builder.ensure(Rls {
1143             stage: run.builder.top_stage,
1144             target: run.target,
1145         });
1146     }
1147
1148     fn run(self, builder: &Builder) -> Option<PathBuf> {
1149         let stage = self.stage;
1150         let target = self.target;
1151         assert!(builder.config.extended);
1152
1153         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1154         let src = builder.src.join("src/tools/rls");
1155         let release_num = builder.release_num("rls");
1156         let name = pkgname(builder, "rls");
1157         let version = builder.rls_info.version(builder, &release_num);
1158
1159         let tmp = tmpdir(builder);
1160         let image = tmp.join("rls-image");
1161         drop(fs::remove_dir_all(&image));
1162         t!(fs::create_dir_all(&image));
1163
1164         // Prepare the image directory
1165         // We expect RLS to build, because we've exited this step above if tool
1166         // state for RLS isn't testing.
1167         let rls = builder.ensure(tool::Rls {
1168             compiler: builder.compiler(stage, builder.config.build),
1169             target, extra_features: Vec::new()
1170         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1171
1172         builder.install(&rls, &image.join("bin"), 0o755);
1173         let doc = image.join("share/doc/rls");
1174         builder.install(&src.join("README.md"), &doc, 0o644);
1175         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1176         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1177
1178         // Prepare the overlay
1179         let overlay = tmp.join("rls-overlay");
1180         drop(fs::remove_dir_all(&overlay));
1181         t!(fs::create_dir_all(&overlay));
1182         builder.install(&src.join("README.md"), &overlay, 0o644);
1183         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1184         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1185         builder.create(&overlay.join("version"), &version);
1186
1187         // Generate the installer tarball
1188         let mut cmd = rust_installer(builder);
1189         cmd.arg("generate")
1190            .arg("--product-name=Rust")
1191            .arg("--rel-manifest-dir=rustlib")
1192            .arg("--success-message=RLS-ready-to-serve.")
1193            .arg("--image-dir").arg(&image)
1194            .arg("--work-dir").arg(&tmpdir(builder))
1195            .arg("--output-dir").arg(&distdir(builder))
1196            .arg("--non-installed-overlay").arg(&overlay)
1197            .arg(format!("--package-name={}-{}", name, target))
1198            .arg("--legacy-manifest-dirs=rustlib,cargo")
1199            .arg("--component-name=rls-preview");
1200
1201         builder.run(&mut cmd);
1202         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1203     }
1204 }
1205
1206 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1207 pub struct Clippy {
1208     pub stage: u32,
1209     pub target: Interned<String>,
1210 }
1211
1212 impl Step for Clippy {
1213     type Output = Option<PathBuf>;
1214     const ONLY_HOSTS: bool = true;
1215
1216     fn should_run(run: ShouldRun) -> ShouldRun {
1217         run.path("clippy")
1218     }
1219
1220     fn make_run(run: RunConfig) {
1221         run.builder.ensure(Clippy {
1222             stage: run.builder.top_stage,
1223             target: run.target,
1224         });
1225     }
1226
1227     fn run(self, builder: &Builder) -> Option<PathBuf> {
1228         let stage = self.stage;
1229         let target = self.target;
1230         assert!(builder.config.extended);
1231
1232         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1233         let src = builder.src.join("src/tools/clippy");
1234         let release_num = builder.release_num("clippy");
1235         let name = pkgname(builder, "clippy");
1236         let version = builder.clippy_info.version(builder, &release_num);
1237
1238         let tmp = tmpdir(builder);
1239         let image = tmp.join("clippy-image");
1240         drop(fs::remove_dir_all(&image));
1241         builder.create_dir(&image);
1242
1243         // Prepare the image directory
1244         // We expect clippy to build, because we've exited this step above if tool
1245         // state for clippy isn't testing.
1246         let clippy = builder.ensure(tool::Clippy {
1247             compiler: builder.compiler(stage, builder.config.build),
1248             target, extra_features: Vec::new()
1249         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1250         let cargoclippy = builder.ensure(tool::CargoClippy {
1251             compiler: builder.compiler(stage, builder.config.build),
1252             target, extra_features: Vec::new()
1253         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1254
1255         builder.install(&clippy, &image.join("bin"), 0o755);
1256         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1257         let doc = image.join("share/doc/clippy");
1258         builder.install(&src.join("README.md"), &doc, 0o644);
1259         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1260         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1261
1262         // Prepare the overlay
1263         let overlay = tmp.join("clippy-overlay");
1264         drop(fs::remove_dir_all(&overlay));
1265         t!(fs::create_dir_all(&overlay));
1266         builder.install(&src.join("README.md"), &overlay, 0o644);
1267         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1268         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1269         builder.create(&overlay.join("version"), &version);
1270
1271         // Generate the installer tarball
1272         let mut cmd = rust_installer(builder);
1273         cmd.arg("generate")
1274            .arg("--product-name=Rust")
1275            .arg("--rel-manifest-dir=rustlib")
1276            .arg("--success-message=clippy-ready-to-serve.")
1277            .arg("--image-dir").arg(&image)
1278            .arg("--work-dir").arg(&tmpdir(builder))
1279            .arg("--output-dir").arg(&distdir(builder))
1280            .arg("--non-installed-overlay").arg(&overlay)
1281            .arg(format!("--package-name={}-{}", name, target))
1282            .arg("--legacy-manifest-dirs=rustlib,cargo")
1283            .arg("--component-name=clippy-preview");
1284
1285         builder.run(&mut cmd);
1286         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1287     }
1288 }
1289
1290 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1291 pub struct Rustfmt {
1292     pub stage: u32,
1293     pub target: Interned<String>,
1294 }
1295
1296 impl Step for Rustfmt {
1297     type Output = Option<PathBuf>;
1298     const ONLY_HOSTS: bool = true;
1299
1300     fn should_run(run: ShouldRun) -> ShouldRun {
1301         run.path("rustfmt")
1302     }
1303
1304     fn make_run(run: RunConfig) {
1305         run.builder.ensure(Rustfmt {
1306             stage: run.builder.top_stage,
1307             target: run.target,
1308         });
1309     }
1310
1311     fn run(self, builder: &Builder) -> Option<PathBuf> {
1312         let stage = self.stage;
1313         let target = self.target;
1314
1315         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1316         let src = builder.src.join("src/tools/rustfmt");
1317         let release_num = builder.release_num("rustfmt");
1318         let name = pkgname(builder, "rustfmt");
1319         let version = builder.rustfmt_info.version(builder, &release_num);
1320
1321         let tmp = tmpdir(builder);
1322         let image = tmp.join("rustfmt-image");
1323         drop(fs::remove_dir_all(&image));
1324         builder.create_dir(&image);
1325
1326         // Prepare the image directory
1327         let rustfmt = builder.ensure(tool::Rustfmt {
1328             compiler: builder.compiler(stage, builder.config.build),
1329             target, extra_features: Vec::new()
1330         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1331         let cargofmt = builder.ensure(tool::Cargofmt {
1332             compiler: builder.compiler(stage, builder.config.build),
1333             target, extra_features: Vec::new()
1334         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1335
1336         builder.install(&rustfmt, &image.join("bin"), 0o755);
1337         builder.install(&cargofmt, &image.join("bin"), 0o755);
1338         let doc = image.join("share/doc/rustfmt");
1339         builder.install(&src.join("README.md"), &doc, 0o644);
1340         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1341         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1342
1343         // Prepare the overlay
1344         let overlay = tmp.join("rustfmt-overlay");
1345         drop(fs::remove_dir_all(&overlay));
1346         builder.create_dir(&overlay);
1347         builder.install(&src.join("README.md"), &overlay, 0o644);
1348         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1349         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1350         builder.create(&overlay.join("version"), &version);
1351
1352         // Generate the installer tarball
1353         let mut cmd = rust_installer(builder);
1354         cmd.arg("generate")
1355            .arg("--product-name=Rust")
1356            .arg("--rel-manifest-dir=rustlib")
1357            .arg("--success-message=rustfmt-ready-to-fmt.")
1358            .arg("--image-dir").arg(&image)
1359            .arg("--work-dir").arg(&tmpdir(builder))
1360            .arg("--output-dir").arg(&distdir(builder))
1361            .arg("--non-installed-overlay").arg(&overlay)
1362            .arg(format!("--package-name={}-{}", name, target))
1363            .arg("--legacy-manifest-dirs=rustlib,cargo")
1364            .arg("--component-name=rustfmt-preview");
1365
1366         builder.run(&mut cmd);
1367         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1368     }
1369 }
1370
1371 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1372 pub struct Extended {
1373     stage: u32,
1374     host: Interned<String>,
1375     target: Interned<String>,
1376 }
1377
1378 impl Step for Extended {
1379     type Output = ();
1380     const DEFAULT: bool = true;
1381     const ONLY_HOSTS: bool = true;
1382
1383     fn should_run(run: ShouldRun) -> ShouldRun {
1384         let builder = run.builder;
1385         run.path("extended").default_condition(builder.config.extended)
1386     }
1387
1388     fn make_run(run: RunConfig) {
1389         run.builder.ensure(Extended {
1390             stage: run.builder.top_stage,
1391             host: run.builder.config.build,
1392             target: run.target,
1393         });
1394     }
1395
1396     /// Creates a combined installer for the specified target in the provided stage.
1397     fn run(self, builder: &Builder) {
1398         let stage = self.stage;
1399         let target = self.target;
1400
1401         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1402
1403         let rustc_installer = builder.ensure(Rustc {
1404             compiler: builder.compiler(stage, target),
1405         });
1406         let cargo_installer = builder.ensure(Cargo { stage, target });
1407         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1408         let rls_installer = builder.ensure(Rls { stage, target });
1409         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1410         let clippy_installer = builder.ensure(Clippy { stage, target });
1411         let lldb_installer = builder.ensure(Lldb { target });
1412         let mingw_installer = builder.ensure(Mingw { host: target });
1413         let analysis_installer = builder.ensure(Analysis {
1414             compiler: builder.compiler(stage, self.host),
1415             target
1416         });
1417
1418         let docs_installer = builder.ensure(Docs { stage, host: target, });
1419         let std_installer = builder.ensure(Std {
1420             compiler: builder.compiler(stage, self.host),
1421             target,
1422         });
1423
1424         let tmp = tmpdir(builder);
1425         let overlay = tmp.join("extended-overlay");
1426         let etc = builder.src.join("src/etc/installer");
1427         let work = tmp.join("work");
1428
1429         let _ = fs::remove_dir_all(&overlay);
1430         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1431         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1432         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1433         let version = builder.rust_version();
1434         builder.create(&overlay.join("version"), &version);
1435         if let Some(sha) = builder.rust_sha() {
1436             builder.create(&overlay.join("git-commit-hash"), &sha);
1437         }
1438         builder.install(&etc.join("README.md"), &overlay, 0o644);
1439
1440         // When rust-std package split from rustc, we needed to ensure that during
1441         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1442         // the std files during uninstall. To do this ensure that rustc comes
1443         // before rust-std in the list below.
1444         let mut tarballs = Vec::new();
1445         tarballs.push(rustc_installer);
1446         tarballs.push(cargo_installer);
1447         tarballs.extend(rls_installer.clone());
1448         tarballs.extend(clippy_installer.clone());
1449         tarballs.extend(rustfmt_installer.clone());
1450         tarballs.extend(llvm_tools_installer);
1451         tarballs.extend(lldb_installer);
1452         tarballs.push(analysis_installer);
1453         tarballs.push(std_installer);
1454         if builder.config.docs {
1455             tarballs.push(docs_installer);
1456         }
1457         if target.contains("pc-windows-gnu") {
1458             tarballs.push(mingw_installer.unwrap());
1459         }
1460         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1461         for tarball in &tarballs[1..] {
1462             input_tarballs.push(",");
1463             input_tarballs.push(tarball);
1464         }
1465
1466         let mut cmd = rust_installer(builder);
1467         cmd.arg("combine")
1468             .arg("--product-name=Rust")
1469             .arg("--rel-manifest-dir=rustlib")
1470             .arg("--success-message=Rust-is-ready-to-roll.")
1471             .arg("--work-dir").arg(&work)
1472             .arg("--output-dir").arg(&distdir(builder))
1473             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1474             .arg("--legacy-manifest-dirs=rustlib,cargo")
1475             .arg("--input-tarballs").arg(input_tarballs)
1476             .arg("--non-installed-overlay").arg(&overlay);
1477         builder.run(&mut cmd);
1478
1479         let mut license = String::new();
1480         license += &builder.read(&builder.src.join("COPYRIGHT"));
1481         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1482         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1483         license.push_str("\n");
1484         license.push_str("\n");
1485
1486         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1487         let mut rtf = rtf.to_string();
1488         rtf.push_str("\n");
1489         for line in license.lines() {
1490             rtf.push_str(line);
1491             rtf.push_str("\\line ");
1492         }
1493         rtf.push_str("}");
1494
1495         fn filter(contents: &str, marker: &str) -> String {
1496             let start = format!("tool-{}-start", marker);
1497             let end = format!("tool-{}-end", marker);
1498             let mut lines = Vec::new();
1499             let mut omitted = false;
1500             for line in contents.lines() {
1501                 if line.contains(&start) {
1502                     omitted = true;
1503                 } else if line.contains(&end) {
1504                     omitted = false;
1505                 } else if !omitted {
1506                     lines.push(line);
1507                 }
1508             }
1509
1510             lines.join("\n")
1511         }
1512
1513         let xform = |p: &Path| {
1514             let mut contents = String::new();
1515             t!(t!(File::open(p)).read_to_string(&mut contents));
1516             if rls_installer.is_none() {
1517                 contents = filter(&contents, "rls");
1518             }
1519             if clippy_installer.is_none() {
1520                 contents = filter(&contents, "clippy");
1521             }
1522             if rustfmt_installer.is_none() {
1523                 contents = filter(&contents, "rustfmt");
1524             }
1525             let ret = tmp.join(p.file_name().unwrap());
1526             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1527             return ret
1528         };
1529
1530         if target.contains("apple-darwin") {
1531             let pkg = tmp.join("pkg");
1532             let _ = fs::remove_dir_all(&pkg);
1533
1534             let pkgbuild = |component: &str| {
1535                 let mut cmd = Command::new("pkgbuild");
1536                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1537                     .arg("--scripts").arg(pkg.join(component))
1538                     .arg("--nopayload")
1539                     .arg(pkg.join(component).with_extension("pkg"));
1540                 builder.run(&mut cmd);
1541             };
1542
1543             let prepare = |name: &str| {
1544                 builder.create_dir(&pkg.join(name));
1545                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1546                         &pkg.join(name));
1547                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1548                 pkgbuild(name);
1549             };
1550             prepare("rustc");
1551             prepare("cargo");
1552             prepare("rust-docs");
1553             prepare("rust-std");
1554             prepare("rust-analysis");
1555
1556             if rls_installer.is_some() {
1557                 prepare("rls");
1558             }
1559             if clippy_installer.is_some() {
1560                 prepare("clippy");
1561             }
1562
1563             // create an 'uninstall' package
1564             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1565             pkgbuild("uninstall");
1566
1567             builder.create_dir(&pkg.join("res"));
1568             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1569             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1570             let mut cmd = Command::new("productbuild");
1571             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1572                 .arg("--resources").arg(pkg.join("res"))
1573                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1574                                                     pkgname(builder, "rust"),
1575                                                     target)))
1576                 .arg("--package-path").arg(&pkg);
1577             builder.run(&mut cmd);
1578         }
1579
1580         if target.contains("windows") {
1581             let exe = tmp.join("exe");
1582             let _ = fs::remove_dir_all(&exe);
1583
1584             let prepare = |name: &str| {
1585                 builder.create_dir(&exe.join(name));
1586                 let dir = if name == "rust-std" || name == "rust-analysis" {
1587                     format!("{}-{}", name, target)
1588                 } else if name == "rls" {
1589                     "rls-preview".to_string()
1590                 } else if name == "clippy" {
1591                     "clippy-preview".to_string()
1592                 } else {
1593                     name.to_string()
1594                 };
1595                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1596                             .join(dir),
1597                         &exe.join(name));
1598                 builder.remove(&exe.join(name).join("manifest.in"));
1599             };
1600             prepare("rustc");
1601             prepare("cargo");
1602             prepare("rust-analysis");
1603             prepare("rust-docs");
1604             prepare("rust-std");
1605             if rls_installer.is_some() {
1606                 prepare("rls");
1607             }
1608             if clippy_installer.is_some() {
1609                 prepare("clippy");
1610             }
1611             if target.contains("windows-gnu") {
1612                 prepare("rust-mingw");
1613             }
1614
1615             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1616             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1617             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1618             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1619             builder.create(&exe.join("LICENSE.txt"), &license);
1620
1621             // Generate exe installer
1622             let mut cmd = Command::new("iscc");
1623             cmd.arg("rust.iss")
1624                 .current_dir(&exe);
1625             if target.contains("windows-gnu") {
1626                 cmd.arg("/dMINGW");
1627             }
1628             add_env(builder, &mut cmd, target);
1629             builder.run(&mut cmd);
1630             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1631                     &distdir(builder),
1632                     0o755);
1633
1634             // Generate msi installer
1635             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1636             let heat = wix.join("bin/heat.exe");
1637             let candle = wix.join("bin/candle.exe");
1638             let light = wix.join("bin/light.exe");
1639
1640             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1641             builder.run(Command::new(&heat)
1642                             .current_dir(&exe)
1643                             .arg("dir")
1644                             .arg("rustc")
1645                             .args(&heat_flags)
1646                             .arg("-cg").arg("RustcGroup")
1647                             .arg("-dr").arg("Rustc")
1648                             .arg("-var").arg("var.RustcDir")
1649                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1650             builder.run(Command::new(&heat)
1651                             .current_dir(&exe)
1652                             .arg("dir")
1653                             .arg("rust-docs")
1654                             .args(&heat_flags)
1655                             .arg("-cg").arg("DocsGroup")
1656                             .arg("-dr").arg("Docs")
1657                             .arg("-var").arg("var.DocsDir")
1658                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1659                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1660             builder.run(Command::new(&heat)
1661                             .current_dir(&exe)
1662                             .arg("dir")
1663                             .arg("cargo")
1664                             .args(&heat_flags)
1665                             .arg("-cg").arg("CargoGroup")
1666                             .arg("-dr").arg("Cargo")
1667                             .arg("-var").arg("var.CargoDir")
1668                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1669                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1670             builder.run(Command::new(&heat)
1671                             .current_dir(&exe)
1672                             .arg("dir")
1673                             .arg("rust-std")
1674                             .args(&heat_flags)
1675                             .arg("-cg").arg("StdGroup")
1676                             .arg("-dr").arg("Std")
1677                             .arg("-var").arg("var.StdDir")
1678                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1679             if rls_installer.is_some() {
1680                 builder.run(Command::new(&heat)
1681                                 .current_dir(&exe)
1682                                 .arg("dir")
1683                                 .arg("rls")
1684                                 .args(&heat_flags)
1685                                 .arg("-cg").arg("RlsGroup")
1686                                 .arg("-dr").arg("Rls")
1687                                 .arg("-var").arg("var.RlsDir")
1688                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1689                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1690             }
1691             if clippy_installer.is_some() {
1692                 builder.run(Command::new(&heat)
1693                                 .current_dir(&exe)
1694                                 .arg("dir")
1695                                 .arg("clippy")
1696                                 .args(&heat_flags)
1697                                 .arg("-cg").arg("ClippyGroup")
1698                                 .arg("-dr").arg("Clippy")
1699                                 .arg("-var").arg("var.ClippyDir")
1700                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1701                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1702             }
1703             builder.run(Command::new(&heat)
1704                             .current_dir(&exe)
1705                             .arg("dir")
1706                             .arg("rust-analysis")
1707                             .args(&heat_flags)
1708                             .arg("-cg").arg("AnalysisGroup")
1709                             .arg("-dr").arg("Analysis")
1710                             .arg("-var").arg("var.AnalysisDir")
1711                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1712                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1713             if target.contains("windows-gnu") {
1714                 builder.run(Command::new(&heat)
1715                                 .current_dir(&exe)
1716                                 .arg("dir")
1717                                 .arg("rust-mingw")
1718                                 .args(&heat_flags)
1719                                 .arg("-cg").arg("GccGroup")
1720                                 .arg("-dr").arg("Gcc")
1721                                 .arg("-var").arg("var.GccDir")
1722                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1723             }
1724
1725             let candle = |input: &Path| {
1726                 let output = exe.join(input.file_stem().unwrap())
1727                                 .with_extension("wixobj");
1728                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1729                 let mut cmd = Command::new(&candle);
1730                 cmd.current_dir(&exe)
1731                     .arg("-nologo")
1732                     .arg("-dRustcDir=rustc")
1733                     .arg("-dDocsDir=rust-docs")
1734                     .arg("-dCargoDir=cargo")
1735                     .arg("-dStdDir=rust-std")
1736                     .arg("-dAnalysisDir=rust-analysis")
1737                     .arg("-arch").arg(&arch)
1738                     .arg("-out").arg(&output)
1739                     .arg(&input);
1740                 add_env(builder, &mut cmd, target);
1741
1742                 if rls_installer.is_some() {
1743                     cmd.arg("-dRlsDir=rls");
1744                 }
1745                 if clippy_installer.is_some() {
1746                     cmd.arg("-dClippyDir=clippy");
1747                 }
1748                 if target.contains("windows-gnu") {
1749                     cmd.arg("-dGccDir=rust-mingw");
1750                 }
1751                 builder.run(&mut cmd);
1752             };
1753             candle(&xform(&etc.join("msi/rust.wxs")));
1754             candle(&etc.join("msi/ui.wxs"));
1755             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1756             candle("RustcGroup.wxs".as_ref());
1757             candle("DocsGroup.wxs".as_ref());
1758             candle("CargoGroup.wxs".as_ref());
1759             candle("StdGroup.wxs".as_ref());
1760             if rls_installer.is_some() {
1761                 candle("RlsGroup.wxs".as_ref());
1762             }
1763             if clippy_installer.is_some() {
1764                 candle("ClippyGroup.wxs".as_ref());
1765             }
1766             candle("AnalysisGroup.wxs".as_ref());
1767
1768             if target.contains("windows-gnu") {
1769                 candle("GccGroup.wxs".as_ref());
1770             }
1771
1772             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1773             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1774             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1775
1776             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1777             let mut cmd = Command::new(&light);
1778             cmd.arg("-nologo")
1779                 .arg("-ext").arg("WixUIExtension")
1780                 .arg("-ext").arg("WixUtilExtension")
1781                 .arg("-out").arg(exe.join(&filename))
1782                 .arg("rust.wixobj")
1783                 .arg("ui.wixobj")
1784                 .arg("rustwelcomedlg.wixobj")
1785                 .arg("RustcGroup.wixobj")
1786                 .arg("DocsGroup.wixobj")
1787                 .arg("CargoGroup.wixobj")
1788                 .arg("StdGroup.wixobj")
1789                 .arg("AnalysisGroup.wixobj")
1790                 .current_dir(&exe);
1791
1792             if rls_installer.is_some() {
1793                 cmd.arg("RlsGroup.wixobj");
1794             }
1795             if clippy_installer.is_some() {
1796                 cmd.arg("ClippyGroup.wixobj");
1797             }
1798
1799             if target.contains("windows-gnu") {
1800                 cmd.arg("GccGroup.wixobj");
1801             }
1802             // ICE57 wrongly complains about the shortcuts
1803             cmd.arg("-sice:ICE57");
1804
1805             builder.run(&mut cmd);
1806
1807             if !builder.config.dry_run {
1808                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1809             }
1810         }
1811     }
1812 }
1813
1814 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1815     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1816     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1817        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1818        .env("CFG_RELEASE", builder.rust_release())
1819        .env("CFG_VER_MAJOR", parts.next().unwrap())
1820        .env("CFG_VER_MINOR", parts.next().unwrap())
1821        .env("CFG_VER_PATCH", parts.next().unwrap())
1822        .env("CFG_VER_BUILD", "0") // just needed to build
1823        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1824        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1825        .env("CFG_BUILD", target)
1826        .env("CFG_CHANNEL", &builder.config.channel);
1827
1828     if target.contains("windows-gnu") {
1829        cmd.env("CFG_MINGW", "1")
1830           .env("CFG_ABI", "GNU");
1831     } else {
1832        cmd.env("CFG_MINGW", "0")
1833           .env("CFG_ABI", "MSVC");
1834     }
1835
1836     if target.contains("x86_64") {
1837        cmd.env("CFG_PLATFORM", "x64");
1838     } else {
1839        cmd.env("CFG_PLATFORM", "x86");
1840     }
1841 }
1842
1843 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1844 pub struct HashSign;
1845
1846 impl Step for HashSign {
1847     type Output = ();
1848     const ONLY_HOSTS: bool = true;
1849
1850     fn should_run(run: ShouldRun) -> ShouldRun {
1851         run.path("hash-and-sign")
1852     }
1853
1854     fn make_run(run: RunConfig) {
1855         run.builder.ensure(HashSign);
1856     }
1857
1858     fn run(self, builder: &Builder) {
1859         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1860         if builder.config.dry_run {
1861             return;
1862         }
1863         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1864             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1865         });
1866         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1867             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1868         });
1869         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1870             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1871         });
1872         let mut pass = String::new();
1873         t!(t!(File::open(&file)).read_to_string(&mut pass));
1874
1875         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1876
1877         cmd.arg(sign);
1878         cmd.arg(distdir(builder));
1879         cmd.arg(today.trim());
1880         cmd.arg(builder.rust_package_vers());
1881         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1882         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1883         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1884         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1885         cmd.arg(builder.llvm_tools_package_vers());
1886         cmd.arg(builder.lldb_package_vers());
1887         cmd.arg(addr);
1888
1889         builder.create_dir(&distdir(builder));
1890
1891         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1892         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1893         let status = t!(child.wait());
1894         assert!(status.success());
1895     }
1896 }
1897
1898 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1899 // LLVM tools are linked dynamically.
1900 // Note: This function does no yet support Windows but we also don't support
1901 //       linking LLVM tools dynamically on Windows yet.
1902 fn maybe_install_llvm_dylib(builder: &Builder,
1903                             target: Interned<String>,
1904                             image: &Path) {
1905     let src_libdir = builder
1906         .llvm_out(target)
1907         .join("lib");
1908     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1909     t!(fs::create_dir_all(&dst_libdir));
1910
1911     if target.contains("apple-darwin") {
1912         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1913         if llvm_dylib_path.exists() {
1914             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1915         }
1916         return
1917     }
1918
1919     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1920     // Since tools link to the latter rather than the former, we have to
1921     // follow the symlink to find out what to distribute.
1922     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1923     if llvm_dylib_path.exists() {
1924         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1925             panic!("dist: Error calling canonicalize path `{}`: {}",
1926                    llvm_dylib_path.display(), e);
1927         });
1928
1929
1930         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1931     }
1932 }
1933
1934 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1935 pub struct LlvmTools {
1936     pub stage: u32,
1937     pub target: Interned<String>,
1938 }
1939
1940 impl Step for LlvmTools {
1941     type Output = Option<PathBuf>;
1942     const ONLY_HOSTS: bool = true;
1943
1944     fn should_run(run: ShouldRun) -> ShouldRun {
1945         run.path("llvm-tools")
1946     }
1947
1948     fn make_run(run: RunConfig) {
1949         run.builder.ensure(LlvmTools {
1950             stage: run.builder.top_stage,
1951             target: run.target,
1952         });
1953     }
1954
1955     fn run(self, builder: &Builder) -> Option<PathBuf> {
1956         let stage = self.stage;
1957         let target = self.target;
1958         assert!(builder.config.extended);
1959
1960         /* run only if llvm-config isn't used */
1961         if let Some(config) = builder.config.target_config.get(&target) {
1962             if let Some(ref _s) = config.llvm_config {
1963                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1964                     stage, target));
1965                 return None;
1966             }
1967         }
1968
1969         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1970         let src = builder.src.join("src/llvm");
1971         let name = pkgname(builder, "llvm-tools");
1972
1973         let tmp = tmpdir(builder);
1974         let image = tmp.join("llvm-tools-image");
1975         drop(fs::remove_dir_all(&image));
1976
1977         // Prepare the image directory
1978         let src_bindir = builder
1979             .llvm_out(target)
1980             .join("bin");
1981         let dst_bindir = image.join("lib/rustlib")
1982             .join(&*target)
1983             .join("bin");
1984         t!(fs::create_dir_all(&dst_bindir));
1985         for tool in LLVM_TOOLS {
1986             let exe = src_bindir.join(exe(tool, &target));
1987             builder.install(&exe, &dst_bindir, 0o755);
1988         }
1989
1990         // Prepare the overlay
1991         let overlay = tmp.join("llvm-tools-overlay");
1992         drop(fs::remove_dir_all(&overlay));
1993         builder.create_dir(&overlay);
1994         builder.install(&src.join("README.txt"), &overlay, 0o644);
1995         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1996         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1997
1998         // Generate the installer tarball
1999         let mut cmd = rust_installer(builder);
2000         cmd.arg("generate")
2001             .arg("--product-name=Rust")
2002             .arg("--rel-manifest-dir=rustlib")
2003             .arg("--success-message=llvm-tools-installed.")
2004             .arg("--image-dir").arg(&image)
2005             .arg("--work-dir").arg(&tmpdir(builder))
2006             .arg("--output-dir").arg(&distdir(builder))
2007             .arg("--non-installed-overlay").arg(&overlay)
2008             .arg(format!("--package-name={}-{}", name, target))
2009             .arg("--legacy-manifest-dirs=rustlib,cargo")
2010             .arg("--component-name=llvm-tools-preview");
2011
2012
2013         builder.run(&mut cmd);
2014         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2015     }
2016 }
2017
2018 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2019 pub struct Lldb {
2020     pub target: Interned<String>,
2021 }
2022
2023 impl Step for Lldb {
2024     type Output = Option<PathBuf>;
2025     const ONLY_HOSTS: bool = true;
2026     const DEFAULT: bool = true;
2027
2028     fn should_run(run: ShouldRun) -> ShouldRun {
2029         run.path("src/tools/lldb")
2030     }
2031
2032     fn make_run(run: RunConfig) {
2033         run.builder.ensure(Lldb {
2034             target: run.target,
2035         });
2036     }
2037
2038     fn run(self, builder: &Builder) -> Option<PathBuf> {
2039         let target = self.target;
2040
2041         if builder.config.dry_run {
2042             return None;
2043         }
2044
2045         let bindir = builder
2046             .llvm_out(target)
2047             .join("bin");
2048         let lldb_exe = bindir.join(exe("lldb", &target));
2049         if !lldb_exe.exists() {
2050             return None;
2051         }
2052
2053         builder.info(&format!("Dist Lldb ({})", target));
2054         let src = builder.src.join("src/tools/lldb");
2055         let name = pkgname(builder, "lldb");
2056
2057         let tmp = tmpdir(builder);
2058         let image = tmp.join("lldb-image");
2059         drop(fs::remove_dir_all(&image));
2060
2061         // Prepare the image directory
2062         let root = image.join("lib/rustlib").join(&*target);
2063         let dst = root.join("bin");
2064         t!(fs::create_dir_all(&dst));
2065         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2066             let exe = bindir.join(exe(program, &target));
2067             builder.install(&exe, &dst, 0o755);
2068         }
2069
2070         // The libraries.
2071         let libdir = builder.llvm_out(target).join("lib");
2072         let dst = root.join("lib");
2073         t!(fs::create_dir_all(&dst));
2074         for entry in t!(fs::read_dir(&libdir)) {
2075             let entry = entry.unwrap();
2076             if let Ok(name) = entry.file_name().into_string() {
2077                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2078                     if t!(entry.file_type()).is_symlink() {
2079                         builder.copy_to_folder(&entry.path(), &dst);
2080                     } else {
2081                        builder.install(&entry.path(), &dst, 0o755);
2082                     }
2083                 }
2084             }
2085         }
2086
2087         // The lldb scripts might be installed in lib/python$version
2088         // or in lib64/python$version.  If lib64 exists, use it;
2089         // otherwise lib.
2090         let libdir = builder.llvm_out(target).join("lib64");
2091         let (libdir, libdir_name) = if libdir.exists() {
2092             (libdir, "lib64")
2093         } else {
2094             (builder.llvm_out(target).join("lib"), "lib")
2095         };
2096         for entry in t!(fs::read_dir(&libdir)) {
2097             let entry = t!(entry);
2098             if let Ok(name) = entry.file_name().into_string() {
2099                 if name.starts_with("python") {
2100                     let dst = root.join(libdir_name)
2101                         .join(entry.file_name());
2102                     t!(fs::create_dir_all(&dst));
2103                     builder.cp_r(&entry.path(), &dst);
2104                     break;
2105                 }
2106             }
2107         }
2108
2109         // Prepare the overlay
2110         let overlay = tmp.join("lldb-overlay");
2111         drop(fs::remove_dir_all(&overlay));
2112         builder.create_dir(&overlay);
2113         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2114         builder.create(&overlay.join("version"), &builder.lldb_vers());
2115
2116         // Generate the installer tarball
2117         let mut cmd = rust_installer(builder);
2118         cmd.arg("generate")
2119             .arg("--product-name=Rust")
2120             .arg("--rel-manifest-dir=rustlib")
2121             .arg("--success-message=lldb-installed.")
2122             .arg("--image-dir").arg(&image)
2123             .arg("--work-dir").arg(&tmpdir(builder))
2124             .arg("--output-dir").arg(&distdir(builder))
2125             .arg("--non-installed-overlay").arg(&overlay)
2126             .arg(format!("--package-name={}-{}", name, target))
2127             .arg("--legacy-manifest-dirs=rustlib,cargo")
2128             .arg("--component-name=lldb-preview");
2129
2130
2131         builder.run(&mut cmd);
2132         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2133     }
2134 }