src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs;
  23 use std::io::Write;
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use crate::{Compiler, Mode, LLVM_TOOLS};
  30 use crate::channel;
  31 use crate::util::{libdir, is_dylib, exe};
  32 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  33 use crate::compile;
  34 use crate::tool::{self, Tool};
  35 use crate::cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/liballoc",
 861             "src/libbacktrace",
 862             "src/libcore",
 863             "src/libpanic_abort",
 864             "src/libpanic_unwind",
 865             "src/librustc_asan",
 866             "src/librustc_lsan",
 867             "src/librustc_msan",
 868             "src/librustc_tsan",
 869             "src/libstd",
 870             "src/libunwind",
 871             "src/libtest",
 872             "src/libterm",
 873             "src/libprofiler_builtins",
 874             "src/stdsimd",
 875             "src/libproc_macro",
 876             "src/tools/rustc-std-workspace-core",
 877         ];
 878
 879         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 880         for file in src_files.iter() {
 881             builder.copy(&builder.src.join(file), &dst_src.join(file));
 882         }
 883
 884         // Create source tarball in rust-installer format
 885         let mut cmd = rust_installer(builder);
 886         cmd.arg("generate")
 887            .arg("--product-name=Rust")
 888            .arg("--rel-manifest-dir=rustlib")
 889            .arg("--success-message=Awesome-Source.")
 890            .arg("--image-dir").arg(&image)
 891            .arg("--work-dir").arg(&tmpdir(builder))
 892            .arg("--output-dir").arg(&distdir(builder))
 893            .arg(format!("--package-name={}", name))
 894            .arg("--component-name=rust-src")
 895            .arg("--legacy-manifest-dirs=rustlib,cargo");
 896         builder.run(&mut cmd);
 897
 898         builder.remove_dir(&image);
 899         distdir(builder).join(&format!("{}.tar.gz", name))
 900     }
 901 }
 902
 903 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 904
 905 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 906 pub struct PlainSourceTarball;
 907
 908 impl Step for PlainSourceTarball {
 909     /// Produces the location of the tarball generated
 910     type Output = PathBuf;
 911     const DEFAULT: bool = true;
 912     const ONLY_HOSTS: bool = true;
 913
 914     fn should_run(run: ShouldRun) -> ShouldRun {
 915         let builder = run.builder;
 916         run.path("src").default_condition(builder.config.rust_dist_src)
 917     }
 918
 919     fn make_run(run: RunConfig) {
 920         run.builder.ensure(PlainSourceTarball);
 921     }
 922
 923     /// Creates the plain source tarball
 924     fn run(self, builder: &Builder) -> PathBuf {
 925         builder.info("Create plain source tarball");
 926
 927         // Make sure that the root folder of tarball has the correct name
 928         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 929         let plain_dst_src = tmpdir(builder).join(&plain_name);
 930         let _ = fs::remove_dir_all(&plain_dst_src);
 931         t!(fs::create_dir_all(&plain_dst_src));
 932
 933         // This is the set of root paths which will become part of the source package
 934         let src_files = [
 935             "COPYRIGHT",
 936             "LICENSE-APACHE",
 937             "LICENSE-MIT",
 938             "CONTRIBUTING.md",
 939             "README.md",
 940             "RELEASES.md",
 941             "configure",
 942             "x.py",
 943             "config.toml.example",
 944             "Cargo.toml",
 945             "Cargo.lock",
 946         ];
 947         let src_dirs = [
 948             "src",
 949         ];
 950
 951         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 952
 953         // Copy the files normally
 954         for item in &src_files {
 955             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 956         }
 957
 958         // Create the version file
 959         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 960         if let Some(sha) = builder.rust_sha() {
 961             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 962         }
 963
 964         // If we're building from git sources, we need to vendor a complete distribution.
 965         if builder.rust_info.is_git() {
 966             // Get cargo-vendor installed, if it isn't already.
 967             let mut has_cargo_vendor = false;
 968             let mut cmd = Command::new(&builder.initial_cargo);
 969             for line in output(cmd.arg("install").arg("--list")).lines() {
 970                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 971             }
 972             if !has_cargo_vendor {
 973                 let mut cmd = builder.cargo(
 974                     builder.compiler(0, builder.config.build),
 975                     Mode::ToolBootstrap,
 976                     builder.config.build,
 977                     "install"
 978                 );
 979                 cmd.arg("--force")
 980                    .arg("--debug")
 981                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 982                    .arg("cargo-vendor");
 983                 builder.run(&mut cmd);
 984             }
 985
 986             // Vendor all Cargo dependencies
 987             let mut cmd = Command::new(&builder.initial_cargo);
 988             cmd.arg("vendor")
 989                .current_dir(&plain_dst_src);
 990             builder.run(&mut cmd);
 991         }
 992
 993         // Create plain source tarball
 994         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
 995         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
 996         tarball.set_extension(""); // strip .gz
 997         tarball.set_extension(""); // strip .tar
 998         if let Some(dir) = tarball.parent() {
 999             builder.create_dir(&dir);
1000         }
1001         builder.info("running installer");
1002         let mut cmd = rust_installer(builder);
1003         cmd.arg("tarball")
1004            .arg("--input").arg(&plain_name)
1005            .arg("--output").arg(&tarball)
1006            .arg("--work-dir=.")
1007            .current_dir(tmpdir(builder));
1008         builder.run(&mut cmd);
1009         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1010     }
1011 }
1012
1013 // We have to run a few shell scripts, which choke quite a bit on both `\`
1014 // characters and on `C:\` paths, so normalize both of them away.
1015 pub fn sanitize_sh(path: &Path) -> String {
1016     let path = path.to_str().unwrap().replace("\\", "/");
1017     return change_drive(&path).unwrap_or(path);
1018
1019     fn change_drive(s: &str) -> Option<String> {
1020         let mut ch = s.chars();
1021         let drive = ch.next().unwrap_or('C');
1022         if ch.next() != Some(':') {
1023             return None
1024         }
1025         if ch.next() != Some('/') {
1026             return None
1027         }
1028         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1029     }
1030 }
1031
1032 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1033 pub struct Cargo {
1034     pub stage: u32,
1035     pub target: Interned<String>,
1036 }
1037
1038 impl Step for Cargo {
1039     type Output = PathBuf;
1040     const ONLY_HOSTS: bool = true;
1041
1042     fn should_run(run: ShouldRun) -> ShouldRun {
1043         run.path("cargo")
1044     }
1045
1046     fn make_run(run: RunConfig) {
1047         run.builder.ensure(Cargo {
1048             stage: run.builder.top_stage,
1049             target: run.target,
1050         });
1051     }
1052
1053     fn run(self, builder: &Builder) -> PathBuf {
1054         let stage = self.stage;
1055         let target = self.target;
1056
1057         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1058         let src = builder.src.join("src/tools/cargo");
1059         let etc = src.join("src/etc");
1060         let release_num = builder.release_num("cargo");
1061         let name = pkgname(builder, "cargo");
1062         let version = builder.cargo_info.version(builder, &release_num);
1063
1064         let tmp = tmpdir(builder);
1065         let image = tmp.join("cargo-image");
1066         drop(fs::remove_dir_all(&image));
1067         builder.create_dir(&image);
1068
1069         // Prepare the image directory
1070         builder.create_dir(&image.join("share/zsh/site-functions"));
1071         builder.create_dir(&image.join("etc/bash_completion.d"));
1072         let cargo = builder.ensure(tool::Cargo {
1073             compiler: builder.compiler(stage, builder.config.build),
1074             target
1075         });
1076         builder.install(&cargo, &image.join("bin"), 0o755);
1077         for man in t!(etc.join("man").read_dir()) {
1078             let man = t!(man);
1079             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1080         }
1081         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1082         builder.copy(&etc.join("cargo.bashcomp.sh"),
1083              &image.join("etc/bash_completion.d/cargo"));
1084         let doc = image.join("share/doc/cargo");
1085         builder.install(&src.join("README.md"), &doc, 0o644);
1086         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1087         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1088         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1089
1090         // Prepare the overlay
1091         let overlay = tmp.join("cargo-overlay");
1092         drop(fs::remove_dir_all(&overlay));
1093         builder.create_dir(&overlay);
1094         builder.install(&src.join("README.md"), &overlay, 0o644);
1095         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1096         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1097         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1098         builder.create(&overlay.join("version"), &version);
1099
1100         // Generate the installer tarball
1101         let mut cmd = rust_installer(builder);
1102         cmd.arg("generate")
1103            .arg("--product-name=Rust")
1104            .arg("--rel-manifest-dir=rustlib")
1105            .arg("--success-message=Rust-is-ready-to-roll.")
1106            .arg("--image-dir").arg(&image)
1107            .arg("--work-dir").arg(&tmpdir(builder))
1108            .arg("--output-dir").arg(&distdir(builder))
1109            .arg("--non-installed-overlay").arg(&overlay)
1110            .arg(format!("--package-name={}-{}", name, target))
1111            .arg("--component-name=cargo")
1112            .arg("--legacy-manifest-dirs=rustlib,cargo");
1113         builder.run(&mut cmd);
1114         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1115     }
1116 }
1117
1118 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1119 pub struct Rls {
1120     pub stage: u32,
1121     pub target: Interned<String>,
1122 }
1123
1124 impl Step for Rls {
1125     type Output = Option<PathBuf>;
1126     const ONLY_HOSTS: bool = true;
1127
1128     fn should_run(run: ShouldRun) -> ShouldRun {
1129         run.path("rls")
1130     }
1131
1132     fn make_run(run: RunConfig) {
1133         run.builder.ensure(Rls {
1134             stage: run.builder.top_stage,
1135             target: run.target,
1136         });
1137     }
1138
1139     fn run(self, builder: &Builder) -> Option<PathBuf> {
1140         let stage = self.stage;
1141         let target = self.target;
1142         assert!(builder.config.extended);
1143
1144         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1145         let src = builder.src.join("src/tools/rls");
1146         let release_num = builder.release_num("rls");
1147         let name = pkgname(builder, "rls");
1148         let version = builder.rls_info.version(builder, &release_num);
1149
1150         let tmp = tmpdir(builder);
1151         let image = tmp.join("rls-image");
1152         drop(fs::remove_dir_all(&image));
1153         t!(fs::create_dir_all(&image));
1154
1155         // Prepare the image directory
1156         // We expect RLS to build, because we've exited this step above if tool
1157         // state for RLS isn't testing.
1158         let rls = builder.ensure(tool::Rls {
1159             compiler: builder.compiler(stage, builder.config.build),
1160             target, extra_features: Vec::new()
1161         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1162
1163         builder.install(&rls, &image.join("bin"), 0o755);
1164         let doc = image.join("share/doc/rls");
1165         builder.install(&src.join("README.md"), &doc, 0o644);
1166         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1167         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1168
1169         // Prepare the overlay
1170         let overlay = tmp.join("rls-overlay");
1171         drop(fs::remove_dir_all(&overlay));
1172         t!(fs::create_dir_all(&overlay));
1173         builder.install(&src.join("README.md"), &overlay, 0o644);
1174         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1175         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1176         builder.create(&overlay.join("version"), &version);
1177
1178         // Generate the installer tarball
1179         let mut cmd = rust_installer(builder);
1180         cmd.arg("generate")
1181            .arg("--product-name=Rust")
1182            .arg("--rel-manifest-dir=rustlib")
1183            .arg("--success-message=RLS-ready-to-serve.")
1184            .arg("--image-dir").arg(&image)
1185            .arg("--work-dir").arg(&tmpdir(builder))
1186            .arg("--output-dir").arg(&distdir(builder))
1187            .arg("--non-installed-overlay").arg(&overlay)
1188            .arg(format!("--package-name={}-{}", name, target))
1189            .arg("--legacy-manifest-dirs=rustlib,cargo")
1190            .arg("--component-name=rls-preview");
1191
1192         builder.run(&mut cmd);
1193         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1194     }
1195 }
1196
1197 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1198 pub struct Clippy {
1199     pub stage: u32,
1200     pub target: Interned<String>,
1201 }
1202
1203 impl Step for Clippy {
1204     type Output = Option<PathBuf>;
1205     const ONLY_HOSTS: bool = true;
1206
1207     fn should_run(run: ShouldRun) -> ShouldRun {
1208         run.path("clippy")
1209     }
1210
1211     fn make_run(run: RunConfig) {
1212         run.builder.ensure(Clippy {
1213             stage: run.builder.top_stage,
1214             target: run.target,
1215         });
1216     }
1217
1218     fn run(self, builder: &Builder) -> Option<PathBuf> {
1219         let stage = self.stage;
1220         let target = self.target;
1221         assert!(builder.config.extended);
1222
1223         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1224         let src = builder.src.join("src/tools/clippy");
1225         let release_num = builder.release_num("clippy");
1226         let name = pkgname(builder, "clippy");
1227         let version = builder.clippy_info.version(builder, &release_num);
1228
1229         let tmp = tmpdir(builder);
1230         let image = tmp.join("clippy-image");
1231         drop(fs::remove_dir_all(&image));
1232         builder.create_dir(&image);
1233
1234         // Prepare the image directory
1235         // We expect clippy to build, because we've exited this step above if tool
1236         // state for clippy isn't testing.
1237         let clippy = builder.ensure(tool::Clippy {
1238             compiler: builder.compiler(stage, builder.config.build),
1239             target, extra_features: Vec::new()
1240         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1241         let cargoclippy = builder.ensure(tool::CargoClippy {
1242             compiler: builder.compiler(stage, builder.config.build),
1243             target, extra_features: Vec::new()
1244         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1245
1246         builder.install(&clippy, &image.join("bin"), 0o755);
1247         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1248         let doc = image.join("share/doc/clippy");
1249         builder.install(&src.join("README.md"), &doc, 0o644);
1250         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1251         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1252
1253         // Prepare the overlay
1254         let overlay = tmp.join("clippy-overlay");
1255         drop(fs::remove_dir_all(&overlay));
1256         t!(fs::create_dir_all(&overlay));
1257         builder.install(&src.join("README.md"), &overlay, 0o644);
1258         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1259         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1260         builder.create(&overlay.join("version"), &version);
1261
1262         // Generate the installer tarball
1263         let mut cmd = rust_installer(builder);
1264         cmd.arg("generate")
1265            .arg("--product-name=Rust")
1266            .arg("--rel-manifest-dir=rustlib")
1267            .arg("--success-message=clippy-ready-to-serve.")
1268            .arg("--image-dir").arg(&image)
1269            .arg("--work-dir").arg(&tmpdir(builder))
1270            .arg("--output-dir").arg(&distdir(builder))
1271            .arg("--non-installed-overlay").arg(&overlay)
1272            .arg(format!("--package-name={}-{}", name, target))
1273            .arg("--legacy-manifest-dirs=rustlib,cargo")
1274            .arg("--component-name=clippy-preview");
1275
1276         builder.run(&mut cmd);
1277         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1278     }
1279 }
1280
1281 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1282 pub struct Rustfmt {
1283     pub stage: u32,
1284     pub target: Interned<String>,
1285 }
1286
1287 impl Step for Rustfmt {
1288     type Output = Option<PathBuf>;
1289     const ONLY_HOSTS: bool = true;
1290
1291     fn should_run(run: ShouldRun) -> ShouldRun {
1292         run.path("rustfmt")
1293     }
1294
1295     fn make_run(run: RunConfig) {
1296         run.builder.ensure(Rustfmt {
1297             stage: run.builder.top_stage,
1298             target: run.target,
1299         });
1300     }
1301
1302     fn run(self, builder: &Builder) -> Option<PathBuf> {
1303         let stage = self.stage;
1304         let target = self.target;
1305
1306         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1307         let src = builder.src.join("src/tools/rustfmt");
1308         let release_num = builder.release_num("rustfmt");
1309         let name = pkgname(builder, "rustfmt");
1310         let version = builder.rustfmt_info.version(builder, &release_num);
1311
1312         let tmp = tmpdir(builder);
1313         let image = tmp.join("rustfmt-image");
1314         drop(fs::remove_dir_all(&image));
1315         builder.create_dir(&image);
1316
1317         // Prepare the image directory
1318         let rustfmt = builder.ensure(tool::Rustfmt {
1319             compiler: builder.compiler(stage, builder.config.build),
1320             target, extra_features: Vec::new()
1321         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1322         let cargofmt = builder.ensure(tool::Cargofmt {
1323             compiler: builder.compiler(stage, builder.config.build),
1324             target, extra_features: Vec::new()
1325         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1326
1327         builder.install(&rustfmt, &image.join("bin"), 0o755);
1328         builder.install(&cargofmt, &image.join("bin"), 0o755);
1329         let doc = image.join("share/doc/rustfmt");
1330         builder.install(&src.join("README.md"), &doc, 0o644);
1331         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1332         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1333
1334         // Prepare the overlay
1335         let overlay = tmp.join("rustfmt-overlay");
1336         drop(fs::remove_dir_all(&overlay));
1337         builder.create_dir(&overlay);
1338         builder.install(&src.join("README.md"), &overlay, 0o644);
1339         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1340         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1341         builder.create(&overlay.join("version"), &version);
1342
1343         // Generate the installer tarball
1344         let mut cmd = rust_installer(builder);
1345         cmd.arg("generate")
1346            .arg("--product-name=Rust")
1347            .arg("--rel-manifest-dir=rustlib")
1348            .arg("--success-message=rustfmt-ready-to-fmt.")
1349            .arg("--image-dir").arg(&image)
1350            .arg("--work-dir").arg(&tmpdir(builder))
1351            .arg("--output-dir").arg(&distdir(builder))
1352            .arg("--non-installed-overlay").arg(&overlay)
1353            .arg(format!("--package-name={}-{}", name, target))
1354            .arg("--legacy-manifest-dirs=rustlib,cargo")
1355            .arg("--component-name=rustfmt-preview");
1356
1357         builder.run(&mut cmd);
1358         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1359     }
1360 }
1361
1362 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1363 pub struct Extended {
1364     stage: u32,
1365     host: Interned<String>,
1366     target: Interned<String>,
1367 }
1368
1369 impl Step for Extended {
1370     type Output = ();
1371     const DEFAULT: bool = true;
1372     const ONLY_HOSTS: bool = true;
1373
1374     fn should_run(run: ShouldRun) -> ShouldRun {
1375         let builder = run.builder;
1376         run.path("extended").default_condition(builder.config.extended)
1377     }
1378
1379     fn make_run(run: RunConfig) {
1380         run.builder.ensure(Extended {
1381             stage: run.builder.top_stage,
1382             host: run.builder.config.build,
1383             target: run.target,
1384         });
1385     }
1386
1387     /// Creates a combined installer for the specified target in the provided stage.
1388     fn run(self, builder: &Builder) {
1389         let stage = self.stage;
1390         let target = self.target;
1391
1392         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1393
1394         let rustc_installer = builder.ensure(Rustc {
1395             compiler: builder.compiler(stage, target),
1396         });
1397         let cargo_installer = builder.ensure(Cargo { stage, target });
1398         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1399         let rls_installer = builder.ensure(Rls { stage, target });
1400         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1401         let clippy_installer = builder.ensure(Clippy { stage, target });
1402         let lldb_installer = builder.ensure(Lldb { target });
1403         let mingw_installer = builder.ensure(Mingw { host: target });
1404         let analysis_installer = builder.ensure(Analysis {
1405             compiler: builder.compiler(stage, self.host),
1406             target
1407         });
1408
1409         let docs_installer = builder.ensure(Docs { stage, host: target, });
1410         let std_installer = builder.ensure(Std {
1411             compiler: builder.compiler(stage, self.host),
1412             target,
1413         });
1414
1415         let tmp = tmpdir(builder);
1416         let overlay = tmp.join("extended-overlay");
1417         let etc = builder.src.join("src/etc/installer");
1418         let work = tmp.join("work");
1419
1420         let _ = fs::remove_dir_all(&overlay);
1421         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1422         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1423         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1424         let version = builder.rust_version();
1425         builder.create(&overlay.join("version"), &version);
1426         if let Some(sha) = builder.rust_sha() {
1427             builder.create(&overlay.join("git-commit-hash"), &sha);
1428         }
1429         builder.install(&etc.join("README.md"), &overlay, 0o644);
1430
1431         // When rust-std package split from rustc, we needed to ensure that during
1432         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1433         // the std files during uninstall. To do this ensure that rustc comes
1434         // before rust-std in the list below.
1435         let mut tarballs = Vec::new();
1436         tarballs.push(rustc_installer);
1437         tarballs.push(cargo_installer);
1438         tarballs.extend(rls_installer.clone());
1439         tarballs.extend(clippy_installer.clone());
1440         tarballs.extend(rustfmt_installer.clone());
1441         tarballs.extend(llvm_tools_installer);
1442         tarballs.extend(lldb_installer);
1443         tarballs.push(analysis_installer);
1444         tarballs.push(std_installer);
1445         if builder.config.docs {
1446             tarballs.push(docs_installer);
1447         }
1448         if target.contains("pc-windows-gnu") {
1449             tarballs.push(mingw_installer.unwrap());
1450         }
1451         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1452         for tarball in &tarballs[1..] {
1453             input_tarballs.push(",");
1454             input_tarballs.push(tarball);
1455         }
1456
1457         let mut cmd = rust_installer(builder);
1458         cmd.arg("combine")
1459             .arg("--product-name=Rust")
1460             .arg("--rel-manifest-dir=rustlib")
1461             .arg("--success-message=Rust-is-ready-to-roll.")
1462             .arg("--work-dir").arg(&work)
1463             .arg("--output-dir").arg(&distdir(builder))
1464             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1465             .arg("--legacy-manifest-dirs=rustlib,cargo")
1466             .arg("--input-tarballs").arg(input_tarballs)
1467             .arg("--non-installed-overlay").arg(&overlay);
1468         builder.run(&mut cmd);
1469
1470         let mut license = String::new();
1471         license += &builder.read(&builder.src.join("COPYRIGHT"));
1472         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1473         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1474         license.push_str("\n");
1475         license.push_str("\n");
1476
1477         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1478         let mut rtf = rtf.to_string();
1479         rtf.push_str("\n");
1480         for line in license.lines() {
1481             rtf.push_str(line);
1482             rtf.push_str("\\line ");
1483         }
1484         rtf.push_str("}");
1485
1486         fn filter(contents: &str, marker: &str) -> String {
1487             let start = format!("tool-{}-start", marker);
1488             let end = format!("tool-{}-end", marker);
1489             let mut lines = Vec::new();
1490             let mut omitted = false;
1491             for line in contents.lines() {
1492                 if line.contains(&start) {
1493                     omitted = true;
1494                 } else if line.contains(&end) {
1495                     omitted = false;
1496                 } else if !omitted {
1497                     lines.push(line);
1498                 }
1499             }
1500
1501             lines.join("\n")
1502         }
1503
1504         let xform = |p: &Path| {
1505             let mut contents = t!(fs::read_to_string(p));
1506             if rls_installer.is_none() {
1507                 contents = filter(&contents, "rls");
1508             }
1509             if clippy_installer.is_none() {
1510                 contents = filter(&contents, "clippy");
1511             }
1512             if rustfmt_installer.is_none() {
1513                 contents = filter(&contents, "rustfmt");
1514             }
1515             let ret = tmp.join(p.file_name().unwrap());
1516             t!(fs::write(&ret, &contents));
1517             ret
1518         };
1519
1520         if target.contains("apple-darwin") {
1521             let pkg = tmp.join("pkg");
1522             let _ = fs::remove_dir_all(&pkg);
1523
1524             let pkgbuild = |component: &str| {
1525                 let mut cmd = Command::new("pkgbuild");
1526                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1527                     .arg("--scripts").arg(pkg.join(component))
1528                     .arg("--nopayload")
1529                     .arg(pkg.join(component).with_extension("pkg"));
1530                 builder.run(&mut cmd);
1531             };
1532
1533             let prepare = |name: &str| {
1534                 builder.create_dir(&pkg.join(name));
1535                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1536                         &pkg.join(name));
1537                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1538                 pkgbuild(name);
1539             };
1540             prepare("rustc");
1541             prepare("cargo");
1542             prepare("rust-docs");
1543             prepare("rust-std");
1544             prepare("rust-analysis");
1545
1546             if rls_installer.is_some() {
1547                 prepare("rls");
1548             }
1549             if clippy_installer.is_some() {
1550                 prepare("clippy");
1551             }
1552
1553             // create an 'uninstall' package
1554             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1555             pkgbuild("uninstall");
1556
1557             builder.create_dir(&pkg.join("res"));
1558             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1559             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1560             let mut cmd = Command::new("productbuild");
1561             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1562                 .arg("--resources").arg(pkg.join("res"))
1563                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1564                                                     pkgname(builder, "rust"),
1565                                                     target)))
1566                 .arg("--package-path").arg(&pkg);
1567             builder.run(&mut cmd);
1568         }
1569
1570         if target.contains("windows") {
1571             let exe = tmp.join("exe");
1572             let _ = fs::remove_dir_all(&exe);
1573
1574             let prepare = |name: &str| {
1575                 builder.create_dir(&exe.join(name));
1576                 let dir = if name == "rust-std" || name == "rust-analysis" {
1577                     format!("{}-{}", name, target)
1578                 } else if name == "rls" {
1579                     "rls-preview".to_string()
1580                 } else if name == "clippy" {
1581                     "clippy-preview".to_string()
1582                 } else {
1583                     name.to_string()
1584                 };
1585                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1586                             .join(dir),
1587                         &exe.join(name));
1588                 builder.remove(&exe.join(name).join("manifest.in"));
1589             };
1590             prepare("rustc");
1591             prepare("cargo");
1592             prepare("rust-analysis");
1593             prepare("rust-docs");
1594             prepare("rust-std");
1595             if rls_installer.is_some() {
1596                 prepare("rls");
1597             }
1598             if clippy_installer.is_some() {
1599                 prepare("clippy");
1600             }
1601             if target.contains("windows-gnu") {
1602                 prepare("rust-mingw");
1603             }
1604
1605             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1606             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1607             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1608             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1609             builder.create(&exe.join("LICENSE.txt"), &license);
1610
1611             // Generate exe installer
1612             let mut cmd = Command::new("iscc");
1613             cmd.arg("rust.iss")
1614                 .current_dir(&exe);
1615             if target.contains("windows-gnu") {
1616                 cmd.arg("/dMINGW");
1617             }
1618             add_env(builder, &mut cmd, target);
1619             builder.run(&mut cmd);
1620             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1621                     &distdir(builder),
1622                     0o755);
1623
1624             // Generate msi installer
1625             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1626             let heat = wix.join("bin/heat.exe");
1627             let candle = wix.join("bin/candle.exe");
1628             let light = wix.join("bin/light.exe");
1629
1630             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1631             builder.run(Command::new(&heat)
1632                             .current_dir(&exe)
1633                             .arg("dir")
1634                             .arg("rustc")
1635                             .args(&heat_flags)
1636                             .arg("-cg").arg("RustcGroup")
1637                             .arg("-dr").arg("Rustc")
1638                             .arg("-var").arg("var.RustcDir")
1639                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1640             builder.run(Command::new(&heat)
1641                             .current_dir(&exe)
1642                             .arg("dir")
1643                             .arg("rust-docs")
1644                             .args(&heat_flags)
1645                             .arg("-cg").arg("DocsGroup")
1646                             .arg("-dr").arg("Docs")
1647                             .arg("-var").arg("var.DocsDir")
1648                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1649                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1650             builder.run(Command::new(&heat)
1651                             .current_dir(&exe)
1652                             .arg("dir")
1653                             .arg("cargo")
1654                             .args(&heat_flags)
1655                             .arg("-cg").arg("CargoGroup")
1656                             .arg("-dr").arg("Cargo")
1657                             .arg("-var").arg("var.CargoDir")
1658                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1659                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1660             builder.run(Command::new(&heat)
1661                             .current_dir(&exe)
1662                             .arg("dir")
1663                             .arg("rust-std")
1664                             .args(&heat_flags)
1665                             .arg("-cg").arg("StdGroup")
1666                             .arg("-dr").arg("Std")
1667                             .arg("-var").arg("var.StdDir")
1668                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1669             if rls_installer.is_some() {
1670                 builder.run(Command::new(&heat)
1671                                 .current_dir(&exe)
1672                                 .arg("dir")
1673                                 .arg("rls")
1674                                 .args(&heat_flags)
1675                                 .arg("-cg").arg("RlsGroup")
1676                                 .arg("-dr").arg("Rls")
1677                                 .arg("-var").arg("var.RlsDir")
1678                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1679                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1680             }
1681             if clippy_installer.is_some() {
1682                 builder.run(Command::new(&heat)
1683                                 .current_dir(&exe)
1684                                 .arg("dir")
1685                                 .arg("clippy")
1686                                 .args(&heat_flags)
1687                                 .arg("-cg").arg("ClippyGroup")
1688                                 .arg("-dr").arg("Clippy")
1689                                 .arg("-var").arg("var.ClippyDir")
1690                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1691                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1692             }
1693             builder.run(Command::new(&heat)
1694                             .current_dir(&exe)
1695                             .arg("dir")
1696                             .arg("rust-analysis")
1697                             .args(&heat_flags)
1698                             .arg("-cg").arg("AnalysisGroup")
1699                             .arg("-dr").arg("Analysis")
1700                             .arg("-var").arg("var.AnalysisDir")
1701                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1702                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1703             if target.contains("windows-gnu") {
1704                 builder.run(Command::new(&heat)
1705                                 .current_dir(&exe)
1706                                 .arg("dir")
1707                                 .arg("rust-mingw")
1708                                 .args(&heat_flags)
1709                                 .arg("-cg").arg("GccGroup")
1710                                 .arg("-dr").arg("Gcc")
1711                                 .arg("-var").arg("var.GccDir")
1712                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1713             }
1714
1715             let candle = |input: &Path| {
1716                 let output = exe.join(input.file_stem().unwrap())
1717                                 .with_extension("wixobj");
1718                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1719                 let mut cmd = Command::new(&candle);
1720                 cmd.current_dir(&exe)
1721                     .arg("-nologo")
1722                     .arg("-dRustcDir=rustc")
1723                     .arg("-dDocsDir=rust-docs")
1724                     .arg("-dCargoDir=cargo")
1725                     .arg("-dStdDir=rust-std")
1726                     .arg("-dAnalysisDir=rust-analysis")
1727                     .arg("-arch").arg(&arch)
1728                     .arg("-out").arg(&output)
1729                     .arg(&input);
1730                 add_env(builder, &mut cmd, target);
1731
1732                 if rls_installer.is_some() {
1733                     cmd.arg("-dRlsDir=rls");
1734                 }
1735                 if clippy_installer.is_some() {
1736                     cmd.arg("-dClippyDir=clippy");
1737                 }
1738                 if target.contains("windows-gnu") {
1739                     cmd.arg("-dGccDir=rust-mingw");
1740                 }
1741                 builder.run(&mut cmd);
1742             };
1743             candle(&xform(&etc.join("msi/rust.wxs")));
1744             candle(&etc.join("msi/ui.wxs"));
1745             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1746             candle("RustcGroup.wxs".as_ref());
1747             candle("DocsGroup.wxs".as_ref());
1748             candle("CargoGroup.wxs".as_ref());
1749             candle("StdGroup.wxs".as_ref());
1750             if rls_installer.is_some() {
1751                 candle("RlsGroup.wxs".as_ref());
1752             }
1753             if clippy_installer.is_some() {
1754                 candle("ClippyGroup.wxs".as_ref());
1755             }
1756             candle("AnalysisGroup.wxs".as_ref());
1757
1758             if target.contains("windows-gnu") {
1759                 candle("GccGroup.wxs".as_ref());
1760             }
1761
1762             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1763             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1764             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1765
1766             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1767             let mut cmd = Command::new(&light);
1768             cmd.arg("-nologo")
1769                 .arg("-ext").arg("WixUIExtension")
1770                 .arg("-ext").arg("WixUtilExtension")
1771                 .arg("-out").arg(exe.join(&filename))
1772                 .arg("rust.wixobj")
1773                 .arg("ui.wixobj")
1774                 .arg("rustwelcomedlg.wixobj")
1775                 .arg("RustcGroup.wixobj")
1776                 .arg("DocsGroup.wixobj")
1777                 .arg("CargoGroup.wixobj")
1778                 .arg("StdGroup.wixobj")
1779                 .arg("AnalysisGroup.wixobj")
1780                 .current_dir(&exe);
1781
1782             if rls_installer.is_some() {
1783                 cmd.arg("RlsGroup.wixobj");
1784             }
1785             if clippy_installer.is_some() {
1786                 cmd.arg("ClippyGroup.wixobj");
1787             }
1788
1789             if target.contains("windows-gnu") {
1790                 cmd.arg("GccGroup.wixobj");
1791             }
1792             // ICE57 wrongly complains about the shortcuts
1793             cmd.arg("-sice:ICE57");
1794
1795             builder.run(&mut cmd);
1796
1797             if !builder.config.dry_run {
1798                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1799             }
1800         }
1801     }
1802 }
1803
1804 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1805     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1806     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1807        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1808        .env("CFG_RELEASE", builder.rust_release())
1809        .env("CFG_VER_MAJOR", parts.next().unwrap())
1810        .env("CFG_VER_MINOR", parts.next().unwrap())
1811        .env("CFG_VER_PATCH", parts.next().unwrap())
1812        .env("CFG_VER_BUILD", "0") // just needed to build
1813        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1814        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1815        .env("CFG_BUILD", target)
1816        .env("CFG_CHANNEL", &builder.config.channel);
1817
1818     if target.contains("windows-gnu") {
1819        cmd.env("CFG_MINGW", "1")
1820           .env("CFG_ABI", "GNU");
1821     } else {
1822        cmd.env("CFG_MINGW", "0")
1823           .env("CFG_ABI", "MSVC");
1824     }
1825
1826     if target.contains("x86_64") {
1827        cmd.env("CFG_PLATFORM", "x64");
1828     } else {
1829        cmd.env("CFG_PLATFORM", "x86");
1830     }
1831 }
1832
1833 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1834 pub struct HashSign;
1835
1836 impl Step for HashSign {
1837     type Output = ();
1838     const ONLY_HOSTS: bool = true;
1839
1840     fn should_run(run: ShouldRun) -> ShouldRun {
1841         run.path("hash-and-sign")
1842     }
1843
1844     fn make_run(run: RunConfig) {
1845         run.builder.ensure(HashSign);
1846     }
1847
1848     fn run(self, builder: &Builder) {
1849         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1850         if builder.config.dry_run {
1851             return;
1852         }
1853         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1854             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1855         });
1856         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1857             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1858         });
1859         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1860             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1861         });
1862         let pass = t!(fs::read_to_string(&file));
1863
1864         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1865
1866         cmd.arg(sign);
1867         cmd.arg(distdir(builder));
1868         cmd.arg(today.trim());
1869         cmd.arg(builder.rust_package_vers());
1870         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1871         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1872         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1873         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1874         cmd.arg(builder.llvm_tools_package_vers());
1875         cmd.arg(builder.lldb_package_vers());
1876         cmd.arg(addr);
1877
1878         builder.create_dir(&distdir(builder));
1879
1880         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1881         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1882         let status = t!(child.wait());
1883         assert!(status.success());
1884     }
1885 }
1886
1887 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1888 // LLVM tools are linked dynamically.
1889 // Note: This function does no yet support Windows but we also don't support
1890 //       linking LLVM tools dynamically on Windows yet.
1891 fn maybe_install_llvm_dylib(builder: &Builder,
1892                             target: Interned<String>,
1893                             image: &Path) {
1894     let src_libdir = builder
1895         .llvm_out(target)
1896         .join("lib");
1897     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1898     t!(fs::create_dir_all(&dst_libdir));
1899
1900     if target.contains("apple-darwin") {
1901         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1902         if llvm_dylib_path.exists() {
1903             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1904         }
1905         return
1906     }
1907
1908     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1909     // Since tools link to the latter rather than the former, we have to
1910     // follow the symlink to find out what to distribute.
1911     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1912     if llvm_dylib_path.exists() {
1913         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1914             panic!("dist: Error calling canonicalize path `{}`: {}",
1915                    llvm_dylib_path.display(), e);
1916         });
1917
1918
1919         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1920     }
1921 }
1922
1923 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1924 pub struct LlvmTools {
1925     pub stage: u32,
1926     pub target: Interned<String>,
1927 }
1928
1929 impl Step for LlvmTools {
1930     type Output = Option<PathBuf>;
1931     const ONLY_HOSTS: bool = true;
1932
1933     fn should_run(run: ShouldRun) -> ShouldRun {
1934         run.path("llvm-tools")
1935     }
1936
1937     fn make_run(run: RunConfig) {
1938         run.builder.ensure(LlvmTools {
1939             stage: run.builder.top_stage,
1940             target: run.target,
1941         });
1942     }
1943
1944     fn run(self, builder: &Builder) -> Option<PathBuf> {
1945         let stage = self.stage;
1946         let target = self.target;
1947         assert!(builder.config.extended);
1948
1949         /* run only if llvm-config isn't used */
1950         if let Some(config) = builder.config.target_config.get(&target) {
1951             if let Some(ref _s) = config.llvm_config {
1952                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1953                     stage, target));
1954                 return None;
1955             }
1956         }
1957
1958         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1959         let src = builder.src.join("src/llvm");
1960         let name = pkgname(builder, "llvm-tools");
1961
1962         let tmp = tmpdir(builder);
1963         let image = tmp.join("llvm-tools-image");
1964         drop(fs::remove_dir_all(&image));
1965
1966         // Prepare the image directory
1967         let src_bindir = builder
1968             .llvm_out(target)
1969             .join("bin");
1970         let dst_bindir = image.join("lib/rustlib")
1971             .join(&*target)
1972             .join("bin");
1973         t!(fs::create_dir_all(&dst_bindir));
1974         for tool in LLVM_TOOLS {
1975             let exe = src_bindir.join(exe(tool, &target));
1976             builder.install(&exe, &dst_bindir, 0o755);
1977         }
1978
1979         // Prepare the overlay
1980         let overlay = tmp.join("llvm-tools-overlay");
1981         drop(fs::remove_dir_all(&overlay));
1982         builder.create_dir(&overlay);
1983         builder.install(&src.join("README.txt"), &overlay, 0o644);
1984         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1985         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1986
1987         // Generate the installer tarball
1988         let mut cmd = rust_installer(builder);
1989         cmd.arg("generate")
1990             .arg("--product-name=Rust")
1991             .arg("--rel-manifest-dir=rustlib")
1992             .arg("--success-message=llvm-tools-installed.")
1993             .arg("--image-dir").arg(&image)
1994             .arg("--work-dir").arg(&tmpdir(builder))
1995             .arg("--output-dir").arg(&distdir(builder))
1996             .arg("--non-installed-overlay").arg(&overlay)
1997             .arg(format!("--package-name={}-{}", name, target))
1998             .arg("--legacy-manifest-dirs=rustlib,cargo")
1999             .arg("--component-name=llvm-tools-preview");
2000
2001
2002         builder.run(&mut cmd);
2003         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2004     }
2005 }
2006
2007 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2008 pub struct Lldb {
2009     pub target: Interned<String>,
2010 }
2011
2012 impl Step for Lldb {
2013     type Output = Option<PathBuf>;
2014     const ONLY_HOSTS: bool = true;
2015     const DEFAULT: bool = true;
2016
2017     fn should_run(run: ShouldRun) -> ShouldRun {
2018         run.path("src/tools/lldb")
2019     }
2020
2021     fn make_run(run: RunConfig) {
2022         run.builder.ensure(Lldb {
2023             target: run.target,
2024         });
2025     }
2026
2027     fn run(self, builder: &Builder) -> Option<PathBuf> {
2028         let target = self.target;
2029
2030         if builder.config.dry_run {
2031             return None;
2032         }
2033
2034         let bindir = builder
2035             .llvm_out(target)
2036             .join("bin");
2037         let lldb_exe = bindir.join(exe("lldb", &target));
2038         if !lldb_exe.exists() {
2039             return None;
2040         }
2041
2042         builder.info(&format!("Dist Lldb ({})", target));
2043         let src = builder.src.join("src/tools/lldb");
2044         let name = pkgname(builder, "lldb");
2045
2046         let tmp = tmpdir(builder);
2047         let image = tmp.join("lldb-image");
2048         drop(fs::remove_dir_all(&image));
2049
2050         // Prepare the image directory
2051         let root = image.join("lib/rustlib").join(&*target);
2052         let dst = root.join("bin");
2053         t!(fs::create_dir_all(&dst));
2054         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2055             let exe = bindir.join(exe(program, &target));
2056             builder.install(&exe, &dst, 0o755);
2057         }
2058
2059         // The libraries.
2060         let libdir = builder.llvm_out(target).join("lib");
2061         let dst = root.join("lib");
2062         t!(fs::create_dir_all(&dst));
2063         for entry in t!(fs::read_dir(&libdir)) {
2064             let entry = entry.unwrap();
2065             if let Ok(name) = entry.file_name().into_string() {
2066                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2067                     if t!(entry.file_type()).is_symlink() {
2068                         builder.copy_to_folder(&entry.path(), &dst);
2069                     } else {
2070                        builder.install(&entry.path(), &dst, 0o755);
2071                     }
2072                 }
2073             }
2074         }
2075
2076         // The lldb scripts might be installed in lib/python$version
2077         // or in lib64/python$version.  If lib64 exists, use it;
2078         // otherwise lib.
2079         let libdir = builder.llvm_out(target).join("lib64");
2080         let (libdir, libdir_name) = if libdir.exists() {
2081             (libdir, "lib64")
2082         } else {
2083             (builder.llvm_out(target).join("lib"), "lib")
2084         };
2085         for entry in t!(fs::read_dir(&libdir)) {
2086             let entry = t!(entry);
2087             if let Ok(name) = entry.file_name().into_string() {
2088                 if name.starts_with("python") {
2089                     let dst = root.join(libdir_name)
2090                         .join(entry.file_name());
2091                     t!(fs::create_dir_all(&dst));
2092                     builder.cp_r(&entry.path(), &dst);
2093                     break;
2094                 }
2095             }
2096         }
2097
2098         // Prepare the overlay
2099         let overlay = tmp.join("lldb-overlay");
2100         drop(fs::remove_dir_all(&overlay));
2101         builder.create_dir(&overlay);
2102         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2103         builder.create(&overlay.join("version"), &builder.lldb_vers());
2104
2105         // Generate the installer tarball
2106         let mut cmd = rust_installer(builder);
2107         cmd.arg("generate")
2108             .arg("--product-name=Rust")
2109             .arg("--rel-manifest-dir=rustlib")
2110             .arg("--success-message=lldb-installed.")
2111             .arg("--image-dir").arg(&image)
2112             .arg("--work-dir").arg(&tmpdir(builder))
2113             .arg("--output-dir").arg(&distdir(builder))
2114             .arg("--non-installed-overlay").arg(&overlay)
2115             .arg(format!("--package-name={}-{}", name, target))
2116             .arg("--legacy-manifest-dirs=rustlib,cargo")
2117             .arg("--component-name=lldb-preview");
2118
2119
2120         builder.run(&mut cmd);
2121         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2122     }
2123 }