src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs;
  23 use std::io::Write;
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use crate::{Compiler, Mode, LLVM_TOOLS};
  30 use crate::channel;
  31 use crate::util::{libdir, is_dylib, exe};
  32 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  33 use crate::compile;
  34 use crate::tool::{self, Tool};
  35 use crate::cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(builder: &Builder, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, builder.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, builder.rls_package_vers())
  43     } else if component == "clippy" {
  44         format!("{}-{}", component, builder.clippy_package_vers())
  45     } else if component == "rustfmt" {
  46         format!("{}-{}", component, builder.rustfmt_package_vers())
  47     } else if component == "llvm-tools" {
  48         format!("{}-{}", component, builder.llvm_tools_package_vers())
  49     } else if component == "lldb" {
  50         format!("{}-{}", component, builder.lldb_package_vers())
  51     } else {
  52         assert!(component.starts_with("rust"));
  53         format!("{}-{}", component, builder.rust_package_vers())
  54     }
  55 }
  56
  57 fn distdir(builder: &Builder) -> PathBuf {
  58     builder.out.join("dist")
  59 }
  60
  61 pub fn tmpdir(builder: &Builder) -> PathBuf {
  62     builder.out.join("tmp/dist")
  63 }
  64
  65 fn rust_installer(builder: &Builder) -> Command {
  66     builder.tool_cmd(Tool::RustInstaller)
  67 }
  68
  69 fn missing_tool(tool_name: &str, skip: bool) {
  70     if skip {
  71         println!("Unable to build {}, skipping dist", tool_name)
  72     } else {
  73         panic!("Unable to build {}", tool_name)
  74     }
  75 }
  76
  77 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  78 pub struct Docs {
  79     pub stage: u32,
  80     pub host: Interned<String>,
  81 }
  82
  83 impl Step for Docs {
  84     type Output = PathBuf;
  85     const DEFAULT: bool = true;
  86
  87     fn should_run(run: ShouldRun) -> ShouldRun {
  88         run.path("src/doc")
  89     }
  90
  91     fn make_run(run: RunConfig) {
  92         run.builder.ensure(Docs {
  93             stage: run.builder.top_stage,
  94             host: run.target,
  95         });
  96     }
  97
  98     /// Builds the `rust-docs` installer component.
  99     fn run(self, builder: &Builder) -> PathBuf {
 100         let host = self.host;
 101
 102         let name = pkgname(builder, "rust-docs");
 103
 104         builder.info(&format!("Dist docs ({})", host));
 105         if !builder.config.docs {
 106             builder.info("\tskipping - docs disabled");
 107             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 108         }
 109
 110         builder.default_doc(None);
 111
 112         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 113         let _ = fs::remove_dir_all(&image);
 114
 115         let dst = image.join("share/doc/rust/html");
 116         t!(fs::create_dir_all(&dst));
 117         let src = builder.doc_out(host);
 118         builder.cp_r(&src, &dst);
 119
 120         let mut cmd = rust_installer(builder);
 121         cmd.arg("generate")
 122            .arg("--product-name=Rust-Documentation")
 123            .arg("--rel-manifest-dir=rustlib")
 124            .arg("--success-message=Rust-documentation-is-installed.")
 125            .arg("--image-dir").arg(&image)
 126            .arg("--work-dir").arg(&tmpdir(builder))
 127            .arg("--output-dir").arg(&distdir(builder))
 128            .arg(format!("--package-name={}-{}", name, host))
 129            .arg("--component-name=rust-docs")
 130            .arg("--legacy-manifest-dirs=rustlib,cargo")
 131            .arg("--bulk-dirs=share/doc/rust/html");
 132         builder.run(&mut cmd);
 133         builder.remove_dir(&image);
 134
 135         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 136     }
 137 }
 138
 139 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 140 pub struct RustcDocs {
 141     pub stage: u32,
 142     pub host: Interned<String>,
 143 }
 144
 145 impl Step for RustcDocs {
 146     type Output = PathBuf;
 147     const DEFAULT: bool = true;
 148
 149     fn should_run(run: ShouldRun) -> ShouldRun {
 150         run.path("src/librustc")
 151     }
 152
 153     fn make_run(run: RunConfig) {
 154         run.builder.ensure(RustcDocs {
 155             stage: run.builder.top_stage,
 156             host: run.target,
 157         });
 158     }
 159
 160     /// Builds the `rustc-docs` installer component.
 161     fn run(self, builder: &Builder) -> PathBuf {
 162         let host = self.host;
 163
 164         let name = pkgname(builder, "rustc-docs");
 165
 166         builder.info(&format!("Dist compiler docs ({})", host));
 167         if !builder.config.compiler_docs {
 168             builder.info("\tskipping - compiler docs disabled");
 169             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 170         }
 171
 172         builder.default_doc(None);
 173
 174         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 175         let _ = fs::remove_dir_all(&image);
 176
 177         let dst = image.join("share/doc/rust/html");
 178         t!(fs::create_dir_all(&dst));
 179         let src = builder.compiler_doc_out(host);
 180         builder.cp_r(&src, &dst);
 181
 182         let mut cmd = rust_installer(builder);
 183         cmd.arg("generate")
 184            .arg("--product-name=Rustc-Documentation")
 185            .arg("--rel-manifest-dir=rustlib")
 186            .arg("--success-message=Rustc-documentation-is-installed.")
 187            .arg("--image-dir").arg(&image)
 188            .arg("--work-dir").arg(&tmpdir(builder))
 189            .arg("--output-dir").arg(&distdir(builder))
 190            .arg(format!("--package-name={}-{}", name, host))
 191            .arg("--component-name=rustc-docs")
 192            .arg("--legacy-manifest-dirs=rustlib,cargo")
 193            .arg("--bulk-dirs=share/doc/rust/html");
 194         builder.run(&mut cmd);
 195         builder.remove_dir(&image);
 196
 197         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 198     }
 199 }
 200
 201 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 202     let mut found = Vec::with_capacity(files.len());
 203
 204     for file in files {
 205         let file_path =
 206             path.iter()
 207                 .map(|dir| dir.join(file))
 208                 .find(|p| p.exists());
 209
 210         if let Some(file_path) = file_path {
 211             found.push(file_path);
 212         } else {
 213             panic!("Could not find '{}' in {:?}", file, path);
 214         }
 215     }
 216
 217     found
 218 }
 219
 220 fn make_win_dist(
 221     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 222 ) {
 223     //Ask gcc where it keeps its stuff
 224     let mut cmd = Command::new(builder.cc(target_triple));
 225     cmd.arg("-print-search-dirs");
 226     let gcc_out = output(&mut cmd);
 227
 228     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 229     let mut lib_path = Vec::new();
 230
 231     for line in gcc_out.lines() {
 232         let idx = line.find(':').unwrap();
 233         let key = &line[..idx];
 234         let trim_chars: &[_] = &[' ', '='];
 235         let value =
 236             line[(idx + 1)..]
 237                 .trim_left_matches(trim_chars)
 238                 .split(';')
 239                 .map(PathBuf::from);
 240
 241         if key == "programs" {
 242             bin_path.extend(value);
 243         } else if key == "libraries" {
 244             lib_path.extend(value);
 245         }
 246     }
 247
 248     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 249     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 250     if target_triple.starts_with("i686-") {
 251         rustc_dlls.push("libgcc_s_dw2-1.dll");
 252     } else {
 253         rustc_dlls.push("libgcc_s_seh-1.dll");
 254     }
 255
 256     let target_libs = [ //MinGW libs
 257         "libgcc.a",
 258         "libgcc_eh.a",
 259         "libgcc_s.a",
 260         "libm.a",
 261         "libmingw32.a",
 262         "libmingwex.a",
 263         "libstdc++.a",
 264         "libiconv.a",
 265         "libmoldname.a",
 266         "libpthread.a",
 267         //Windows import libs
 268         "libadvapi32.a",
 269         "libbcrypt.a",
 270         "libcomctl32.a",
 271         "libcomdlg32.a",
 272         "libcredui.a",
 273         "libcrypt32.a",
 274         "libdbghelp.a",
 275         "libgdi32.a",
 276         "libimagehlp.a",
 277         "libiphlpapi.a",
 278         "libkernel32.a",
 279         "libmsimg32.a",
 280         "libmsvcrt.a",
 281         "libodbc32.a",
 282         "libole32.a",
 283         "liboleaut32.a",
 284         "libopengl32.a",
 285         "libpsapi.a",
 286         "librpcrt4.a",
 287         "libsecur32.a",
 288         "libsetupapi.a",
 289         "libshell32.a",
 290         "libsynchronization.a",
 291         "libuser32.a",
 292         "libuserenv.a",
 293         "libuuid.a",
 294         "libwinhttp.a",
 295         "libwinmm.a",
 296         "libwinspool.a",
 297         "libws2_32.a",
 298         "libwsock32.a",
 299     ];
 300
 301     //Find mingw artifacts we want to bundle
 302     let target_tools = find_files(&target_tools, &bin_path);
 303     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 304     let target_libs = find_files(&target_libs, &lib_path);
 305
 306     // Copy runtime dlls next to rustc.exe
 307     let dist_bin_dir = rust_root.join("bin/");
 308     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 309     for src in rustc_dlls {
 310         builder.copy_to_folder(&src, &dist_bin_dir);
 311     }
 312
 313     //Copy platform tools to platform-specific bin directory
 314     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 315     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 316     for src in target_tools {
 317         builder.copy_to_folder(&src, &target_bin_dir);
 318     }
 319
 320     // Warn windows-gnu users that the bundled GCC cannot compile C files
 321     builder.create(
 322         &target_bin_dir.join("GCC-WARNING.txt"),
 323         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 324          used as a linker. In order to be able to compile projects containing C code use\
 325          the GCC provided by MinGW or Cygwin."
 326     );
 327
 328     //Copy platform libs to platform-specific lib directory
 329     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 330     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 331     for src in target_libs {
 332         builder.copy_to_folder(&src, &target_lib_dir);
 333     }
 334 }
 335
 336 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 337 pub struct Mingw {
 338     pub host: Interned<String>,
 339 }
 340
 341 impl Step for Mingw {
 342     type Output = Option<PathBuf>;
 343     const DEFAULT: bool = true;
 344
 345     fn should_run(run: ShouldRun) -> ShouldRun {
 346         run.never()
 347     }
 348
 349     fn make_run(run: RunConfig) {
 350         run.builder.ensure(Mingw { host: run.target });
 351     }
 352
 353     /// Build the `rust-mingw` installer component.
 354     ///
 355     /// This contains all the bits and pieces to run the MinGW Windows targets
 356     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 357     fn run(self, builder: &Builder) -> Option<PathBuf> {
 358         let host = self.host;
 359
 360         if !host.contains("pc-windows-gnu") {
 361             return None;
 362         }
 363
 364         builder.info(&format!("Dist mingw ({})", host));
 365         let name = pkgname(builder, "rust-mingw");
 366         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 367         let _ = fs::remove_dir_all(&image);
 368         t!(fs::create_dir_all(&image));
 369
 370         // The first argument is a "temporary directory" which is just
 371         // thrown away (this contains the runtime DLLs included in the rustc package
 372         // above) and the second argument is where to place all the MinGW components
 373         // (which is what we want).
 374         make_win_dist(&tmpdir(builder), &image, host, &builder);
 375
 376         let mut cmd = rust_installer(builder);
 377         cmd.arg("generate")
 378            .arg("--product-name=Rust-MinGW")
 379            .arg("--rel-manifest-dir=rustlib")
 380            .arg("--success-message=Rust-MinGW-is-installed.")
 381            .arg("--image-dir").arg(&image)
 382            .arg("--work-dir").arg(&tmpdir(builder))
 383            .arg("--output-dir").arg(&distdir(builder))
 384            .arg(format!("--package-name={}-{}", name, host))
 385            .arg("--component-name=rust-mingw")
 386            .arg("--legacy-manifest-dirs=rustlib,cargo");
 387         builder.run(&mut cmd);
 388         t!(fs::remove_dir_all(&image));
 389         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 390     }
 391 }
 392
 393 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 394 pub struct Rustc {
 395     pub compiler: Compiler,
 396 }
 397
 398 impl Step for Rustc {
 399     type Output = PathBuf;
 400     const DEFAULT: bool = true;
 401     const ONLY_HOSTS: bool = true;
 402
 403     fn should_run(run: ShouldRun) -> ShouldRun {
 404         run.path("src/librustc")
 405     }
 406
 407     fn make_run(run: RunConfig) {
 408         run.builder.ensure(Rustc {
 409             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 410         });
 411     }
 412
 413     /// Creates the `rustc` installer component.
 414     fn run(self, builder: &Builder) -> PathBuf {
 415         let compiler = self.compiler;
 416         let host = self.compiler.host;
 417
 418         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 419         let name = pkgname(builder, "rustc");
 420         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 421         let _ = fs::remove_dir_all(&image);
 422         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 423         let _ = fs::remove_dir_all(&overlay);
 424
 425         // Prepare the rustc "image", what will actually end up getting installed
 426         prepare_image(builder, compiler, &image);
 427
 428         // Prepare the overlay which is part of the tarball but won't actually be
 429         // installed
 430         let cp = |file: &str| {
 431             builder.install(&builder.src.join(file), &overlay, 0o644);
 432         };
 433         cp("COPYRIGHT");
 434         cp("LICENSE-APACHE");
 435         cp("LICENSE-MIT");
 436         cp("README.md");
 437         // tiny morsel of metadata is used by rust-packaging
 438         let version = builder.rust_version();
 439         builder.create(&overlay.join("version"), &version);
 440         if let Some(sha) = builder.rust_sha() {
 441             builder.create(&overlay.join("git-commit-hash"), &sha);
 442         }
 443
 444         // On MinGW we've got a few runtime DLL dependencies that we need to
 445         // include. The first argument to this script is where to put these DLLs
 446         // (the image we're creating), and the second argument is a junk directory
 447         // to ignore all other MinGW stuff the script creates.
 448         //
 449         // On 32-bit MinGW we're always including a DLL which needs some extra
 450         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 451         // anything requiring us to distribute a license, but it's likely the
 452         // install will *also* include the rust-mingw package, which also needs
 453         // licenses, so to be safe we just include it here in all MinGW packages.
 454         if host.contains("pc-windows-gnu") {
 455             make_win_dist(&image, &tmpdir(builder), host, builder);
 456
 457             let dst = image.join("share/doc");
 458             t!(fs::create_dir_all(&dst));
 459             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 460         }
 461
 462         // Finally, wrap everything up in a nice tarball!
 463         let mut cmd = rust_installer(builder);
 464         cmd.arg("generate")
 465            .arg("--product-name=Rust")
 466            .arg("--rel-manifest-dir=rustlib")
 467            .arg("--success-message=Rust-is-ready-to-roll.")
 468            .arg("--image-dir").arg(&image)
 469            .arg("--work-dir").arg(&tmpdir(builder))
 470            .arg("--output-dir").arg(&distdir(builder))
 471            .arg("--non-installed-overlay").arg(&overlay)
 472            .arg(format!("--package-name={}-{}", name, host))
 473            .arg("--component-name=rustc")
 474            .arg("--legacy-manifest-dirs=rustlib,cargo");
 475         builder.run(&mut cmd);
 476         builder.remove_dir(&image);
 477         builder.remove_dir(&overlay);
 478
 479         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 480
 481         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 482             let host = compiler.host;
 483             let src = builder.sysroot(compiler);
 484             let libdir = libdir(&host);
 485
 486             // Copy rustc/rustdoc binaries
 487             t!(fs::create_dir_all(image.join("bin")));
 488             builder.cp_r(&src.join("bin"), &image.join("bin"));
 489
 490             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 491
 492             // Copy runtime DLLs needed by the compiler
 493             if libdir != "bin" {
 494                 for entry in builder.read_dir(&src.join(libdir)) {
 495                     let name = entry.file_name();
 496                     if let Some(s) = name.to_str() {
 497                         if is_dylib(s) {
 498                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 499                         }
 500                     }
 501                 }
 502             }
 503
 504             // Copy over the codegen backends
 505             let backends_src = builder.sysroot_codegen_backends(compiler);
 506             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 507             let backends_dst = image.join(&backends_rel);
 508             t!(fs::create_dir_all(&backends_dst));
 509             builder.cp_r(&backends_src, &backends_dst);
 510
 511             // Copy libLLVM.so to the lib dir as well, if needed. While not
 512             // technically needed by rustc itself it's needed by lots of other
 513             // components like the llvm tools and LLD. LLD is included below and
 514             // tools/LLDB come later, so let's just throw it in the rustc
 515             // component for now.
 516             maybe_install_llvm_dylib(builder, host, image);
 517
 518             // Copy over lld if it's there
 519             if builder.config.lld_enabled {
 520                 let exe = exe("rust-lld", &compiler.host);
 521                 let src = builder.sysroot_libdir(compiler, host)
 522                     .parent()
 523                     .unwrap()
 524                     .join("bin")
 525                     .join(&exe);
 526                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 527                 let dst = image.join("lib/rustlib")
 528                     .join(&*host)
 529                     .join("bin")
 530                     .join(&exe);
 531                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 532                 builder.copy(&src, &dst);
 533             }
 534
 535             // Man pages
 536             t!(fs::create_dir_all(image.join("share/man/man1")));
 537             let man_src = builder.src.join("src/doc/man");
 538             let man_dst = image.join("share/man/man1");
 539             let month_year = t!(time::strftime("%B %Y", &time::now()));
 540             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 541             // to hardlink, and we don't want to edit the source templates
 542             for file_entry in builder.read_dir(&man_src) {
 543                 let page_src = file_entry.path();
 544                 let page_dst = man_dst.join(file_entry.file_name());
 545                 t!(fs::copy(&page_src, &page_dst));
 546                 // template in month/year and version number
 547                 builder.replace_in_file(&page_dst,
 548                                 &[("<INSERT DATE HERE>", &month_year),
 549                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 550             }
 551
 552             // Debugger scripts
 553             builder.ensure(DebuggerScripts {
 554                 sysroot: INTERNER.intern_path(image.to_owned()),
 555                 host,
 556             });
 557
 558             // Misc license info
 559             let cp = |file: &str| {
 560                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 561             };
 562             cp("COPYRIGHT");
 563             cp("LICENSE-APACHE");
 564             cp("LICENSE-MIT");
 565             cp("README.md");
 566         }
 567     }
 568 }
 569
 570 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 571 pub struct DebuggerScripts {
 572     pub sysroot: Interned<PathBuf>,
 573     pub host: Interned<String>,
 574 }
 575
 576 impl Step for DebuggerScripts {
 577     type Output = ();
 578
 579     fn should_run(run: ShouldRun) -> ShouldRun {
 580         run.path("src/lldb_batchmode.py")
 581     }
 582
 583     fn make_run(run: RunConfig) {
 584         run.builder.ensure(DebuggerScripts {
 585             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 586             host: run.target,
 587         });
 588     }
 589
 590     /// Copies debugger scripts for `target` into the `sysroot` specified.
 591     fn run(self, builder: &Builder) {
 592         let host = self.host;
 593         let sysroot = self.sysroot;
 594         let dst = sysroot.join("lib/rustlib/etc");
 595         t!(fs::create_dir_all(&dst));
 596         let cp_debugger_script = |file: &str| {
 597             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 598         };
 599         if host.contains("windows-msvc") {
 600             // windbg debugger scripts
 601             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 602                 0o755);
 603
 604             cp_debugger_script("natvis/intrinsic.natvis");
 605             cp_debugger_script("natvis/liballoc.natvis");
 606             cp_debugger_script("natvis/libcore.natvis");
 607         } else {
 608             cp_debugger_script("debugger_pretty_printers_common.py");
 609
 610             // gdb debugger scripts
 611             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 612                     0o755);
 613
 614             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 615             cp_debugger_script("gdb_rust_pretty_printing.py");
 616
 617             // lldb debugger scripts
 618             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 619                     0o755);
 620
 621             cp_debugger_script("lldb_rust_formatters.py");
 622         }
 623     }
 624 }
 625
 626 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 627 pub struct Std {
 628     pub compiler: Compiler,
 629     pub target: Interned<String>,
 630 }
 631
 632 impl Step for Std {
 633     type Output = PathBuf;
 634     const DEFAULT: bool = true;
 635
 636     fn should_run(run: ShouldRun) -> ShouldRun {
 637         run.path("src/libstd")
 638     }
 639
 640     fn make_run(run: RunConfig) {
 641         run.builder.ensure(Std {
 642             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 643             target: run.target,
 644         });
 645     }
 646
 647     fn run(self, builder: &Builder) -> PathBuf {
 648         let compiler = self.compiler;
 649         let target = self.target;
 650
 651         let name = pkgname(builder, "rust-std");
 652         builder.info(&format!("Dist std stage{} ({} -> {})",
 653             compiler.stage, &compiler.host, target));
 654
 655         // The only true set of target libraries came from the build triple, so
 656         // let's reduce redundant work by only producing archives from that host.
 657         if compiler.host != builder.config.build {
 658             builder.info("\tskipping, not a build host");
 659             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 660         }
 661
 662         // We want to package up as many target libraries as possible
 663         // for the `rust-std` package, so if this is a host target we
 664         // depend on librustc and otherwise we just depend on libtest.
 665         if builder.hosts.iter().any(|t| t == target) {
 666             builder.ensure(compile::Rustc { compiler, target });
 667         } else {
 668             if builder.no_std(target) == Some(true) {
 669                 // the `test` doesn't compile for no-std targets
 670                 builder.ensure(compile::Std { compiler, target });
 671             } else {
 672                 builder.ensure(compile::Test { compiler, target });
 673             }
 674         }
 675
 676         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 677         let _ = fs::remove_dir_all(&image);
 678
 679         let dst = image.join("lib/rustlib").join(target);
 680         t!(fs::create_dir_all(&dst));
 681         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 682         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 683         builder.cp_filtered(&src, &dst, &|path| {
 684             let name = path.file_name().and_then(|s| s.to_str());
 685             name != Some(builder.config.rust_codegen_backends_dir.as_str()) &&
 686                 name != Some("bin")
 687
 688         });
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=std-is-standing-at-the-ready.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(builder))
 697            .arg("--output-dir").arg(&distdir(builder))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-std-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         builder.run(&mut cmd);
 702         builder.remove_dir(&image);
 703         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 708 pub struct Analysis {
 709     pub compiler: Compiler,
 710     pub target: Interned<String>,
 711 }
 712
 713 impl Step for Analysis {
 714     type Output = PathBuf;
 715     const DEFAULT: bool = true;
 716
 717     fn should_run(run: ShouldRun) -> ShouldRun {
 718         let builder = run.builder;
 719         run.path("analysis").default_condition(builder.config.extended)
 720     }
 721
 722     fn make_run(run: RunConfig) {
 723         run.builder.ensure(Analysis {
 724             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 725             target: run.target,
 726         });
 727     }
 728
 729     /// Creates a tarball of save-analysis metadata, if available.
 730     fn run(self, builder: &Builder) -> PathBuf {
 731         let compiler = self.compiler;
 732         let target = self.target;
 733         assert!(builder.config.extended);
 734         builder.info("Dist analysis");
 735         let name = pkgname(builder, "rust-analysis");
 736
 737         if &compiler.host != builder.config.build {
 738             builder.info("\tskipping, not a build host");
 739             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 740         }
 741
 742         builder.ensure(Std { compiler, target });
 743
 744         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 745         // stage2 artifacts is simply copied from stage1 in that case.
 746         let compiler = if builder.force_use_stage1(compiler, target) {
 747             builder.compiler(1, compiler.host)
 748         } else {
 749             compiler.clone()
 750         };
 751
 752         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 753
 754         let src = builder.stage_out(compiler, Mode::Std)
 755             .join(target).join(builder.cargo_dir()).join("deps");
 756
 757         let image_src = src.join("save-analysis");
 758         let dst = image.join("lib/rustlib").join(target).join("analysis");
 759         t!(fs::create_dir_all(&dst));
 760         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 761         builder.cp_r(&image_src, &dst);
 762
 763         let mut cmd = rust_installer(builder);
 764         cmd.arg("generate")
 765            .arg("--product-name=Rust")
 766            .arg("--rel-manifest-dir=rustlib")
 767            .arg("--success-message=save-analysis-saved.")
 768            .arg("--image-dir").arg(&image)
 769            .arg("--work-dir").arg(&tmpdir(builder))
 770            .arg("--output-dir").arg(&distdir(builder))
 771            .arg(format!("--package-name={}-{}", name, target))
 772            .arg(format!("--component-name=rust-analysis-{}", target))
 773            .arg("--legacy-manifest-dirs=rustlib,cargo");
 774         builder.run(&mut cmd);
 775         builder.remove_dir(&image);
 776         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 777     }
 778 }
 779
 780 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 781     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 782         let spath = match path.to_str() {
 783             Some(path) => path,
 784             None => return false,
 785         };
 786         if spath.ends_with("~") || spath.ends_with(".pyc") {
 787             return false
 788         }
 789         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 790             (spath.ends_with(".ll") ||
 791              spath.ends_with(".td") ||
 792              spath.ends_with(".s")) {
 793             return false
 794         }
 795         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 796             return false
 797         }
 798
 799         let full_path = Path::new(dir).join(path);
 800         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 801             return false;
 802         }
 803
 804         let excludes = [
 805             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 806             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 807             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 808             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 809         ];
 810         !path.iter()
 811              .map(|s| s.to_str().unwrap())
 812              .any(|s| excludes.contains(&s))
 813     }
 814
 815     // Copy the directories using our filter
 816     for item in src_dirs {
 817         let dst = &dst_dir.join(item);
 818         t!(fs::create_dir_all(dst));
 819         builder.cp_filtered(
 820             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 821     }
 822 }
 823
 824 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 825 pub struct Src;
 826
 827 impl Step for Src {
 828     /// The output path of the src installer tarball
 829     type Output = PathBuf;
 830     const DEFAULT: bool = true;
 831     const ONLY_HOSTS: bool = true;
 832
 833     fn should_run(run: ShouldRun) -> ShouldRun {
 834         run.path("src")
 835     }
 836
 837     fn make_run(run: RunConfig) {
 838         run.builder.ensure(Src);
 839     }
 840
 841     /// Creates the `rust-src` installer component
 842     fn run(self, builder: &Builder) -> PathBuf {
 843         builder.info("Dist src");
 844
 845         let name = pkgname(builder, "rust-src");
 846         let image = tmpdir(builder).join(format!("{}-image", name));
 847         let _ = fs::remove_dir_all(&image);
 848
 849         let dst = image.join("lib/rustlib/src");
 850         let dst_src = dst.join("rust");
 851         t!(fs::create_dir_all(&dst_src));
 852
 853         let src_files = [
 854             "Cargo.lock",
 855         ];
 856         // This is the reduced set of paths which will become the rust-src component
 857         // (essentially libstd and all of its path dependencies)
 858         let std_src_dirs = [
 859             "src/build_helper",
 860             "src/liballoc",
 861             "src/libcore",
 862             "src/libpanic_abort",
 863             "src/libpanic_unwind",
 864             "src/librustc_asan",
 865             "src/librustc_lsan",
 866             "src/librustc_msan",
 867             "src/librustc_tsan",
 868             "src/libstd",
 869             "src/libunwind",
 870             "src/libtest",
 871             "src/libterm",
 872             "src/libprofiler_builtins",
 873             "src/stdsimd",
 874             "src/libproc_macro",
 875             "src/tools/rustc-std-workspace-core",
 876         ];
 877
 878         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 879         for file in src_files.iter() {
 880             builder.copy(&builder.src.join(file), &dst_src.join(file));
 881         }
 882
 883         // Create source tarball in rust-installer format
 884         let mut cmd = rust_installer(builder);
 885         cmd.arg("generate")
 886            .arg("--product-name=Rust")
 887            .arg("--rel-manifest-dir=rustlib")
 888            .arg("--success-message=Awesome-Source.")
 889            .arg("--image-dir").arg(&image)
 890            .arg("--work-dir").arg(&tmpdir(builder))
 891            .arg("--output-dir").arg(&distdir(builder))
 892            .arg(format!("--package-name={}", name))
 893            .arg("--component-name=rust-src")
 894            .arg("--legacy-manifest-dirs=rustlib,cargo");
 895         builder.run(&mut cmd);
 896
 897         builder.remove_dir(&image);
 898         distdir(builder).join(&format!("{}.tar.gz", name))
 899     }
 900 }
 901
 902 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 903
 904 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 905 pub struct PlainSourceTarball;
 906
 907 impl Step for PlainSourceTarball {
 908     /// Produces the location of the tarball generated
 909     type Output = PathBuf;
 910     const DEFAULT: bool = true;
 911     const ONLY_HOSTS: bool = true;
 912
 913     fn should_run(run: ShouldRun) -> ShouldRun {
 914         let builder = run.builder;
 915         run.path("src").default_condition(builder.config.rust_dist_src)
 916     }
 917
 918     fn make_run(run: RunConfig) {
 919         run.builder.ensure(PlainSourceTarball);
 920     }
 921
 922     /// Creates the plain source tarball
 923     fn run(self, builder: &Builder) -> PathBuf {
 924         builder.info("Create plain source tarball");
 925
 926         // Make sure that the root folder of tarball has the correct name
 927         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 928         let plain_dst_src = tmpdir(builder).join(&plain_name);
 929         let _ = fs::remove_dir_all(&plain_dst_src);
 930         t!(fs::create_dir_all(&plain_dst_src));
 931
 932         // This is the set of root paths which will become part of the source package
 933         let src_files = [
 934             "COPYRIGHT",
 935             "LICENSE-APACHE",
 936             "LICENSE-MIT",
 937             "CONTRIBUTING.md",
 938             "README.md",
 939             "RELEASES.md",
 940             "configure",
 941             "x.py",
 942             "config.toml.example",
 943             "Cargo.toml",
 944             "Cargo.lock",
 945         ];
 946         let src_dirs = [
 947             "src",
 948         ];
 949
 950         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 951
 952         // Copy the files normally
 953         for item in &src_files {
 954             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 955         }
 956
 957         // Create the version file
 958         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 959         if let Some(sha) = builder.rust_sha() {
 960             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 961         }
 962
 963         // If we're building from git sources, we need to vendor a complete distribution.
 964         if builder.rust_info.is_git() {
 965             // Get cargo-vendor installed, if it isn't already.
 966             let mut has_cargo_vendor = false;
 967             let mut cmd = Command::new(&builder.initial_cargo);
 968             for line in output(cmd.arg("install").arg("--list")).lines() {
 969                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 970             }
 971             if !has_cargo_vendor {
 972                 let mut cmd = builder.cargo(
 973                     builder.compiler(0, builder.config.build),
 974                     Mode::ToolBootstrap,
 975                     builder.config.build,
 976                     "install"
 977                 );
 978                 cmd.arg("--force")
 979                    .arg("--debug")
 980                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 981                    .arg("cargo-vendor");
 982                 builder.run(&mut cmd);
 983             }
 984
 985             // Vendor all Cargo dependencies
 986             let mut cmd = Command::new(&builder.initial_cargo);
 987             cmd.arg("vendor")
 988                .current_dir(&plain_dst_src);
 989             builder.run(&mut cmd);
 990         }
 991
 992         // Create plain source tarball
 993         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
 994         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
 995         tarball.set_extension(""); // strip .gz
 996         tarball.set_extension(""); // strip .tar
 997         if let Some(dir) = tarball.parent() {
 998             builder.create_dir(&dir);
 999         }
1000         builder.info("running installer");
1001         let mut cmd = rust_installer(builder);
1002         cmd.arg("tarball")
1003            .arg("--input").arg(&plain_name)
1004            .arg("--output").arg(&tarball)
1005            .arg("--work-dir=.")
1006            .current_dir(tmpdir(builder));
1007         builder.run(&mut cmd);
1008         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1009     }
1010 }
1011
1012 // We have to run a few shell scripts, which choke quite a bit on both `\`
1013 // characters and on `C:\` paths, so normalize both of them away.
1014 pub fn sanitize_sh(path: &Path) -> String {
1015     let path = path.to_str().unwrap().replace("\\", "/");
1016     return change_drive(&path).unwrap_or(path);
1017
1018     fn change_drive(s: &str) -> Option<String> {
1019         let mut ch = s.chars();
1020         let drive = ch.next().unwrap_or('C');
1021         if ch.next() != Some(':') {
1022             return None
1023         }
1024         if ch.next() != Some('/') {
1025             return None
1026         }
1027         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1028     }
1029 }
1030
1031 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1032 pub struct Cargo {
1033     pub stage: u32,
1034     pub target: Interned<String>,
1035 }
1036
1037 impl Step for Cargo {
1038     type Output = PathBuf;
1039     const ONLY_HOSTS: bool = true;
1040
1041     fn should_run(run: ShouldRun) -> ShouldRun {
1042         run.path("cargo")
1043     }
1044
1045     fn make_run(run: RunConfig) {
1046         run.builder.ensure(Cargo {
1047             stage: run.builder.top_stage,
1048             target: run.target,
1049         });
1050     }
1051
1052     fn run(self, builder: &Builder) -> PathBuf {
1053         let stage = self.stage;
1054         let target = self.target;
1055
1056         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1057         let src = builder.src.join("src/tools/cargo");
1058         let etc = src.join("src/etc");
1059         let release_num = builder.release_num("cargo");
1060         let name = pkgname(builder, "cargo");
1061         let version = builder.cargo_info.version(builder, &release_num);
1062
1063         let tmp = tmpdir(builder);
1064         let image = tmp.join("cargo-image");
1065         drop(fs::remove_dir_all(&image));
1066         builder.create_dir(&image);
1067
1068         // Prepare the image directory
1069         builder.create_dir(&image.join("share/zsh/site-functions"));
1070         builder.create_dir(&image.join("etc/bash_completion.d"));
1071         let cargo = builder.ensure(tool::Cargo {
1072             compiler: builder.compiler(stage, builder.config.build),
1073             target
1074         });
1075         builder.install(&cargo, &image.join("bin"), 0o755);
1076         for man in t!(etc.join("man").read_dir()) {
1077             let man = t!(man);
1078             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1079         }
1080         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1081         builder.copy(&etc.join("cargo.bashcomp.sh"),
1082              &image.join("etc/bash_completion.d/cargo"));
1083         let doc = image.join("share/doc/cargo");
1084         builder.install(&src.join("README.md"), &doc, 0o644);
1085         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1086         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1087         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1088
1089         // Prepare the overlay
1090         let overlay = tmp.join("cargo-overlay");
1091         drop(fs::remove_dir_all(&overlay));
1092         builder.create_dir(&overlay);
1093         builder.install(&src.join("README.md"), &overlay, 0o644);
1094         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1095         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1096         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1097         builder.create(&overlay.join("version"), &version);
1098
1099         // Generate the installer tarball
1100         let mut cmd = rust_installer(builder);
1101         cmd.arg("generate")
1102            .arg("--product-name=Rust")
1103            .arg("--rel-manifest-dir=rustlib")
1104            .arg("--success-message=Rust-is-ready-to-roll.")
1105            .arg("--image-dir").arg(&image)
1106            .arg("--work-dir").arg(&tmpdir(builder))
1107            .arg("--output-dir").arg(&distdir(builder))
1108            .arg("--non-installed-overlay").arg(&overlay)
1109            .arg(format!("--package-name={}-{}", name, target))
1110            .arg("--component-name=cargo")
1111            .arg("--legacy-manifest-dirs=rustlib,cargo");
1112         builder.run(&mut cmd);
1113         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1114     }
1115 }
1116
1117 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1118 pub struct Rls {
1119     pub stage: u32,
1120     pub target: Interned<String>,
1121 }
1122
1123 impl Step for Rls {
1124     type Output = Option<PathBuf>;
1125     const ONLY_HOSTS: bool = true;
1126
1127     fn should_run(run: ShouldRun) -> ShouldRun {
1128         run.path("rls")
1129     }
1130
1131     fn make_run(run: RunConfig) {
1132         run.builder.ensure(Rls {
1133             stage: run.builder.top_stage,
1134             target: run.target,
1135         });
1136     }
1137
1138     fn run(self, builder: &Builder) -> Option<PathBuf> {
1139         let stage = self.stage;
1140         let target = self.target;
1141         assert!(builder.config.extended);
1142
1143         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1144         let src = builder.src.join("src/tools/rls");
1145         let release_num = builder.release_num("rls");
1146         let name = pkgname(builder, "rls");
1147         let version = builder.rls_info.version(builder, &release_num);
1148
1149         let tmp = tmpdir(builder);
1150         let image = tmp.join("rls-image");
1151         drop(fs::remove_dir_all(&image));
1152         t!(fs::create_dir_all(&image));
1153
1154         // Prepare the image directory
1155         // We expect RLS to build, because we've exited this step above if tool
1156         // state for RLS isn't testing.
1157         let rls = builder.ensure(tool::Rls {
1158             compiler: builder.compiler(stage, builder.config.build),
1159             target, extra_features: Vec::new()
1160         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1161
1162         builder.install(&rls, &image.join("bin"), 0o755);
1163         let doc = image.join("share/doc/rls");
1164         builder.install(&src.join("README.md"), &doc, 0o644);
1165         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1166         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1167
1168         // Prepare the overlay
1169         let overlay = tmp.join("rls-overlay");
1170         drop(fs::remove_dir_all(&overlay));
1171         t!(fs::create_dir_all(&overlay));
1172         builder.install(&src.join("README.md"), &overlay, 0o644);
1173         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1174         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1175         builder.create(&overlay.join("version"), &version);
1176
1177         // Generate the installer tarball
1178         let mut cmd = rust_installer(builder);
1179         cmd.arg("generate")
1180            .arg("--product-name=Rust")
1181            .arg("--rel-manifest-dir=rustlib")
1182            .arg("--success-message=RLS-ready-to-serve.")
1183            .arg("--image-dir").arg(&image)
1184            .arg("--work-dir").arg(&tmpdir(builder))
1185            .arg("--output-dir").arg(&distdir(builder))
1186            .arg("--non-installed-overlay").arg(&overlay)
1187            .arg(format!("--package-name={}-{}", name, target))
1188            .arg("--legacy-manifest-dirs=rustlib,cargo")
1189            .arg("--component-name=rls-preview");
1190
1191         builder.run(&mut cmd);
1192         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1193     }
1194 }
1195
1196 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1197 pub struct Clippy {
1198     pub stage: u32,
1199     pub target: Interned<String>,
1200 }
1201
1202 impl Step for Clippy {
1203     type Output = Option<PathBuf>;
1204     const ONLY_HOSTS: bool = true;
1205
1206     fn should_run(run: ShouldRun) -> ShouldRun {
1207         run.path("clippy")
1208     }
1209
1210     fn make_run(run: RunConfig) {
1211         run.builder.ensure(Clippy {
1212             stage: run.builder.top_stage,
1213             target: run.target,
1214         });
1215     }
1216
1217     fn run(self, builder: &Builder) -> Option<PathBuf> {
1218         let stage = self.stage;
1219         let target = self.target;
1220         assert!(builder.config.extended);
1221
1222         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1223         let src = builder.src.join("src/tools/clippy");
1224         let release_num = builder.release_num("clippy");
1225         let name = pkgname(builder, "clippy");
1226         let version = builder.clippy_info.version(builder, &release_num);
1227
1228         let tmp = tmpdir(builder);
1229         let image = tmp.join("clippy-image");
1230         drop(fs::remove_dir_all(&image));
1231         builder.create_dir(&image);
1232
1233         // Prepare the image directory
1234         // We expect clippy to build, because we've exited this step above if tool
1235         // state for clippy isn't testing.
1236         let clippy = builder.ensure(tool::Clippy {
1237             compiler: builder.compiler(stage, builder.config.build),
1238             target, extra_features: Vec::new()
1239         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1240         let cargoclippy = builder.ensure(tool::CargoClippy {
1241             compiler: builder.compiler(stage, builder.config.build),
1242             target, extra_features: Vec::new()
1243         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1244
1245         builder.install(&clippy, &image.join("bin"), 0o755);
1246         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1247         let doc = image.join("share/doc/clippy");
1248         builder.install(&src.join("README.md"), &doc, 0o644);
1249         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1250         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1251
1252         // Prepare the overlay
1253         let overlay = tmp.join("clippy-overlay");
1254         drop(fs::remove_dir_all(&overlay));
1255         t!(fs::create_dir_all(&overlay));
1256         builder.install(&src.join("README.md"), &overlay, 0o644);
1257         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1258         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1259         builder.create(&overlay.join("version"), &version);
1260
1261         // Generate the installer tarball
1262         let mut cmd = rust_installer(builder);
1263         cmd.arg("generate")
1264            .arg("--product-name=Rust")
1265            .arg("--rel-manifest-dir=rustlib")
1266            .arg("--success-message=clippy-ready-to-serve.")
1267            .arg("--image-dir").arg(&image)
1268            .arg("--work-dir").arg(&tmpdir(builder))
1269            .arg("--output-dir").arg(&distdir(builder))
1270            .arg("--non-installed-overlay").arg(&overlay)
1271            .arg(format!("--package-name={}-{}", name, target))
1272            .arg("--legacy-manifest-dirs=rustlib,cargo")
1273            .arg("--component-name=clippy-preview");
1274
1275         builder.run(&mut cmd);
1276         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1277     }
1278 }
1279
1280 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1281 pub struct Rustfmt {
1282     pub stage: u32,
1283     pub target: Interned<String>,
1284 }
1285
1286 impl Step for Rustfmt {
1287     type Output = Option<PathBuf>;
1288     const ONLY_HOSTS: bool = true;
1289
1290     fn should_run(run: ShouldRun) -> ShouldRun {
1291         run.path("rustfmt")
1292     }
1293
1294     fn make_run(run: RunConfig) {
1295         run.builder.ensure(Rustfmt {
1296             stage: run.builder.top_stage,
1297             target: run.target,
1298         });
1299     }
1300
1301     fn run(self, builder: &Builder) -> Option<PathBuf> {
1302         let stage = self.stage;
1303         let target = self.target;
1304
1305         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1306         let src = builder.src.join("src/tools/rustfmt");
1307         let release_num = builder.release_num("rustfmt");
1308         let name = pkgname(builder, "rustfmt");
1309         let version = builder.rustfmt_info.version(builder, &release_num);
1310
1311         let tmp = tmpdir(builder);
1312         let image = tmp.join("rustfmt-image");
1313         drop(fs::remove_dir_all(&image));
1314         builder.create_dir(&image);
1315
1316         // Prepare the image directory
1317         let rustfmt = builder.ensure(tool::Rustfmt {
1318             compiler: builder.compiler(stage, builder.config.build),
1319             target, extra_features: Vec::new()
1320         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1321         let cargofmt = builder.ensure(tool::Cargofmt {
1322             compiler: builder.compiler(stage, builder.config.build),
1323             target, extra_features: Vec::new()
1324         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1325
1326         builder.install(&rustfmt, &image.join("bin"), 0o755);
1327         builder.install(&cargofmt, &image.join("bin"), 0o755);
1328         let doc = image.join("share/doc/rustfmt");
1329         builder.install(&src.join("README.md"), &doc, 0o644);
1330         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1331         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1332
1333         // Prepare the overlay
1334         let overlay = tmp.join("rustfmt-overlay");
1335         drop(fs::remove_dir_all(&overlay));
1336         builder.create_dir(&overlay);
1337         builder.install(&src.join("README.md"), &overlay, 0o644);
1338         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1339         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1340         builder.create(&overlay.join("version"), &version);
1341
1342         // Generate the installer tarball
1343         let mut cmd = rust_installer(builder);
1344         cmd.arg("generate")
1345            .arg("--product-name=Rust")
1346            .arg("--rel-manifest-dir=rustlib")
1347            .arg("--success-message=rustfmt-ready-to-fmt.")
1348            .arg("--image-dir").arg(&image)
1349            .arg("--work-dir").arg(&tmpdir(builder))
1350            .arg("--output-dir").arg(&distdir(builder))
1351            .arg("--non-installed-overlay").arg(&overlay)
1352            .arg(format!("--package-name={}-{}", name, target))
1353            .arg("--legacy-manifest-dirs=rustlib,cargo")
1354            .arg("--component-name=rustfmt-preview");
1355
1356         builder.run(&mut cmd);
1357         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1358     }
1359 }
1360
1361 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1362 pub struct Extended {
1363     stage: u32,
1364     host: Interned<String>,
1365     target: Interned<String>,
1366 }
1367
1368 impl Step for Extended {
1369     type Output = ();
1370     const DEFAULT: bool = true;
1371     const ONLY_HOSTS: bool = true;
1372
1373     fn should_run(run: ShouldRun) -> ShouldRun {
1374         let builder = run.builder;
1375         run.path("extended").default_condition(builder.config.extended)
1376     }
1377
1378     fn make_run(run: RunConfig) {
1379         run.builder.ensure(Extended {
1380             stage: run.builder.top_stage,
1381             host: run.builder.config.build,
1382             target: run.target,
1383         });
1384     }
1385
1386     /// Creates a combined installer for the specified target in the provided stage.
1387     fn run(self, builder: &Builder) {
1388         let stage = self.stage;
1389         let target = self.target;
1390
1391         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1392
1393         let rustc_installer = builder.ensure(Rustc {
1394             compiler: builder.compiler(stage, target),
1395         });
1396         let cargo_installer = builder.ensure(Cargo { stage, target });
1397         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1398         let rls_installer = builder.ensure(Rls { stage, target });
1399         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1400         let clippy_installer = builder.ensure(Clippy { stage, target });
1401         let lldb_installer = builder.ensure(Lldb { target });
1402         let mingw_installer = builder.ensure(Mingw { host: target });
1403         let analysis_installer = builder.ensure(Analysis {
1404             compiler: builder.compiler(stage, self.host),
1405             target
1406         });
1407
1408         let docs_installer = builder.ensure(Docs { stage, host: target, });
1409         let std_installer = builder.ensure(Std {
1410             compiler: builder.compiler(stage, self.host),
1411             target,
1412         });
1413
1414         let tmp = tmpdir(builder);
1415         let overlay = tmp.join("extended-overlay");
1416         let etc = builder.src.join("src/etc/installer");
1417         let work = tmp.join("work");
1418
1419         let _ = fs::remove_dir_all(&overlay);
1420         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1421         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1422         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1423         let version = builder.rust_version();
1424         builder.create(&overlay.join("version"), &version);
1425         if let Some(sha) = builder.rust_sha() {
1426             builder.create(&overlay.join("git-commit-hash"), &sha);
1427         }
1428         builder.install(&etc.join("README.md"), &overlay, 0o644);
1429
1430         // When rust-std package split from rustc, we needed to ensure that during
1431         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1432         // the std files during uninstall. To do this ensure that rustc comes
1433         // before rust-std in the list below.
1434         let mut tarballs = Vec::new();
1435         tarballs.push(rustc_installer);
1436         tarballs.push(cargo_installer);
1437         tarballs.extend(rls_installer.clone());
1438         tarballs.extend(clippy_installer.clone());
1439         tarballs.extend(rustfmt_installer.clone());
1440         tarballs.extend(llvm_tools_installer);
1441         tarballs.extend(lldb_installer);
1442         tarballs.push(analysis_installer);
1443         tarballs.push(std_installer);
1444         if builder.config.docs {
1445             tarballs.push(docs_installer);
1446         }
1447         if target.contains("pc-windows-gnu") {
1448             tarballs.push(mingw_installer.unwrap());
1449         }
1450         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1451         for tarball in &tarballs[1..] {
1452             input_tarballs.push(",");
1453             input_tarballs.push(tarball);
1454         }
1455
1456         let mut cmd = rust_installer(builder);
1457         cmd.arg("combine")
1458             .arg("--product-name=Rust")
1459             .arg("--rel-manifest-dir=rustlib")
1460             .arg("--success-message=Rust-is-ready-to-roll.")
1461             .arg("--work-dir").arg(&work)
1462             .arg("--output-dir").arg(&distdir(builder))
1463             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1464             .arg("--legacy-manifest-dirs=rustlib,cargo")
1465             .arg("--input-tarballs").arg(input_tarballs)
1466             .arg("--non-installed-overlay").arg(&overlay);
1467         builder.run(&mut cmd);
1468
1469         let mut license = String::new();
1470         license += &builder.read(&builder.src.join("COPYRIGHT"));
1471         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1472         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1473         license.push_str("\n");
1474         license.push_str("\n");
1475
1476         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1477         let mut rtf = rtf.to_string();
1478         rtf.push_str("\n");
1479         for line in license.lines() {
1480             rtf.push_str(line);
1481             rtf.push_str("\\line ");
1482         }
1483         rtf.push_str("}");
1484
1485         fn filter(contents: &str, marker: &str) -> String {
1486             let start = format!("tool-{}-start", marker);
1487             let end = format!("tool-{}-end", marker);
1488             let mut lines = Vec::new();
1489             let mut omitted = false;
1490             for line in contents.lines() {
1491                 if line.contains(&start) {
1492                     omitted = true;
1493                 } else if line.contains(&end) {
1494                     omitted = false;
1495                 } else if !omitted {
1496                     lines.push(line);
1497                 }
1498             }
1499
1500             lines.join("\n")
1501         }
1502
1503         let xform = |p: &Path| {
1504             let mut contents = t!(fs::read_to_string(p));
1505             if rls_installer.is_none() {
1506                 contents = filter(&contents, "rls");
1507             }
1508             if clippy_installer.is_none() {
1509                 contents = filter(&contents, "clippy");
1510             }
1511             if rustfmt_installer.is_none() {
1512                 contents = filter(&contents, "rustfmt");
1513             }
1514             let ret = tmp.join(p.file_name().unwrap());
1515             t!(fs::write(&ret, &contents));
1516             ret
1517         };
1518
1519         if target.contains("apple-darwin") {
1520             let pkg = tmp.join("pkg");
1521             let _ = fs::remove_dir_all(&pkg);
1522
1523             let pkgbuild = |component: &str| {
1524                 let mut cmd = Command::new("pkgbuild");
1525                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1526                     .arg("--scripts").arg(pkg.join(component))
1527                     .arg("--nopayload")
1528                     .arg(pkg.join(component).with_extension("pkg"));
1529                 builder.run(&mut cmd);
1530             };
1531
1532             let prepare = |name: &str| {
1533                 builder.create_dir(&pkg.join(name));
1534                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1535                         &pkg.join(name));
1536                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1537                 pkgbuild(name);
1538             };
1539             prepare("rustc");
1540             prepare("cargo");
1541             prepare("rust-docs");
1542             prepare("rust-std");
1543             prepare("rust-analysis");
1544
1545             if rls_installer.is_some() {
1546                 prepare("rls");
1547             }
1548             if clippy_installer.is_some() {
1549                 prepare("clippy");
1550             }
1551
1552             // create an 'uninstall' package
1553             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1554             pkgbuild("uninstall");
1555
1556             builder.create_dir(&pkg.join("res"));
1557             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1558             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1559             let mut cmd = Command::new("productbuild");
1560             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1561                 .arg("--resources").arg(pkg.join("res"))
1562                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1563                                                     pkgname(builder, "rust"),
1564                                                     target)))
1565                 .arg("--package-path").arg(&pkg);
1566             builder.run(&mut cmd);
1567         }
1568
1569         if target.contains("windows") {
1570             let exe = tmp.join("exe");
1571             let _ = fs::remove_dir_all(&exe);
1572
1573             let prepare = |name: &str| {
1574                 builder.create_dir(&exe.join(name));
1575                 let dir = if name == "rust-std" || name == "rust-analysis" {
1576                     format!("{}-{}", name, target)
1577                 } else if name == "rls" {
1578                     "rls-preview".to_string()
1579                 } else if name == "clippy" {
1580                     "clippy-preview".to_string()
1581                 } else {
1582                     name.to_string()
1583                 };
1584                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1585                             .join(dir),
1586                         &exe.join(name));
1587                 builder.remove(&exe.join(name).join("manifest.in"));
1588             };
1589             prepare("rustc");
1590             prepare("cargo");
1591             prepare("rust-analysis");
1592             prepare("rust-docs");
1593             prepare("rust-std");
1594             if rls_installer.is_some() {
1595                 prepare("rls");
1596             }
1597             if clippy_installer.is_some() {
1598                 prepare("clippy");
1599             }
1600             if target.contains("windows-gnu") {
1601                 prepare("rust-mingw");
1602             }
1603
1604             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1605             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1606             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1607             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1608             builder.create(&exe.join("LICENSE.txt"), &license);
1609
1610             // Generate exe installer
1611             let mut cmd = Command::new("iscc");
1612             cmd.arg("rust.iss")
1613                 .current_dir(&exe);
1614             if target.contains("windows-gnu") {
1615                 cmd.arg("/dMINGW");
1616             }
1617             add_env(builder, &mut cmd, target);
1618             builder.run(&mut cmd);
1619             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1620                     &distdir(builder),
1621                     0o755);
1622
1623             // Generate msi installer
1624             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1625             let heat = wix.join("bin/heat.exe");
1626             let candle = wix.join("bin/candle.exe");
1627             let light = wix.join("bin/light.exe");
1628
1629             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1630             builder.run(Command::new(&heat)
1631                             .current_dir(&exe)
1632                             .arg("dir")
1633                             .arg("rustc")
1634                             .args(&heat_flags)
1635                             .arg("-cg").arg("RustcGroup")
1636                             .arg("-dr").arg("Rustc")
1637                             .arg("-var").arg("var.RustcDir")
1638                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1639             builder.run(Command::new(&heat)
1640                             .current_dir(&exe)
1641                             .arg("dir")
1642                             .arg("rust-docs")
1643                             .args(&heat_flags)
1644                             .arg("-cg").arg("DocsGroup")
1645                             .arg("-dr").arg("Docs")
1646                             .arg("-var").arg("var.DocsDir")
1647                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1648                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1649             builder.run(Command::new(&heat)
1650                             .current_dir(&exe)
1651                             .arg("dir")
1652                             .arg("cargo")
1653                             .args(&heat_flags)
1654                             .arg("-cg").arg("CargoGroup")
1655                             .arg("-dr").arg("Cargo")
1656                             .arg("-var").arg("var.CargoDir")
1657                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1658                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1659             builder.run(Command::new(&heat)
1660                             .current_dir(&exe)
1661                             .arg("dir")
1662                             .arg("rust-std")
1663                             .args(&heat_flags)
1664                             .arg("-cg").arg("StdGroup")
1665                             .arg("-dr").arg("Std")
1666                             .arg("-var").arg("var.StdDir")
1667                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1668             if rls_installer.is_some() {
1669                 builder.run(Command::new(&heat)
1670                                 .current_dir(&exe)
1671                                 .arg("dir")
1672                                 .arg("rls")
1673                                 .args(&heat_flags)
1674                                 .arg("-cg").arg("RlsGroup")
1675                                 .arg("-dr").arg("Rls")
1676                                 .arg("-var").arg("var.RlsDir")
1677                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1678                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1679             }
1680             if clippy_installer.is_some() {
1681                 builder.run(Command::new(&heat)
1682                                 .current_dir(&exe)
1683                                 .arg("dir")
1684                                 .arg("clippy")
1685                                 .args(&heat_flags)
1686                                 .arg("-cg").arg("ClippyGroup")
1687                                 .arg("-dr").arg("Clippy")
1688                                 .arg("-var").arg("var.ClippyDir")
1689                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1690                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1691             }
1692             builder.run(Command::new(&heat)
1693                             .current_dir(&exe)
1694                             .arg("dir")
1695                             .arg("rust-analysis")
1696                             .args(&heat_flags)
1697                             .arg("-cg").arg("AnalysisGroup")
1698                             .arg("-dr").arg("Analysis")
1699                             .arg("-var").arg("var.AnalysisDir")
1700                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1701                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1702             if target.contains("windows-gnu") {
1703                 builder.run(Command::new(&heat)
1704                                 .current_dir(&exe)
1705                                 .arg("dir")
1706                                 .arg("rust-mingw")
1707                                 .args(&heat_flags)
1708                                 .arg("-cg").arg("GccGroup")
1709                                 .arg("-dr").arg("Gcc")
1710                                 .arg("-var").arg("var.GccDir")
1711                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1712             }
1713
1714             let candle = |input: &Path| {
1715                 let output = exe.join(input.file_stem().unwrap())
1716                                 .with_extension("wixobj");
1717                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1718                 let mut cmd = Command::new(&candle);
1719                 cmd.current_dir(&exe)
1720                     .arg("-nologo")
1721                     .arg("-dRustcDir=rustc")
1722                     .arg("-dDocsDir=rust-docs")
1723                     .arg("-dCargoDir=cargo")
1724                     .arg("-dStdDir=rust-std")
1725                     .arg("-dAnalysisDir=rust-analysis")
1726                     .arg("-arch").arg(&arch)
1727                     .arg("-out").arg(&output)
1728                     .arg(&input);
1729                 add_env(builder, &mut cmd, target);
1730
1731                 if rls_installer.is_some() {
1732                     cmd.arg("-dRlsDir=rls");
1733                 }
1734                 if clippy_installer.is_some() {
1735                     cmd.arg("-dClippyDir=clippy");
1736                 }
1737                 if target.contains("windows-gnu") {
1738                     cmd.arg("-dGccDir=rust-mingw");
1739                 }
1740                 builder.run(&mut cmd);
1741             };
1742             candle(&xform(&etc.join("msi/rust.wxs")));
1743             candle(&etc.join("msi/ui.wxs"));
1744             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1745             candle("RustcGroup.wxs".as_ref());
1746             candle("DocsGroup.wxs".as_ref());
1747             candle("CargoGroup.wxs".as_ref());
1748             candle("StdGroup.wxs".as_ref());
1749             if rls_installer.is_some() {
1750                 candle("RlsGroup.wxs".as_ref());
1751             }
1752             if clippy_installer.is_some() {
1753                 candle("ClippyGroup.wxs".as_ref());
1754             }
1755             candle("AnalysisGroup.wxs".as_ref());
1756
1757             if target.contains("windows-gnu") {
1758                 candle("GccGroup.wxs".as_ref());
1759             }
1760
1761             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1762             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1763             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1764
1765             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1766             let mut cmd = Command::new(&light);
1767             cmd.arg("-nologo")
1768                 .arg("-ext").arg("WixUIExtension")
1769                 .arg("-ext").arg("WixUtilExtension")
1770                 .arg("-out").arg(exe.join(&filename))
1771                 .arg("rust.wixobj")
1772                 .arg("ui.wixobj")
1773                 .arg("rustwelcomedlg.wixobj")
1774                 .arg("RustcGroup.wixobj")
1775                 .arg("DocsGroup.wixobj")
1776                 .arg("CargoGroup.wixobj")
1777                 .arg("StdGroup.wixobj")
1778                 .arg("AnalysisGroup.wixobj")
1779                 .current_dir(&exe);
1780
1781             if rls_installer.is_some() {
1782                 cmd.arg("RlsGroup.wixobj");
1783             }
1784             if clippy_installer.is_some() {
1785                 cmd.arg("ClippyGroup.wixobj");
1786             }
1787
1788             if target.contains("windows-gnu") {
1789                 cmd.arg("GccGroup.wixobj");
1790             }
1791             // ICE57 wrongly complains about the shortcuts
1792             cmd.arg("-sice:ICE57");
1793
1794             builder.run(&mut cmd);
1795
1796             if !builder.config.dry_run {
1797                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1798             }
1799         }
1800     }
1801 }
1802
1803 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1804     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1805     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1806        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1807        .env("CFG_RELEASE", builder.rust_release())
1808        .env("CFG_VER_MAJOR", parts.next().unwrap())
1809        .env("CFG_VER_MINOR", parts.next().unwrap())
1810        .env("CFG_VER_PATCH", parts.next().unwrap())
1811        .env("CFG_VER_BUILD", "0") // just needed to build
1812        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1813        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1814        .env("CFG_BUILD", target)
1815        .env("CFG_CHANNEL", &builder.config.channel);
1816
1817     if target.contains("windows-gnu") {
1818        cmd.env("CFG_MINGW", "1")
1819           .env("CFG_ABI", "GNU");
1820     } else {
1821        cmd.env("CFG_MINGW", "0")
1822           .env("CFG_ABI", "MSVC");
1823     }
1824
1825     if target.contains("x86_64") {
1826        cmd.env("CFG_PLATFORM", "x64");
1827     } else {
1828        cmd.env("CFG_PLATFORM", "x86");
1829     }
1830 }
1831
1832 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1833 pub struct HashSign;
1834
1835 impl Step for HashSign {
1836     type Output = ();
1837     const ONLY_HOSTS: bool = true;
1838
1839     fn should_run(run: ShouldRun) -> ShouldRun {
1840         run.path("hash-and-sign")
1841     }
1842
1843     fn make_run(run: RunConfig) {
1844         run.builder.ensure(HashSign);
1845     }
1846
1847     fn run(self, builder: &Builder) {
1848         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1849         if builder.config.dry_run {
1850             return;
1851         }
1852         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1853             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1854         });
1855         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1856             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1857         });
1858         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1859             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1860         });
1861         let pass = t!(fs::read_to_string(&file));
1862
1863         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1864
1865         cmd.arg(sign);
1866         cmd.arg(distdir(builder));
1867         cmd.arg(today.trim());
1868         cmd.arg(builder.rust_package_vers());
1869         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
1870         cmd.arg(builder.package_vers(&builder.release_num("rls")));
1871         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
1872         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
1873         cmd.arg(builder.llvm_tools_package_vers());
1874         cmd.arg(builder.lldb_package_vers());
1875         cmd.arg(addr);
1876
1877         builder.create_dir(&distdir(builder));
1878
1879         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1880         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1881         let status = t!(child.wait());
1882         assert!(status.success());
1883     }
1884 }
1885
1886 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
1887 // LLVM tools are linked dynamically.
1888 // Note: This function does no yet support Windows but we also don't support
1889 //       linking LLVM tools dynamically on Windows yet.
1890 fn maybe_install_llvm_dylib(builder: &Builder,
1891                             target: Interned<String>,
1892                             image: &Path) {
1893     let src_libdir = builder
1894         .llvm_out(target)
1895         .join("lib");
1896     let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib");
1897     t!(fs::create_dir_all(&dst_libdir));
1898
1899     if target.contains("apple-darwin") {
1900         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
1901         if llvm_dylib_path.exists() {
1902             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1903         }
1904         return
1905     }
1906
1907     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
1908     // Since tools link to the latter rather than the former, we have to
1909     // follow the symlink to find out what to distribute.
1910     let llvm_dylib_path = src_libdir.join("libLLVM.so");
1911     if llvm_dylib_path.exists() {
1912         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
1913             panic!("dist: Error calling canonicalize path `{}`: {}",
1914                    llvm_dylib_path.display(), e);
1915         });
1916
1917
1918         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
1919     }
1920 }
1921
1922 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
1923 pub struct LlvmTools {
1924     pub stage: u32,
1925     pub target: Interned<String>,
1926 }
1927
1928 impl Step for LlvmTools {
1929     type Output = Option<PathBuf>;
1930     const ONLY_HOSTS: bool = true;
1931
1932     fn should_run(run: ShouldRun) -> ShouldRun {
1933         run.path("llvm-tools")
1934     }
1935
1936     fn make_run(run: RunConfig) {
1937         run.builder.ensure(LlvmTools {
1938             stage: run.builder.top_stage,
1939             target: run.target,
1940         });
1941     }
1942
1943     fn run(self, builder: &Builder) -> Option<PathBuf> {
1944         let stage = self.stage;
1945         let target = self.target;
1946         assert!(builder.config.extended);
1947
1948         /* run only if llvm-config isn't used */
1949         if let Some(config) = builder.config.target_config.get(&target) {
1950             if let Some(ref _s) = config.llvm_config {
1951                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
1952                     stage, target));
1953                 return None;
1954             }
1955         }
1956
1957         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
1958         let src = builder.src.join("src/llvm");
1959         let name = pkgname(builder, "llvm-tools");
1960
1961         let tmp = tmpdir(builder);
1962         let image = tmp.join("llvm-tools-image");
1963         drop(fs::remove_dir_all(&image));
1964
1965         // Prepare the image directory
1966         let src_bindir = builder
1967             .llvm_out(target)
1968             .join("bin");
1969         let dst_bindir = image.join("lib/rustlib")
1970             .join(&*target)
1971             .join("bin");
1972         t!(fs::create_dir_all(&dst_bindir));
1973         for tool in LLVM_TOOLS {
1974             let exe = src_bindir.join(exe(tool, &target));
1975             builder.install(&exe, &dst_bindir, 0o755);
1976         }
1977
1978         // Prepare the overlay
1979         let overlay = tmp.join("llvm-tools-overlay");
1980         drop(fs::remove_dir_all(&overlay));
1981         builder.create_dir(&overlay);
1982         builder.install(&src.join("README.txt"), &overlay, 0o644);
1983         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
1984         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
1985
1986         // Generate the installer tarball
1987         let mut cmd = rust_installer(builder);
1988         cmd.arg("generate")
1989             .arg("--product-name=Rust")
1990             .arg("--rel-manifest-dir=rustlib")
1991             .arg("--success-message=llvm-tools-installed.")
1992             .arg("--image-dir").arg(&image)
1993             .arg("--work-dir").arg(&tmpdir(builder))
1994             .arg("--output-dir").arg(&distdir(builder))
1995             .arg("--non-installed-overlay").arg(&overlay)
1996             .arg(format!("--package-name={}-{}", name, target))
1997             .arg("--legacy-manifest-dirs=rustlib,cargo")
1998             .arg("--component-name=llvm-tools-preview");
1999
2000
2001         builder.run(&mut cmd);
2002         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2003     }
2004 }
2005
2006 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2007 pub struct Lldb {
2008     pub target: Interned<String>,
2009 }
2010
2011 impl Step for Lldb {
2012     type Output = Option<PathBuf>;
2013     const ONLY_HOSTS: bool = true;
2014     const DEFAULT: bool = true;
2015
2016     fn should_run(run: ShouldRun) -> ShouldRun {
2017         run.path("src/tools/lldb")
2018     }
2019
2020     fn make_run(run: RunConfig) {
2021         run.builder.ensure(Lldb {
2022             target: run.target,
2023         });
2024     }
2025
2026     fn run(self, builder: &Builder) -> Option<PathBuf> {
2027         let target = self.target;
2028
2029         if builder.config.dry_run {
2030             return None;
2031         }
2032
2033         let bindir = builder
2034             .llvm_out(target)
2035             .join("bin");
2036         let lldb_exe = bindir.join(exe("lldb", &target));
2037         if !lldb_exe.exists() {
2038             return None;
2039         }
2040
2041         builder.info(&format!("Dist Lldb ({})", target));
2042         let src = builder.src.join("src/tools/lldb");
2043         let name = pkgname(builder, "lldb");
2044
2045         let tmp = tmpdir(builder);
2046         let image = tmp.join("lldb-image");
2047         drop(fs::remove_dir_all(&image));
2048
2049         // Prepare the image directory
2050         let root = image.join("lib/rustlib").join(&*target);
2051         let dst = root.join("bin");
2052         t!(fs::create_dir_all(&dst));
2053         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2054             let exe = bindir.join(exe(program, &target));
2055             builder.install(&exe, &dst, 0o755);
2056         }
2057
2058         // The libraries.
2059         let libdir = builder.llvm_out(target).join("lib");
2060         let dst = root.join("lib");
2061         t!(fs::create_dir_all(&dst));
2062         for entry in t!(fs::read_dir(&libdir)) {
2063             let entry = entry.unwrap();
2064             if let Ok(name) = entry.file_name().into_string() {
2065                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2066                     if t!(entry.file_type()).is_symlink() {
2067                         builder.copy_to_folder(&entry.path(), &dst);
2068                     } else {
2069                        builder.install(&entry.path(), &dst, 0o755);
2070                     }
2071                 }
2072             }
2073         }
2074
2075         // The lldb scripts might be installed in lib/python$version
2076         // or in lib64/python$version.  If lib64 exists, use it;
2077         // otherwise lib.
2078         let libdir = builder.llvm_out(target).join("lib64");
2079         let (libdir, libdir_name) = if libdir.exists() {
2080             (libdir, "lib64")
2081         } else {
2082             (builder.llvm_out(target).join("lib"), "lib")
2083         };
2084         for entry in t!(fs::read_dir(&libdir)) {
2085             let entry = t!(entry);
2086             if let Ok(name) = entry.file_name().into_string() {
2087                 if name.starts_with("python") {
2088                     let dst = root.join(libdir_name)
2089                         .join(entry.file_name());
2090                     t!(fs::create_dir_all(&dst));
2091                     builder.cp_r(&entry.path(), &dst);
2092                     break;
2093                 }
2094             }
2095         }
2096
2097         // Prepare the overlay
2098         let overlay = tmp.join("lldb-overlay");
2099         drop(fs::remove_dir_all(&overlay));
2100         builder.create_dir(&overlay);
2101         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2102         builder.create(&overlay.join("version"), &builder.lldb_vers());
2103
2104         // Generate the installer tarball
2105         let mut cmd = rust_installer(builder);
2106         cmd.arg("generate")
2107             .arg("--product-name=Rust")
2108             .arg("--rel-manifest-dir=rustlib")
2109             .arg("--success-message=lldb-installed.")
2110             .arg("--image-dir").arg(&image)
2111             .arg("--work-dir").arg(&tmpdir(builder))
2112             .arg("--output-dir").arg(&distdir(builder))
2113             .arg("--non-installed-overlay").arg(&overlay)
2114             .arg(format!("--package-name={}-{}", name, target))
2115             .arg("--legacy-manifest-dirs=rustlib,cargo")
2116             .arg("--component-name=lldb-preview");
2117
2118
2119         builder.run(&mut cmd);
2120         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2121     }
2122 }