src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36
  37 pub fn pkgname(build: &Build, component: &str) -> String {
  38     if component == "cargo" {
  39         format!("{}-{}", component, build.cargo_package_vers())
  40     } else if component == "rls" {
  41         format!("{}-{}", component, build.rls_package_vers())
  42     } else if component == "rustfmt" {
  43         format!("{}-{}", component, build.rustfmt_package_vers())
  44     } else {
  45         assert!(component.starts_with("rust"));
  46         format!("{}-{}", component, build.rust_package_vers())
  47     }
  48 }
  49
  50 fn distdir(build: &Build) -> PathBuf {
  51     build.out.join("dist")
  52 }
  53
  54 pub fn tmpdir(build: &Build) -> PathBuf {
  55     build.out.join("tmp/dist")
  56 }
  57
  58 fn rust_installer(builder: &Builder) -> Command {
  59     builder.tool_cmd(Tool::RustInstaller)
  60 }
  61
  62 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  63 pub struct Docs {
  64     pub stage: u32,
  65     pub host: Interned<String>,
  66 }
  67
  68 impl Step for Docs {
  69     type Output = PathBuf;
  70     const DEFAULT: bool = true;
  71     const ONLY_BUILD_TARGETS: bool = true;
  72
  73     fn should_run(run: ShouldRun) -> ShouldRun {
  74         run.path("src/doc")
  75     }
  76
  77     fn make_run(run: RunConfig) {
  78         run.builder.ensure(Docs {
  79             stage: run.builder.top_stage,
  80             host: run.target,
  81         });
  82     }
  83
  84     /// Builds the `rust-docs` installer component.
  85     fn run(self, builder: &Builder) -> PathBuf {
  86         let build = builder.build;
  87         let host = self.host;
  88
  89         let name = pkgname(build, "rust-docs");
  90
  91         println!("Dist docs ({})", host);
  92         if !build.config.docs {
  93             println!("\tskipping - docs disabled");
  94             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  95         }
  96
  97         builder.default_doc(None);
  98
  99         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 100         let _ = fs::remove_dir_all(&image);
 101
 102         let dst = image.join("share/doc/rust/html");
 103         t!(fs::create_dir_all(&dst));
 104         let src = build.out.join(host).join("doc");
 105         cp_r(&src, &dst);
 106
 107         let mut cmd = rust_installer(builder);
 108         cmd.arg("generate")
 109            .arg("--product-name=Rust-Documentation")
 110            .arg("--rel-manifest-dir=rustlib")
 111            .arg("--success-message=Rust-documentation-is-installed.")
 112            .arg("--image-dir").arg(&image)
 113            .arg("--work-dir").arg(&tmpdir(build))
 114            .arg("--output-dir").arg(&distdir(build))
 115            .arg(format!("--package-name={}-{}", name, host))
 116            .arg("--component-name=rust-docs")
 117            .arg("--legacy-manifest-dirs=rustlib,cargo")
 118            .arg("--bulk-dirs=share/doc/rust/html");
 119         build.run(&mut cmd);
 120         t!(fs::remove_dir_all(&image));
 121
 122         // As part of this step, *also* copy the docs directory to a directory which
 123         // buildbot typically uploads.
 124         if host == build.build {
 125             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 126             t!(fs::create_dir_all(&dst));
 127             cp_r(&src, &dst);
 128         }
 129
 130         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 131     }
 132 }
 133
 134 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 135     let mut found = Vec::with_capacity(files.len());
 136
 137     for file in files {
 138         let file_path =
 139             path.iter()
 140                 .map(|dir| dir.join(file))
 141                 .find(|p| p.exists());
 142
 143         if let Some(file_path) = file_path {
 144             found.push(file_path);
 145         } else {
 146             panic!("Could not find '{}' in {:?}", file, path);
 147         }
 148     }
 149
 150     found
 151 }
 152
 153 fn make_win_dist(
 154     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 155 ) {
 156     //Ask gcc where it keeps its stuff
 157     let mut cmd = Command::new(build.cc(target_triple));
 158     cmd.arg("-print-search-dirs");
 159     let gcc_out = output(&mut cmd);
 160
 161     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 162     let mut lib_path = Vec::new();
 163
 164     for line in gcc_out.lines() {
 165         let idx = line.find(':').unwrap();
 166         let key = &line[..idx];
 167         let trim_chars: &[_] = &[' ', '='];
 168         let value =
 169             line[(idx + 1)..]
 170                 .trim_left_matches(trim_chars)
 171                 .split(';')
 172                 .map(PathBuf::from);
 173
 174         if key == "programs" {
 175             bin_path.extend(value);
 176         } else if key == "libraries" {
 177             lib_path.extend(value);
 178         }
 179     }
 180
 181     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 182     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 183     if target_triple.starts_with("i686-") {
 184         rustc_dlls.push("libgcc_s_dw2-1.dll");
 185     } else {
 186         rustc_dlls.push("libgcc_s_seh-1.dll");
 187     }
 188
 189     let target_libs = [ //MinGW libs
 190         "libgcc.a",
 191         "libgcc_eh.a",
 192         "libgcc_s.a",
 193         "libm.a",
 194         "libmingw32.a",
 195         "libmingwex.a",
 196         "libstdc++.a",
 197         "libiconv.a",
 198         "libmoldname.a",
 199         "libpthread.a",
 200         //Windows import libs
 201         "libadvapi32.a",
 202         "libbcrypt.a",
 203         "libcomctl32.a",
 204         "libcomdlg32.a",
 205         "libcrypt32.a",
 206         "libgdi32.a",
 207         "libimagehlp.a",
 208         "libiphlpapi.a",
 209         "libkernel32.a",
 210         "libmsvcrt.a",
 211         "libodbc32.a",
 212         "libole32.a",
 213         "liboleaut32.a",
 214         "libopengl32.a",
 215         "libpsapi.a",
 216         "librpcrt4.a",
 217         "libsetupapi.a",
 218         "libshell32.a",
 219         "libuser32.a",
 220         "libuserenv.a",
 221         "libuuid.a",
 222         "libwinhttp.a",
 223         "libwinmm.a",
 224         "libwinspool.a",
 225         "libws2_32.a",
 226         "libwsock32.a",
 227     ];
 228
 229     //Find mingw artifacts we want to bundle
 230     let target_tools = find_files(&target_tools, &bin_path);
 231     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 232     let target_libs = find_files(&target_libs, &lib_path);
 233
 234     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 235         let file_name = src.file_name().unwrap();
 236         let dest = dest_folder.join(file_name);
 237         copy(src, &dest);
 238     }
 239
 240     //Copy runtime dlls next to rustc.exe
 241     let dist_bin_dir = rust_root.join("bin/");
 242     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 243     for src in rustc_dlls {
 244         copy_to_folder(&src, &dist_bin_dir);
 245     }
 246
 247     //Copy platform tools to platform-specific bin directory
 248     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 249     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 250     for src in target_tools {
 251         copy_to_folder(&src, &target_bin_dir);
 252     }
 253
 254     //Copy platform libs to platform-specific lib directory
 255     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 256     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 257     for src in target_libs {
 258         copy_to_folder(&src, &target_lib_dir);
 259     }
 260 }
 261
 262 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 263 pub struct Mingw {
 264     host: Interned<String>,
 265 }
 266
 267 impl Step for Mingw {
 268     type Output = Option<PathBuf>;
 269     const DEFAULT: bool = true;
 270     const ONLY_BUILD_TARGETS: bool = true;
 271
 272     fn should_run(run: ShouldRun) -> ShouldRun {
 273         run.never()
 274     }
 275
 276     fn make_run(run: RunConfig) {
 277         run.builder.ensure(Mingw { host: run.target });
 278     }
 279
 280     /// Build the `rust-mingw` installer component.
 281     ///
 282     /// This contains all the bits and pieces to run the MinGW Windows targets
 283     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 284     fn run(self, builder: &Builder) -> Option<PathBuf> {
 285         let build = builder.build;
 286         let host = self.host;
 287
 288         if !host.contains("pc-windows-gnu") {
 289             return None;
 290         }
 291
 292         println!("Dist mingw ({})", host);
 293         let name = pkgname(build, "rust-mingw");
 294         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 295         let _ = fs::remove_dir_all(&image);
 296         t!(fs::create_dir_all(&image));
 297
 298         // The first argument is a "temporary directory" which is just
 299         // thrown away (this contains the runtime DLLs included in the rustc package
 300         // above) and the second argument is where to place all the MinGW components
 301         // (which is what we want).
 302         make_win_dist(&tmpdir(build), &image, host, &build);
 303
 304         let mut cmd = rust_installer(builder);
 305         cmd.arg("generate")
 306            .arg("--product-name=Rust-MinGW")
 307            .arg("--rel-manifest-dir=rustlib")
 308            .arg("--success-message=Rust-MinGW-is-installed.")
 309            .arg("--image-dir").arg(&image)
 310            .arg("--work-dir").arg(&tmpdir(build))
 311            .arg("--output-dir").arg(&distdir(build))
 312            .arg(format!("--package-name={}-{}", name, host))
 313            .arg("--component-name=rust-mingw")
 314            .arg("--legacy-manifest-dirs=rustlib,cargo");
 315         build.run(&mut cmd);
 316         t!(fs::remove_dir_all(&image));
 317         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 318     }
 319 }
 320
 321 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 322 pub struct Rustc {
 323     pub compiler: Compiler,
 324 }
 325
 326 impl Step for Rustc {
 327     type Output = PathBuf;
 328     const DEFAULT: bool = true;
 329     const ONLY_HOSTS: bool = true;
 330     const ONLY_BUILD_TARGETS: bool = true;
 331
 332     fn should_run(run: ShouldRun) -> ShouldRun {
 333         run.path("src/librustc")
 334     }
 335
 336     fn make_run(run: RunConfig) {
 337         run.builder.ensure(Rustc {
 338             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 339         });
 340     }
 341
 342     /// Creates the `rustc` installer component.
 343     fn run(self, builder: &Builder) -> PathBuf {
 344         let build = builder.build;
 345         let compiler = self.compiler;
 346         let host = self.compiler.host;
 347
 348         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 349         let name = pkgname(build, "rustc");
 350         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 351         let _ = fs::remove_dir_all(&image);
 352         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 353         let _ = fs::remove_dir_all(&overlay);
 354
 355         // Prepare the rustc "image", what will actually end up getting installed
 356         prepare_image(builder, compiler, &image);
 357
 358         // Prepare the overlay which is part of the tarball but won't actually be
 359         // installed
 360         let cp = |file: &str| {
 361             install(&build.src.join(file), &overlay, 0o644);
 362         };
 363         cp("COPYRIGHT");
 364         cp("LICENSE-APACHE");
 365         cp("LICENSE-MIT");
 366         cp("README.md");
 367         // tiny morsel of metadata is used by rust-packaging
 368         let version = build.rust_version();
 369         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 370         if let Some(sha) = build.rust_sha() {
 371             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 372         }
 373
 374         // On MinGW we've got a few runtime DLL dependencies that we need to
 375         // include. The first argument to this script is where to put these DLLs
 376         // (the image we're creating), and the second argument is a junk directory
 377         // to ignore all other MinGW stuff the script creates.
 378         //
 379         // On 32-bit MinGW we're always including a DLL which needs some extra
 380         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 381         // anything requiring us to distribute a license, but it's likely the
 382         // install will *also* include the rust-mingw package, which also needs
 383         // licenses, so to be safe we just include it here in all MinGW packages.
 384         if host.contains("pc-windows-gnu") {
 385             make_win_dist(&image, &tmpdir(build), host, build);
 386
 387             let dst = image.join("share/doc");
 388             t!(fs::create_dir_all(&dst));
 389             cp_r(&build.src.join("src/etc/third-party"), &dst);
 390         }
 391
 392         // Finally, wrap everything up in a nice tarball!
 393         let mut cmd = rust_installer(builder);
 394         cmd.arg("generate")
 395            .arg("--product-name=Rust")
 396            .arg("--rel-manifest-dir=rustlib")
 397            .arg("--success-message=Rust-is-ready-to-roll.")
 398            .arg("--image-dir").arg(&image)
 399            .arg("--work-dir").arg(&tmpdir(build))
 400            .arg("--output-dir").arg(&distdir(build))
 401            .arg("--non-installed-overlay").arg(&overlay)
 402            .arg(format!("--package-name={}-{}", name, host))
 403            .arg("--component-name=rustc")
 404            .arg("--legacy-manifest-dirs=rustlib,cargo");
 405         build.run(&mut cmd);
 406         t!(fs::remove_dir_all(&image));
 407         t!(fs::remove_dir_all(&overlay));
 408
 409         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 410
 411         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 412             let host = compiler.host;
 413             let build = builder.build;
 414             let src = builder.sysroot(compiler);
 415             let libdir = libdir(&host);
 416
 417             // Copy rustc/rustdoc binaries
 418             t!(fs::create_dir_all(image.join("bin")));
 419             cp_r(&src.join("bin"), &image.join("bin"));
 420
 421             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 422
 423             // Copy runtime DLLs needed by the compiler
 424             if libdir != "bin" {
 425                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 426                     let name = entry.file_name();
 427                     if let Some(s) = name.to_str() {
 428                         if is_dylib(s) {
 429                             install(&entry.path(), &image.join(libdir), 0o644);
 430                         }
 431                     }
 432                 }
 433             }
 434
 435             // Man pages
 436             t!(fs::create_dir_all(image.join("share/man/man1")));
 437             cp_r(&build.src.join("src/doc/man"), &image.join("share/man/man1"));
 438
 439             // Debugger scripts
 440             builder.ensure(DebuggerScripts {
 441                 sysroot: INTERNER.intern_path(image.to_owned()),
 442                 host,
 443             });
 444
 445             // Misc license info
 446             let cp = |file: &str| {
 447                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 448             };
 449             cp("COPYRIGHT");
 450             cp("LICENSE-APACHE");
 451             cp("LICENSE-MIT");
 452             cp("README.md");
 453         }
 454     }
 455 }
 456
 457 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 458 pub struct DebuggerScripts {
 459     pub sysroot: Interned<PathBuf>,
 460     pub host: Interned<String>,
 461 }
 462
 463 impl Step for DebuggerScripts {
 464     type Output = ();
 465
 466     fn should_run(run: ShouldRun) -> ShouldRun {
 467         run.path("src/lldb_batchmode.py")
 468     }
 469
 470     fn make_run(run: RunConfig) {
 471         run.builder.ensure(DebuggerScripts {
 472             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 473             host: run.target,
 474         });
 475     }
 476
 477     /// Copies debugger scripts for `target` into the `sysroot` specified.
 478     fn run(self, builder: &Builder) {
 479         let build = builder.build;
 480         let host = self.host;
 481         let sysroot = self.sysroot;
 482         let dst = sysroot.join("lib/rustlib/etc");
 483         t!(fs::create_dir_all(&dst));
 484         let cp_debugger_script = |file: &str| {
 485             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 486         };
 487         if host.contains("windows-msvc") {
 488             // windbg debugger scripts
 489             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 490                 0o755);
 491
 492             cp_debugger_script("natvis/liballoc.natvis");
 493             cp_debugger_script("natvis/libcore.natvis");
 494         } else {
 495             cp_debugger_script("debugger_pretty_printers_common.py");
 496
 497             // gdb debugger scripts
 498             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 499                     0o755);
 500
 501             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 502             cp_debugger_script("gdb_rust_pretty_printing.py");
 503
 504             // lldb debugger scripts
 505             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 506                     0o755);
 507
 508             cp_debugger_script("lldb_rust_formatters.py");
 509         }
 510     }
 511 }
 512
 513 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 514 pub struct Std {
 515     pub compiler: Compiler,
 516     pub target: Interned<String>,
 517 }
 518
 519 impl Step for Std {
 520     type Output = PathBuf;
 521     const DEFAULT: bool = true;
 522     const ONLY_BUILD_TARGETS: bool = true;
 523
 524     fn should_run(run: ShouldRun) -> ShouldRun {
 525         run.path("src/libstd")
 526     }
 527
 528     fn make_run(run: RunConfig) {
 529         run.builder.ensure(Std {
 530             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 531             target: run.target,
 532         });
 533     }
 534
 535     fn run(self, builder: &Builder) -> PathBuf {
 536         let build = builder.build;
 537         let compiler = self.compiler;
 538         let target = self.target;
 539
 540         let name = pkgname(build, "rust-std");
 541         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 542
 543         // The only true set of target libraries came from the build triple, so
 544         // let's reduce redundant work by only producing archives from that host.
 545         if compiler.host != build.build {
 546             println!("\tskipping, not a build host");
 547             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 548         }
 549
 550         // We want to package up as many target libraries as possible
 551         // for the `rust-std` package, so if this is a host target we
 552         // depend on librustc and otherwise we just depend on libtest.
 553         if build.hosts.iter().any(|t| t == target) {
 554             builder.ensure(compile::Rustc { compiler, target });
 555         } else {
 556             builder.ensure(compile::Test { compiler, target });
 557         }
 558
 559         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 560         let _ = fs::remove_dir_all(&image);
 561
 562         let dst = image.join("lib/rustlib").join(target);
 563         t!(fs::create_dir_all(&dst));
 564         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 565         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 566         cp_r(&src, &dst);
 567
 568         let mut cmd = rust_installer(builder);
 569         cmd.arg("generate")
 570            .arg("--product-name=Rust")
 571            .arg("--rel-manifest-dir=rustlib")
 572            .arg("--success-message=std-is-standing-at-the-ready.")
 573            .arg("--image-dir").arg(&image)
 574            .arg("--work-dir").arg(&tmpdir(build))
 575            .arg("--output-dir").arg(&distdir(build))
 576            .arg(format!("--package-name={}-{}", name, target))
 577            .arg(format!("--component-name=rust-std-{}", target))
 578            .arg("--legacy-manifest-dirs=rustlib,cargo");
 579         build.run(&mut cmd);
 580         t!(fs::remove_dir_all(&image));
 581         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 582     }
 583 }
 584
 585 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 586 pub struct Analysis {
 587     pub compiler: Compiler,
 588     pub target: Interned<String>,
 589 }
 590
 591 impl Step for Analysis {
 592     type Output = PathBuf;
 593     const DEFAULT: bool = true;
 594     const ONLY_BUILD_TARGETS: bool = true;
 595
 596     fn should_run(run: ShouldRun) -> ShouldRun {
 597         let builder = run.builder;
 598         run.path("analysis").default_condition(builder.build.config.extended)
 599     }
 600
 601     fn make_run(run: RunConfig) {
 602         run.builder.ensure(Analysis {
 603             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 604             target: run.target,
 605         });
 606     }
 607
 608     /// Creates a tarball of save-analysis metadata, if available.
 609     fn run(self, builder: &Builder) -> PathBuf {
 610         let build = builder.build;
 611         let compiler = self.compiler;
 612         let target = self.target;
 613         assert!(build.config.extended);
 614         println!("Dist analysis");
 615         let name = pkgname(build, "rust-analysis");
 616
 617         if &compiler.host != build.build {
 618             println!("\tskipping, not a build host");
 619             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 620         }
 621
 622         builder.ensure(Std { compiler, target });
 623
 624         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 625         // stage2 artifacts is simply copied from stage1 in that case.
 626         let compiler = if build.force_use_stage1(compiler, target) {
 627             builder.compiler(1, compiler.host)
 628         } else {
 629             compiler.clone()
 630         };
 631
 632         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 633
 634         let src = build.stage_out(compiler, Mode::Libstd)
 635             .join(target).join(build.cargo_dir()).join("deps");
 636
 637         let image_src = src.join("save-analysis");
 638         let dst = image.join("lib/rustlib").join(target).join("analysis");
 639         t!(fs::create_dir_all(&dst));
 640         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 641         cp_r(&image_src, &dst);
 642
 643         let mut cmd = rust_installer(builder);
 644         cmd.arg("generate")
 645            .arg("--product-name=Rust")
 646            .arg("--rel-manifest-dir=rustlib")
 647            .arg("--success-message=save-analysis-saved.")
 648            .arg("--image-dir").arg(&image)
 649            .arg("--work-dir").arg(&tmpdir(build))
 650            .arg("--output-dir").arg(&distdir(build))
 651            .arg(format!("--package-name={}-{}", name, target))
 652            .arg(format!("--component-name=rust-analysis-{}", target))
 653            .arg("--legacy-manifest-dirs=rustlib,cargo");
 654         build.run(&mut cmd);
 655         t!(fs::remove_dir_all(&image));
 656         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 657     }
 658 }
 659
 660 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 661     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 662         let spath = match path.to_str() {
 663             Some(path) => path,
 664             None => return false,
 665         };
 666         if spath.ends_with("~") || spath.ends_with(".pyc") {
 667             return false
 668         }
 669         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 670             (spath.ends_with(".ll") ||
 671              spath.ends_with(".td") ||
 672              spath.ends_with(".s")) {
 673             return false
 674         }
 675
 676         let full_path = Path::new(dir).join(path);
 677         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 678             return false;
 679         }
 680
 681         let excludes = [
 682             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 683             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 684             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 685             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 686         ];
 687         !path.iter()
 688              .map(|s| s.to_str().unwrap())
 689              .any(|s| excludes.contains(&s))
 690     }
 691
 692     // Copy the directories using our filter
 693     for item in src_dirs {
 694         let dst = &dst_dir.join(item);
 695         t!(fs::create_dir_all(dst));
 696         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 697     }
 698 }
 699
 700 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 701 pub struct Src;
 702
 703 impl Step for Src {
 704     /// The output path of the src installer tarball
 705     type Output = PathBuf;
 706     const DEFAULT: bool = true;
 707     const ONLY_HOSTS: bool = true;
 708     const ONLY_BUILD_TARGETS: bool = true;
 709     const ONLY_BUILD: bool = true;
 710
 711     fn should_run(run: ShouldRun) -> ShouldRun {
 712         run.path("src")
 713     }
 714
 715     fn make_run(run: RunConfig) {
 716         run.builder.ensure(Src);
 717     }
 718
 719     /// Creates the `rust-src` installer component
 720     fn run(self, builder: &Builder) -> PathBuf {
 721         let build = builder.build;
 722         println!("Dist src");
 723
 724         let name = pkgname(build, "rust-src");
 725         let image = tmpdir(build).join(format!("{}-image", name));
 726         let _ = fs::remove_dir_all(&image);
 727
 728         let dst = image.join("lib/rustlib/src");
 729         let dst_src = dst.join("rust");
 730         t!(fs::create_dir_all(&dst_src));
 731
 732         let src_files = [
 733             "src/Cargo.lock",
 734         ];
 735         // This is the reduced set of paths which will become the rust-src component
 736         // (essentially libstd and all of its path dependencies)
 737         let std_src_dirs = [
 738             "src/build_helper",
 739             "src/liballoc",
 740             "src/liballoc_jemalloc",
 741             "src/liballoc_system",
 742             "src/libbacktrace",
 743             "src/libcompiler_builtins",
 744             "src/libcore",
 745             "src/liblibc",
 746             "src/libpanic_abort",
 747             "src/libpanic_unwind",
 748             "src/librustc_asan",
 749             "src/librustc_lsan",
 750             "src/librustc_msan",
 751             "src/librustc_tsan",
 752             "src/libstd",
 753             "src/libstd_unicode",
 754             "src/libunwind",
 755             "src/rustc/compiler_builtins_shim",
 756             "src/rustc/libc_shim",
 757             "src/libtest",
 758             "src/libterm",
 759             "src/jemalloc",
 760             "src/libprofiler_builtins",
 761         ];
 762         let std_src_dirs_exclude = [
 763             "src/libcompiler_builtins/compiler-rt/test",
 764             "src/jemalloc/test/unit",
 765         ];
 766
 767         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 768         for file in src_files.iter() {
 769             copy(&build.src.join(file), &dst_src.join(file));
 770         }
 771
 772         // Create source tarball in rust-installer format
 773         let mut cmd = rust_installer(builder);
 774         cmd.arg("generate")
 775            .arg("--product-name=Rust")
 776            .arg("--rel-manifest-dir=rustlib")
 777            .arg("--success-message=Awesome-Source.")
 778            .arg("--image-dir").arg(&image)
 779            .arg("--work-dir").arg(&tmpdir(build))
 780            .arg("--output-dir").arg(&distdir(build))
 781            .arg(format!("--package-name={}", name))
 782            .arg("--component-name=rust-src")
 783            .arg("--legacy-manifest-dirs=rustlib,cargo");
 784         build.run(&mut cmd);
 785
 786         t!(fs::remove_dir_all(&image));
 787         distdir(build).join(&format!("{}.tar.gz", name))
 788     }
 789 }
 790
 791 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 792
 793 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 794 pub struct PlainSourceTarball;
 795
 796 impl Step for PlainSourceTarball {
 797     /// Produces the location of the tarball generated
 798     type Output = PathBuf;
 799     const DEFAULT: bool = true;
 800     const ONLY_HOSTS: bool = true;
 801     const ONLY_BUILD_TARGETS: bool = true;
 802     const ONLY_BUILD: bool = true;
 803
 804     fn should_run(run: ShouldRun) -> ShouldRun {
 805         let builder = run.builder;
 806         run.path("src").default_condition(builder.config.rust_dist_src)
 807     }
 808
 809     fn make_run(run: RunConfig) {
 810         run.builder.ensure(PlainSourceTarball);
 811     }
 812
 813     /// Creates the plain source tarball
 814     fn run(self, builder: &Builder) -> PathBuf {
 815         let build = builder.build;
 816         println!("Create plain source tarball");
 817
 818         // Make sure that the root folder of tarball has the correct name
 819         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 820         let plain_dst_src = tmpdir(build).join(&plain_name);
 821         let _ = fs::remove_dir_all(&plain_dst_src);
 822         t!(fs::create_dir_all(&plain_dst_src));
 823
 824         // This is the set of root paths which will become part of the source package
 825         let src_files = [
 826             "COPYRIGHT",
 827             "LICENSE-APACHE",
 828             "LICENSE-MIT",
 829             "CONTRIBUTING.md",
 830             "README.md",
 831             "RELEASES.md",
 832             "configure",
 833             "x.py",
 834             "config.toml.example",
 835         ];
 836         let src_dirs = [
 837             "src",
 838         ];
 839
 840         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 841
 842         // Copy the files normally
 843         for item in &src_files {
 844             copy(&build.src.join(item), &plain_dst_src.join(item));
 845         }
 846
 847         // Create the version file
 848         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 849         if let Some(sha) = build.rust_sha() {
 850             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 851         }
 852
 853         // If we're building from git sources, we need to vendor a complete distribution.
 854         if build.rust_info.is_git() {
 855             // Get cargo-vendor installed, if it isn't already.
 856             let mut has_cargo_vendor = false;
 857             let mut cmd = Command::new(&build.initial_cargo);
 858             for line in output(cmd.arg("install").arg("--list")).lines() {
 859                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 860             }
 861             if !has_cargo_vendor {
 862                 let mut cmd = Command::new(&build.initial_cargo);
 863                 cmd.arg("install")
 864                    .arg("--force")
 865                    .arg("--debug")
 866                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 867                    .arg("cargo-vendor")
 868                    .env("RUSTC", &build.initial_rustc);
 869                 build.run(&mut cmd);
 870             }
 871
 872             // Vendor all Cargo dependencies
 873             let mut cmd = Command::new(&build.initial_cargo);
 874             cmd.arg("vendor")
 875                .current_dir(&plain_dst_src.join("src"));
 876             build.run(&mut cmd);
 877         }
 878
 879         // Create plain source tarball
 880         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 881         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 882         tarball.set_extension(""); // strip .gz
 883         tarball.set_extension(""); // strip .tar
 884         if let Some(dir) = tarball.parent() {
 885             t!(fs::create_dir_all(dir));
 886         }
 887         println!("running installer");
 888         let mut cmd = rust_installer(builder);
 889         cmd.arg("tarball")
 890            .arg("--input").arg(&plain_name)
 891            .arg("--output").arg(&tarball)
 892            .arg("--work-dir=.")
 893            .current_dir(tmpdir(build));
 894         build.run(&mut cmd);
 895         distdir(build).join(&format!("{}.tar.gz", plain_name))
 896     }
 897 }
 898
 899 fn install(src: &Path, dstdir: &Path, perms: u32) {
 900     let dst = dstdir.join(src.file_name().unwrap());
 901     t!(fs::create_dir_all(dstdir));
 902     drop(fs::remove_file(&dst));
 903     {
 904         let mut s = t!(fs::File::open(&src));
 905         let mut d = t!(fs::File::create(&dst));
 906         io::copy(&mut s, &mut d).expect("failed to copy");
 907     }
 908     chmod(&dst, perms);
 909 }
 910
 911 #[cfg(unix)]
 912 fn chmod(path: &Path, perms: u32) {
 913     use std::os::unix::fs::*;
 914     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 915 }
 916 #[cfg(windows)]
 917 fn chmod(_path: &Path, _perms: u32) {}
 918
 919 // We have to run a few shell scripts, which choke quite a bit on both `\`
 920 // characters and on `C:\` paths, so normalize both of them away.
 921 pub fn sanitize_sh(path: &Path) -> String {
 922     let path = path.to_str().unwrap().replace("\\", "/");
 923     return change_drive(&path).unwrap_or(path);
 924
 925     fn change_drive(s: &str) -> Option<String> {
 926         let mut ch = s.chars();
 927         let drive = ch.next().unwrap_or('C');
 928         if ch.next() != Some(':') {
 929             return None
 930         }
 931         if ch.next() != Some('/') {
 932             return None
 933         }
 934         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 935     }
 936 }
 937
 938 fn write_file(path: &Path, data: &[u8]) {
 939     let mut vf = t!(fs::File::create(path));
 940     t!(vf.write_all(data));
 941 }
 942
 943 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 944 pub struct Cargo {
 945     pub stage: u32,
 946     pub target: Interned<String>,
 947 }
 948
 949 impl Step for Cargo {
 950     type Output = PathBuf;
 951     const ONLY_BUILD_TARGETS: bool = true;
 952     const ONLY_HOSTS: bool = true;
 953
 954     fn should_run(run: ShouldRun) -> ShouldRun {
 955         run.path("cargo")
 956     }
 957
 958     fn make_run(run: RunConfig) {
 959         run.builder.ensure(Cargo {
 960             stage: run.builder.top_stage,
 961             target: run.target,
 962         });
 963     }
 964
 965     fn run(self, builder: &Builder) -> PathBuf {
 966         let build = builder.build;
 967         let stage = self.stage;
 968         let target = self.target;
 969
 970         println!("Dist cargo stage{} ({})", stage, target);
 971         let src = build.src.join("src/tools/cargo");
 972         let etc = src.join("src/etc");
 973         let release_num = build.release_num("cargo");
 974         let name = pkgname(build, "cargo");
 975         let version = builder.cargo_info.version(build, &release_num);
 976
 977         let tmp = tmpdir(build);
 978         let image = tmp.join("cargo-image");
 979         drop(fs::remove_dir_all(&image));
 980         t!(fs::create_dir_all(&image));
 981
 982         // Prepare the image directory
 983         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
 984         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
 985         let cargo = builder.ensure(tool::Cargo {
 986             compiler: builder.compiler(stage, build.build),
 987             target
 988         });
 989         install(&cargo, &image.join("bin"), 0o755);
 990         for man in t!(etc.join("man").read_dir()) {
 991             let man = t!(man);
 992             install(&man.path(), &image.join("share/man/man1"), 0o644);
 993         }
 994         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
 995         copy(&etc.join("cargo.bashcomp.sh"),
 996              &image.join("etc/bash_completion.d/cargo"));
 997         let doc = image.join("share/doc/cargo");
 998         install(&src.join("README.md"), &doc, 0o644);
 999         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1000         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1001         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1002
1003         // Prepare the overlay
1004         let overlay = tmp.join("cargo-overlay");
1005         drop(fs::remove_dir_all(&overlay));
1006         t!(fs::create_dir_all(&overlay));
1007         install(&src.join("README.md"), &overlay, 0o644);
1008         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1009         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1010         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1011         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1012
1013         // Generate the installer tarball
1014         let mut cmd = rust_installer(builder);
1015         cmd.arg("generate")
1016            .arg("--product-name=Rust")
1017            .arg("--rel-manifest-dir=rustlib")
1018            .arg("--success-message=Rust-is-ready-to-roll.")
1019            .arg("--image-dir").arg(&image)
1020            .arg("--work-dir").arg(&tmpdir(build))
1021            .arg("--output-dir").arg(&distdir(build))
1022            .arg("--non-installed-overlay").arg(&overlay)
1023            .arg(format!("--package-name={}-{}", name, target))
1024            .arg("--component-name=cargo")
1025            .arg("--legacy-manifest-dirs=rustlib,cargo");
1026         build.run(&mut cmd);
1027         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1028     }
1029 }
1030
1031 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1032 pub struct Rls {
1033     pub stage: u32,
1034     pub target: Interned<String>,
1035 }
1036
1037 impl Step for Rls {
1038     type Output = Option<PathBuf>;
1039     const ONLY_BUILD_TARGETS: bool = true;
1040     const ONLY_HOSTS: bool = true;
1041
1042     fn should_run(run: ShouldRun) -> ShouldRun {
1043         run.path("rls")
1044     }
1045
1046     fn make_run(run: RunConfig) {
1047         run.builder.ensure(Rls {
1048             stage: run.builder.top_stage,
1049             target: run.target,
1050         });
1051     }
1052
1053     fn run(self, builder: &Builder) -> Option<PathBuf> {
1054         let build = builder.build;
1055         let stage = self.stage;
1056         let target = self.target;
1057         assert!(build.config.extended);
1058
1059         if !builder.config.toolstate.rls.testing() {
1060             println!("skipping Dist RLS stage{} ({})", stage, target);
1061             return None
1062         }
1063
1064         println!("Dist RLS stage{} ({})", stage, target);
1065         let src = build.src.join("src/tools/rls");
1066         let release_num = build.release_num("rls");
1067         let name = pkgname(build, "rls");
1068         let version = build.rls_info.version(build, &release_num);
1069
1070         let tmp = tmpdir(build);
1071         let image = tmp.join("rls-image");
1072         drop(fs::remove_dir_all(&image));
1073         t!(fs::create_dir_all(&image));
1074
1075         // Prepare the image directory
1076         // We expect RLS to build, because we've exited this step above if tool
1077         // state for RLS isn't testing.
1078         let rls = builder.ensure(tool::Rls {
1079             compiler: builder.compiler(stage, build.build),
1080             target
1081         }).expect("Rls to build: toolstate is testing");
1082         install(&rls, &image.join("bin"), 0o755);
1083         let doc = image.join("share/doc/rls");
1084         install(&src.join("README.md"), &doc, 0o644);
1085         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1086         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1087
1088         // Prepare the overlay
1089         let overlay = tmp.join("rls-overlay");
1090         drop(fs::remove_dir_all(&overlay));
1091         t!(fs::create_dir_all(&overlay));
1092         install(&src.join("README.md"), &overlay, 0o644);
1093         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1094         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1095         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1096
1097         // Generate the installer tarball
1098         let mut cmd = rust_installer(builder);
1099         cmd.arg("generate")
1100            .arg("--product-name=Rust")
1101            .arg("--rel-manifest-dir=rustlib")
1102            .arg("--success-message=RLS-ready-to-serve.")
1103            .arg("--image-dir").arg(&image)
1104            .arg("--work-dir").arg(&tmpdir(build))
1105            .arg("--output-dir").arg(&distdir(build))
1106            .arg("--non-installed-overlay").arg(&overlay)
1107            .arg(format!("--package-name={}-{}", name, target))
1108            .arg("--legacy-manifest-dirs=rustlib,cargo")
1109            .arg("--component-name=rls-preview");
1110
1111         build.run(&mut cmd);
1112         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1113     }
1114 }
1115
1116
1117 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1118 pub struct Rustfmt {
1119     pub stage: u32,
1120     pub target: Interned<String>,
1121 }
1122
1123 impl Step for Rustfmt {
1124     type Output = Option<PathBuf>;
1125     const ONLY_BUILD_TARGETS: bool = true;
1126     const ONLY_HOSTS: bool = true;
1127
1128     fn should_run(run: ShouldRun) -> ShouldRun {
1129         run.path("rustfmt")
1130     }
1131
1132     fn make_run(run: RunConfig) {
1133         run.builder.ensure(Rustfmt {
1134             stage: run.builder.top_stage,
1135             target: run.target,
1136         });
1137     }
1138
1139     fn run(self, builder: &Builder) -> Option<PathBuf> {
1140         let build = builder.build;
1141         let stage = self.stage;
1142         let target = self.target;
1143         assert!(build.config.extended);
1144
1145         if !builder.config.toolstate.rustfmt.testing() {
1146             println!("skipping Dist Rustfmt stage{} ({})", stage, target);
1147             return None
1148         }
1149
1150         println!("Dist Rustfmt stage{} ({})", stage, target);
1151         let src = build.src.join("src/tools/rustfmt");
1152         let release_num = build.release_num("rustfmt");
1153         let name = pkgname(build, "rustfmt");
1154         let version = build.rustfmt_info.version(build, &release_num);
1155
1156         let tmp = tmpdir(build);
1157         let image = tmp.join("rustfmt-image");
1158         drop(fs::remove_dir_all(&image));
1159         t!(fs::create_dir_all(&image));
1160
1161         // Prepare the image directory
1162         let rustfmt = builder.ensure(tool::Rustfmt {
1163             compiler: builder.compiler(stage, build.build),
1164             target
1165         }).expect("Rustfmt to build: toolstate is testing");
1166         let cargofmt = builder.ensure(tool::Cargofmt {
1167             compiler: builder.compiler(stage, build.build),
1168             target
1169         }).expect("Rustfmt to build: toolstate is testing");
1170         install(&rustfmt, &image.join("bin"), 0o755);
1171         install(&cargofmt, &image.join("bin"), 0o755);
1172         let doc = image.join("share/doc/rustfmt");
1173         install(&src.join("README.md"), &doc, 0o644);
1174         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1175         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1176
1177         // Prepare the overlay
1178         let overlay = tmp.join("rustfmt-overlay");
1179         drop(fs::remove_dir_all(&overlay));
1180         t!(fs::create_dir_all(&overlay));
1181         install(&src.join("README.md"), &overlay, 0o644);
1182         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1183         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1184         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1185
1186         // Generate the installer tarball
1187         let mut cmd = rust_installer(builder);
1188         cmd.arg("generate")
1189            .arg("--product-name=Rust")
1190            .arg("--rel-manifest-dir=rustlib")
1191            .arg("--success-message=rustfmt-ready-to-fmt.")
1192            .arg("--image-dir").arg(&image)
1193            .arg("--work-dir").arg(&tmpdir(build))
1194            .arg("--output-dir").arg(&distdir(build))
1195            .arg("--non-installed-overlay").arg(&overlay)
1196            .arg(format!("--package-name={}-{}", name, target))
1197            .arg("--legacy-manifest-dirs=rustlib,cargo")
1198            .arg("--component-name=rustfmt-preview");
1199
1200         build.run(&mut cmd);
1201         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1202     }
1203 }
1204
1205
1206 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1207 pub struct DontDistWithMiriEnabled;
1208
1209 impl Step for DontDistWithMiriEnabled {
1210     type Output = PathBuf;
1211     const DEFAULT: bool = true;
1212
1213     fn should_run(run: ShouldRun) -> ShouldRun {
1214         let build_miri = run.builder.build.config.test_miri;
1215         run.default_condition(build_miri)
1216     }
1217
1218     fn make_run(run: RunConfig) {
1219         run.builder.ensure(DontDistWithMiriEnabled);
1220     }
1221
1222     fn run(self, _: &Builder) -> PathBuf {
1223         panic!("Do not distribute with miri enabled.\n\
1224                 The distributed libraries would include all MIR (increasing binary size).
1225                 The distributed MIR would include validation statements.");
1226     }
1227 }
1228
1229
1230 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1231 pub struct Extended {
1232     stage: u32,
1233     host: Interned<String>,
1234     target: Interned<String>,
1235 }
1236
1237 impl Step for Extended {
1238     type Output = ();
1239     const DEFAULT: bool = true;
1240     const ONLY_BUILD_TARGETS: bool = true;
1241     const ONLY_HOSTS: bool = true;
1242
1243     fn should_run(run: ShouldRun) -> ShouldRun {
1244         let builder = run.builder;
1245         run.path("extended").default_condition(builder.config.extended)
1246     }
1247
1248     fn make_run(run: RunConfig) {
1249         run.builder.ensure(Extended {
1250             stage: run.builder.top_stage,
1251             host: run.host,
1252             target: run.target,
1253         });
1254     }
1255
1256     /// Creates a combined installer for the specified target in the provided stage.
1257     fn run(self, builder: &Builder) {
1258         let build = builder.build;
1259         let stage = self.stage;
1260         let target = self.target;
1261
1262         println!("Dist extended stage{} ({})", stage, target);
1263
1264         let rustc_installer = builder.ensure(Rustc {
1265             compiler: builder.compiler(stage, target),
1266         });
1267         let cargo_installer = builder.ensure(Cargo { stage, target });
1268         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1269         let rls_installer = builder.ensure(Rls { stage, target });
1270         let mingw_installer = builder.ensure(Mingw { host: target });
1271         let analysis_installer = builder.ensure(Analysis {
1272             compiler: builder.compiler(stage, self.host),
1273             target
1274         });
1275
1276         let docs_installer = builder.ensure(Docs { stage, host: target, });
1277         let std_installer = builder.ensure(Std {
1278             compiler: builder.compiler(stage, self.host),
1279             target,
1280         });
1281
1282         let tmp = tmpdir(build);
1283         let overlay = tmp.join("extended-overlay");
1284         let etc = build.src.join("src/etc/installer");
1285         let work = tmp.join("work");
1286
1287         let _ = fs::remove_dir_all(&overlay);
1288         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1289         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1290         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1291         let version = build.rust_version();
1292         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1293         if let Some(sha) = build.rust_sha() {
1294             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1295         }
1296         install(&etc.join("README.md"), &overlay, 0o644);
1297
1298         // When rust-std package split from rustc, we needed to ensure that during
1299         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1300         // the std files during uninstall. To do this ensure that rustc comes
1301         // before rust-std in the list below.
1302         let mut tarballs = Vec::new();
1303         tarballs.push(rustc_installer);
1304         tarballs.push(cargo_installer);
1305         tarballs.extend(rls_installer.clone());
1306         tarballs.extend(rustfmt_installer.clone());
1307         tarballs.push(analysis_installer);
1308         tarballs.push(std_installer);
1309         if build.config.docs {
1310             tarballs.push(docs_installer);
1311         }
1312         if target.contains("pc-windows-gnu") {
1313             tarballs.push(mingw_installer.unwrap());
1314         }
1315         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1316         for tarball in &tarballs[1..] {
1317             input_tarballs.push(",");
1318             input_tarballs.push(tarball);
1319         }
1320
1321         let mut cmd = rust_installer(builder);
1322         cmd.arg("combine")
1323             .arg("--product-name=Rust")
1324             .arg("--rel-manifest-dir=rustlib")
1325             .arg("--success-message=Rust-is-ready-to-roll.")
1326             .arg("--work-dir").arg(&work)
1327             .arg("--output-dir").arg(&distdir(build))
1328             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1329             .arg("--legacy-manifest-dirs=rustlib,cargo")
1330             .arg("--input-tarballs").arg(input_tarballs)
1331             .arg("--non-installed-overlay").arg(&overlay);
1332         build.run(&mut cmd);
1333
1334         let mut license = String::new();
1335         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1336         license.push_str("\n");
1337         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1338         license.push_str("\n");
1339         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1340
1341         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1342         let mut rtf = rtf.to_string();
1343         rtf.push_str("\n");
1344         for line in license.lines() {
1345             rtf.push_str(line);
1346             rtf.push_str("\\line ");
1347         }
1348         rtf.push_str("}");
1349
1350         fn filter(contents: &str, marker: &str) -> String {
1351             let start = format!("tool-{}-start", marker);
1352             let end = format!("tool-{}-end", marker);
1353             let mut lines = Vec::new();
1354             let mut omitted = false;
1355             for line in contents.lines() {
1356                 if line.contains(&start) {
1357                     omitted = true;
1358                 } else if line.contains(&end) {
1359                     omitted = false;
1360                 } else if !omitted {
1361                     lines.push(line);
1362                 }
1363             }
1364
1365             lines.join("\n")
1366         }
1367
1368         let xform = |p: &Path| {
1369             let mut contents = String::new();
1370             t!(t!(File::open(p)).read_to_string(&mut contents));
1371             if rls_installer.is_none() {
1372                 contents = filter(&contents, "rls");
1373             }
1374             if rustfmt_installer.is_none() {
1375                 contents = filter(&contents, "rustfmt");
1376             }
1377             let ret = tmp.join(p.file_name().unwrap());
1378             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1379             return ret
1380         };
1381
1382         if target.contains("apple-darwin") {
1383             let pkg = tmp.join("pkg");
1384             let _ = fs::remove_dir_all(&pkg);
1385
1386             let pkgbuild = |component: &str| {
1387                 let mut cmd = Command::new("pkgbuild");
1388                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1389                     .arg("--scripts").arg(pkg.join(component))
1390                     .arg("--nopayload")
1391                     .arg(pkg.join(component).with_extension("pkg"));
1392                 build.run(&mut cmd);
1393             };
1394
1395             let prepare = |name: &str| {
1396                 t!(fs::create_dir_all(pkg.join(name)));
1397                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1398                         &pkg.join(name));
1399                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1400                 pkgbuild(name);
1401             };
1402             prepare("rustc");
1403             prepare("cargo");
1404             prepare("rust-docs");
1405             prepare("rust-std");
1406             prepare("rust-analysis");
1407
1408             if rls_installer.is_some() {
1409                 prepare("rls");
1410             }
1411
1412             // create an 'uninstall' package
1413             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1414             pkgbuild("uninstall");
1415
1416             t!(fs::create_dir_all(pkg.join("res")));
1417             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1418             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1419             let mut cmd = Command::new("productbuild");
1420             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1421                 .arg("--resources").arg(pkg.join("res"))
1422                 .arg(distdir(build).join(format!("{}-{}.pkg",
1423                                                     pkgname(build, "rust"),
1424                                                     target)))
1425                 .arg("--package-path").arg(&pkg);
1426             build.run(&mut cmd);
1427         }
1428
1429         if target.contains("windows") {
1430             let exe = tmp.join("exe");
1431             let _ = fs::remove_dir_all(&exe);
1432
1433             let prepare = |name: &str| {
1434                 t!(fs::create_dir_all(exe.join(name)));
1435                 let dir = if name == "rust-std" || name == "rust-analysis" {
1436                     format!("{}-{}", name, target)
1437                 } else if name == "rls" {
1438                     "rls-preview".to_string()
1439                 } else {
1440                     name.to_string()
1441                 };
1442                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1443                             .join(dir),
1444                         &exe.join(name));
1445                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1446             };
1447             prepare("rustc");
1448             prepare("cargo");
1449             prepare("rust-analysis");
1450             prepare("rust-docs");
1451             prepare("rust-std");
1452             if rls_installer.is_some() {
1453                 prepare("rls");
1454             }
1455             if target.contains("windows-gnu") {
1456                 prepare("rust-mingw");
1457             }
1458
1459             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1460             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1461             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1462             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1463             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1464
1465             // Generate exe installer
1466             let mut cmd = Command::new("iscc");
1467             cmd.arg("rust.iss")
1468                 .current_dir(&exe);
1469             if target.contains("windows-gnu") {
1470                 cmd.arg("/dMINGW");
1471             }
1472             add_env(build, &mut cmd, target);
1473             build.run(&mut cmd);
1474             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1475                     &distdir(build),
1476                     0o755);
1477
1478             // Generate msi installer
1479             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1480             let heat = wix.join("bin/heat.exe");
1481             let candle = wix.join("bin/candle.exe");
1482             let light = wix.join("bin/light.exe");
1483
1484             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1485             build.run(Command::new(&heat)
1486                             .current_dir(&exe)
1487                             .arg("dir")
1488                             .arg("rustc")
1489                             .args(&heat_flags)
1490                             .arg("-cg").arg("RustcGroup")
1491                             .arg("-dr").arg("Rustc")
1492                             .arg("-var").arg("var.RustcDir")
1493                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1494             build.run(Command::new(&heat)
1495                             .current_dir(&exe)
1496                             .arg("dir")
1497                             .arg("rust-docs")
1498                             .args(&heat_flags)
1499                             .arg("-cg").arg("DocsGroup")
1500                             .arg("-dr").arg("Docs")
1501                             .arg("-var").arg("var.DocsDir")
1502                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1503                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1504             build.run(Command::new(&heat)
1505                             .current_dir(&exe)
1506                             .arg("dir")
1507                             .arg("cargo")
1508                             .args(&heat_flags)
1509                             .arg("-cg").arg("CargoGroup")
1510                             .arg("-dr").arg("Cargo")
1511                             .arg("-var").arg("var.CargoDir")
1512                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1513                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1514             build.run(Command::new(&heat)
1515                             .current_dir(&exe)
1516                             .arg("dir")
1517                             .arg("rust-std")
1518                             .args(&heat_flags)
1519                             .arg("-cg").arg("StdGroup")
1520                             .arg("-dr").arg("Std")
1521                             .arg("-var").arg("var.StdDir")
1522                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1523             if rls_installer.is_some() {
1524                 build.run(Command::new(&heat)
1525                                 .current_dir(&exe)
1526                                 .arg("dir")
1527                                 .arg("rls")
1528                                 .args(&heat_flags)
1529                                 .arg("-cg").arg("RlsGroup")
1530                                 .arg("-dr").arg("Rls")
1531                                 .arg("-var").arg("var.RlsDir")
1532                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1533                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1534             }
1535             build.run(Command::new(&heat)
1536                             .current_dir(&exe)
1537                             .arg("dir")
1538                             .arg("rust-analysis")
1539                             .args(&heat_flags)
1540                             .arg("-cg").arg("AnalysisGroup")
1541                             .arg("-dr").arg("Analysis")
1542                             .arg("-var").arg("var.AnalysisDir")
1543                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1544                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1545             if target.contains("windows-gnu") {
1546                 build.run(Command::new(&heat)
1547                                 .current_dir(&exe)
1548                                 .arg("dir")
1549                                 .arg("rust-mingw")
1550                                 .args(&heat_flags)
1551                                 .arg("-cg").arg("GccGroup")
1552                                 .arg("-dr").arg("Gcc")
1553                                 .arg("-var").arg("var.GccDir")
1554                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1555             }
1556
1557             let candle = |input: &Path| {
1558                 let output = exe.join(input.file_stem().unwrap())
1559                                 .with_extension("wixobj");
1560                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1561                 let mut cmd = Command::new(&candle);
1562                 cmd.current_dir(&exe)
1563                     .arg("-nologo")
1564                     .arg("-dRustcDir=rustc")
1565                     .arg("-dDocsDir=rust-docs")
1566                     .arg("-dCargoDir=cargo")
1567                     .arg("-dStdDir=rust-std")
1568                     .arg("-dAnalysisDir=rust-analysis")
1569                     .arg("-arch").arg(&arch)
1570                     .arg("-out").arg(&output)
1571                     .arg(&input);
1572                 add_env(build, &mut cmd, target);
1573
1574                 if rls_installer.is_some() {
1575                     cmd.arg("-dRlsDir=rls");
1576                 }
1577                 if target.contains("windows-gnu") {
1578                     cmd.arg("-dGccDir=rust-mingw");
1579                 }
1580                 build.run(&mut cmd);
1581             };
1582             candle(&xform(&etc.join("msi/rust.wxs")));
1583             candle(&etc.join("msi/ui.wxs"));
1584             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1585             candle("RustcGroup.wxs".as_ref());
1586             candle("DocsGroup.wxs".as_ref());
1587             candle("CargoGroup.wxs".as_ref());
1588             candle("StdGroup.wxs".as_ref());
1589             if rls_installer.is_some() {
1590                 candle("RlsGroup.wxs".as_ref());
1591             }
1592             candle("AnalysisGroup.wxs".as_ref());
1593
1594             if target.contains("windows-gnu") {
1595                 candle("GccGroup.wxs".as_ref());
1596             }
1597
1598             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1599             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1600             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1601
1602             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1603             let mut cmd = Command::new(&light);
1604             cmd.arg("-nologo")
1605                 .arg("-ext").arg("WixUIExtension")
1606                 .arg("-ext").arg("WixUtilExtension")
1607                 .arg("-out").arg(exe.join(&filename))
1608                 .arg("rust.wixobj")
1609                 .arg("ui.wixobj")
1610                 .arg("rustwelcomedlg.wixobj")
1611                 .arg("RustcGroup.wixobj")
1612                 .arg("DocsGroup.wixobj")
1613                 .arg("CargoGroup.wixobj")
1614                 .arg("StdGroup.wixobj")
1615                 .arg("AnalysisGroup.wixobj")
1616                 .current_dir(&exe);
1617
1618             if rls_installer.is_some() {
1619                 cmd.arg("RlsGroup.wixobj");
1620             }
1621
1622             if target.contains("windows-gnu") {
1623                 cmd.arg("GccGroup.wixobj");
1624             }
1625             // ICE57 wrongly complains about the shortcuts
1626             cmd.arg("-sice:ICE57");
1627
1628             build.run(&mut cmd);
1629
1630             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1631         }
1632     }
1633 }
1634
1635 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1636     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1637     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1638        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1639        .env("CFG_RELEASE", build.rust_release())
1640        .env("CFG_PRERELEASE_VERSION", channel::CFG_PRERELEASE_VERSION)
1641        .env("CFG_VER_MAJOR", parts.next().unwrap())
1642        .env("CFG_VER_MINOR", parts.next().unwrap())
1643        .env("CFG_VER_PATCH", parts.next().unwrap())
1644        .env("CFG_VER_BUILD", "0") // just needed to build
1645        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1646        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1647        .env("CFG_BUILD", target)
1648        .env("CFG_CHANNEL", &build.config.channel);
1649
1650     if target.contains("windows-gnu") {
1651        cmd.env("CFG_MINGW", "1")
1652           .env("CFG_ABI", "GNU");
1653     } else {
1654        cmd.env("CFG_MINGW", "0")
1655           .env("CFG_ABI", "MSVC");
1656     }
1657
1658     if target.contains("x86_64") {
1659        cmd.env("CFG_PLATFORM", "x64");
1660     } else {
1661        cmd.env("CFG_PLATFORM", "x86");
1662     }
1663 }
1664
1665 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1666 pub struct HashSign;
1667
1668 impl Step for HashSign {
1669     type Output = ();
1670     const ONLY_BUILD_TARGETS: bool = true;
1671     const ONLY_HOSTS: bool = true;
1672     const ONLY_BUILD: bool = true;
1673
1674     fn should_run(run: ShouldRun) -> ShouldRun {
1675         run.path("hash-and-sign")
1676     }
1677
1678     fn make_run(run: RunConfig) {
1679         run.builder.ensure(HashSign);
1680     }
1681
1682     fn run(self, builder: &Builder) {
1683         let build = builder.build;
1684         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1685         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1686             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1687         });
1688         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1689             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1690         });
1691         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1692             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1693         });
1694         let mut pass = String::new();
1695         t!(t!(File::open(&file)).read_to_string(&mut pass));
1696
1697         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1698
1699         cmd.arg(sign);
1700         cmd.arg(distdir(build));
1701         cmd.arg(today.trim());
1702         cmd.arg(build.rust_package_vers());
1703         cmd.arg(build.package_vers(&build.release_num("cargo")));
1704         cmd.arg(build.package_vers(&build.release_num("rls")));
1705         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1706         cmd.arg(addr);
1707
1708         t!(fs::create_dir_all(distdir(build)));
1709
1710         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1711         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1712         let status = t!(child.wait());
1713         assert!(status.success());
1714     }
1715 }