src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36
  37 pub fn pkgname(build: &Build, component: &str) -> String {
  38     if component == "cargo" {
  39         format!("{}-{}", component, build.cargo_package_vers())
  40     } else if component == "rls" {
  41         format!("{}-{}", component, build.rls_package_vers())
  42     } else if component == "rustfmt" {
  43         format!("{}-{}", component, build.rustfmt_package_vers())
  44     } else {
  45         assert!(component.starts_with("rust"));
  46         format!("{}-{}", component, build.rust_package_vers())
  47     }
  48 }
  49
  50 fn distdir(build: &Build) -> PathBuf {
  51     build.out.join("dist")
  52 }
  53
  54 pub fn tmpdir(build: &Build) -> PathBuf {
  55     build.out.join("tmp/dist")
  56 }
  57
  58 fn rust_installer(builder: &Builder) -> Command {
  59     builder.tool_cmd(Tool::RustInstaller)
  60 }
  61
  62 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  63 pub struct Docs {
  64     pub stage: u32,
  65     pub host: Interned<String>,
  66 }
  67
  68 impl Step for Docs {
  69     type Output = PathBuf;
  70     const DEFAULT: bool = true;
  71     const ONLY_BUILD_TARGETS: bool = true;
  72
  73     fn should_run(run: ShouldRun) -> ShouldRun {
  74         run.path("src/doc")
  75     }
  76
  77     fn make_run(run: RunConfig) {
  78         run.builder.ensure(Docs {
  79             stage: run.builder.top_stage,
  80             host: run.target,
  81         });
  82     }
  83
  84     /// Builds the `rust-docs` installer component.
  85     fn run(self, builder: &Builder) -> PathBuf {
  86         let build = builder.build;
  87         let host = self.host;
  88
  89         let name = pkgname(build, "rust-docs");
  90
  91         println!("Dist docs ({})", host);
  92         if !build.config.docs {
  93             println!("\tskipping - docs disabled");
  94             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  95         }
  96
  97         builder.default_doc(None);
  98
  99         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 100         let _ = fs::remove_dir_all(&image);
 101
 102         let dst = image.join("share/doc/rust/html");
 103         t!(fs::create_dir_all(&dst));
 104         let src = build.out.join(host).join("doc");
 105         cp_r(&src, &dst);
 106
 107         let mut cmd = rust_installer(builder);
 108         cmd.arg("generate")
 109            .arg("--product-name=Rust-Documentation")
 110            .arg("--rel-manifest-dir=rustlib")
 111            .arg("--success-message=Rust-documentation-is-installed.")
 112            .arg("--image-dir").arg(&image)
 113            .arg("--work-dir").arg(&tmpdir(build))
 114            .arg("--output-dir").arg(&distdir(build))
 115            .arg(format!("--package-name={}-{}", name, host))
 116            .arg("--component-name=rust-docs")
 117            .arg("--legacy-manifest-dirs=rustlib,cargo")
 118            .arg("--bulk-dirs=share/doc/rust/html");
 119         build.run(&mut cmd);
 120         t!(fs::remove_dir_all(&image));
 121
 122         // As part of this step, *also* copy the docs directory to a directory which
 123         // buildbot typically uploads.
 124         if host == build.build {
 125             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 126             t!(fs::create_dir_all(&dst));
 127             cp_r(&src, &dst);
 128         }
 129
 130         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 131     }
 132 }
 133
 134 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 135     let mut found = Vec::with_capacity(files.len());
 136
 137     for file in files {
 138         let file_path =
 139             path.iter()
 140                 .map(|dir| dir.join(file))
 141                 .find(|p| p.exists());
 142
 143         if let Some(file_path) = file_path {
 144             found.push(file_path);
 145         } else {
 146             panic!("Could not find '{}' in {:?}", file, path);
 147         }
 148     }
 149
 150     found
 151 }
 152
 153 fn make_win_dist(
 154     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 155 ) {
 156     //Ask gcc where it keeps its stuff
 157     let mut cmd = Command::new(build.cc(target_triple));
 158     cmd.arg("-print-search-dirs");
 159     let gcc_out = output(&mut cmd);
 160
 161     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 162     let mut lib_path = Vec::new();
 163
 164     for line in gcc_out.lines() {
 165         let idx = line.find(':').unwrap();
 166         let key = &line[..idx];
 167         let trim_chars: &[_] = &[' ', '='];
 168         let value =
 169             line[(idx + 1)..]
 170                 .trim_left_matches(trim_chars)
 171                 .split(';')
 172                 .map(PathBuf::from);
 173
 174         if key == "programs" {
 175             bin_path.extend(value);
 176         } else if key == "libraries" {
 177             lib_path.extend(value);
 178         }
 179     }
 180
 181     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 182     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 183     if target_triple.starts_with("i686-") {
 184         rustc_dlls.push("libgcc_s_dw2-1.dll");
 185     } else {
 186         rustc_dlls.push("libgcc_s_seh-1.dll");
 187     }
 188
 189     let target_libs = [ //MinGW libs
 190         "libgcc.a",
 191         "libgcc_eh.a",
 192         "libgcc_s.a",
 193         "libm.a",
 194         "libmingw32.a",
 195         "libmingwex.a",
 196         "libstdc++.a",
 197         "libiconv.a",
 198         "libmoldname.a",
 199         "libpthread.a",
 200         //Windows import libs
 201         "libadvapi32.a",
 202         "libbcrypt.a",
 203         "libcomctl32.a",
 204         "libcomdlg32.a",
 205         "libcrypt32.a",
 206         "libgdi32.a",
 207         "libimagehlp.a",
 208         "libiphlpapi.a",
 209         "libkernel32.a",
 210         "libmsvcrt.a",
 211         "libodbc32.a",
 212         "libole32.a",
 213         "liboleaut32.a",
 214         "libopengl32.a",
 215         "libpsapi.a",
 216         "librpcrt4.a",
 217         "libsetupapi.a",
 218         "libshell32.a",
 219         "libuser32.a",
 220         "libuserenv.a",
 221         "libuuid.a",
 222         "libwinhttp.a",
 223         "libwinmm.a",
 224         "libwinspool.a",
 225         "libws2_32.a",
 226         "libwsock32.a",
 227         "libdbghelp.a",
 228         "libmsimg32.a",
 229     ];
 230
 231     //Find mingw artifacts we want to bundle
 232     let target_tools = find_files(&target_tools, &bin_path);
 233     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 234     let target_libs = find_files(&target_libs, &lib_path);
 235
 236     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 237         let file_name = src.file_name().unwrap();
 238         let dest = dest_folder.join(file_name);
 239         copy(src, &dest);
 240     }
 241
 242     //Copy runtime dlls next to rustc.exe
 243     let dist_bin_dir = rust_root.join("bin/");
 244     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 245     for src in rustc_dlls {
 246         copy_to_folder(&src, &dist_bin_dir);
 247     }
 248
 249     //Copy platform tools to platform-specific bin directory
 250     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 251     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 252     for src in target_tools {
 253         copy_to_folder(&src, &target_bin_dir);
 254     }
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     host: Interned<String>,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<PathBuf>;
 271     const DEFAULT: bool = true;
 272     const ONLY_BUILD_TARGETS: bool = true;
 273
 274     fn should_run(run: ShouldRun) -> ShouldRun {
 275         run.never()
 276     }
 277
 278     fn make_run(run: RunConfig) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Build the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder) -> Option<PathBuf> {
 287         let build = builder.build;
 288         let host = self.host;
 289
 290         if !host.contains("pc-windows-gnu") {
 291             return None;
 292         }
 293
 294         println!("Dist mingw ({})", host);
 295         let name = pkgname(build, "rust-mingw");
 296         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 297         let _ = fs::remove_dir_all(&image);
 298         t!(fs::create_dir_all(&image));
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(build), &image, host, &build);
 305
 306         let mut cmd = rust_installer(builder);
 307         cmd.arg("generate")
 308            .arg("--product-name=Rust-MinGW")
 309            .arg("--rel-manifest-dir=rustlib")
 310            .arg("--success-message=Rust-MinGW-is-installed.")
 311            .arg("--image-dir").arg(&image)
 312            .arg("--work-dir").arg(&tmpdir(build))
 313            .arg("--output-dir").arg(&distdir(build))
 314            .arg(format!("--package-name={}-{}", name, host))
 315            .arg("--component-name=rust-mingw")
 316            .arg("--legacy-manifest-dirs=rustlib,cargo");
 317         build.run(&mut cmd);
 318         t!(fs::remove_dir_all(&image));
 319         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 320     }
 321 }
 322
 323 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 324 pub struct Rustc {
 325     pub compiler: Compiler,
 326 }
 327
 328 impl Step for Rustc {
 329     type Output = PathBuf;
 330     const DEFAULT: bool = true;
 331     const ONLY_HOSTS: bool = true;
 332     const ONLY_BUILD_TARGETS: bool = true;
 333
 334     fn should_run(run: ShouldRun) -> ShouldRun {
 335         run.path("src/librustc")
 336     }
 337
 338     fn make_run(run: RunConfig) {
 339         run.builder.ensure(Rustc {
 340             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 341         });
 342     }
 343
 344     /// Creates the `rustc` installer component.
 345     fn run(self, builder: &Builder) -> PathBuf {
 346         let build = builder.build;
 347         let compiler = self.compiler;
 348         let host = self.compiler.host;
 349
 350         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 351         let name = pkgname(build, "rustc");
 352         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 353         let _ = fs::remove_dir_all(&image);
 354         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 355         let _ = fs::remove_dir_all(&overlay);
 356
 357         // Prepare the rustc "image", what will actually end up getting installed
 358         prepare_image(builder, compiler, &image);
 359
 360         // Prepare the overlay which is part of the tarball but won't actually be
 361         // installed
 362         let cp = |file: &str| {
 363             install(&build.src.join(file), &overlay, 0o644);
 364         };
 365         cp("COPYRIGHT");
 366         cp("LICENSE-APACHE");
 367         cp("LICENSE-MIT");
 368         cp("README.md");
 369         // tiny morsel of metadata is used by rust-packaging
 370         let version = build.rust_version();
 371         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 372         if let Some(sha) = build.rust_sha() {
 373             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 374         }
 375
 376         // On MinGW we've got a few runtime DLL dependencies that we need to
 377         // include. The first argument to this script is where to put these DLLs
 378         // (the image we're creating), and the second argument is a junk directory
 379         // to ignore all other MinGW stuff the script creates.
 380         //
 381         // On 32-bit MinGW we're always including a DLL which needs some extra
 382         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 383         // anything requiring us to distribute a license, but it's likely the
 384         // install will *also* include the rust-mingw package, which also needs
 385         // licenses, so to be safe we just include it here in all MinGW packages.
 386         if host.contains("pc-windows-gnu") {
 387             make_win_dist(&image, &tmpdir(build), host, build);
 388
 389             let dst = image.join("share/doc");
 390             t!(fs::create_dir_all(&dst));
 391             cp_r(&build.src.join("src/etc/third-party"), &dst);
 392         }
 393
 394         // Finally, wrap everything up in a nice tarball!
 395         let mut cmd = rust_installer(builder);
 396         cmd.arg("generate")
 397            .arg("--product-name=Rust")
 398            .arg("--rel-manifest-dir=rustlib")
 399            .arg("--success-message=Rust-is-ready-to-roll.")
 400            .arg("--image-dir").arg(&image)
 401            .arg("--work-dir").arg(&tmpdir(build))
 402            .arg("--output-dir").arg(&distdir(build))
 403            .arg("--non-installed-overlay").arg(&overlay)
 404            .arg(format!("--package-name={}-{}", name, host))
 405            .arg("--component-name=rustc")
 406            .arg("--legacy-manifest-dirs=rustlib,cargo");
 407         build.run(&mut cmd);
 408         t!(fs::remove_dir_all(&image));
 409         t!(fs::remove_dir_all(&overlay));
 410
 411         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 412
 413         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 414             let host = compiler.host;
 415             let build = builder.build;
 416             let src = builder.sysroot(compiler);
 417             let libdir = libdir(&host);
 418
 419             // Copy rustc/rustdoc binaries
 420             t!(fs::create_dir_all(image.join("bin")));
 421             cp_r(&src.join("bin"), &image.join("bin"));
 422
 423             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 424
 425             // Copy runtime DLLs needed by the compiler
 426             if libdir != "bin" {
 427                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 428                     let name = entry.file_name();
 429                     if let Some(s) = name.to_str() {
 430                         if is_dylib(s) {
 431                             install(&entry.path(), &image.join(libdir), 0o644);
 432                         }
 433                     }
 434                 }
 435             }
 436
 437             // Copy over the codegen backends
 438             let backends_src = builder.sysroot_libdir(compiler, host)
 439                 .join("codegen-backends");
 440             let backends_dst = image.join("lib/rustlib")
 441                 .join(&*host)
 442                 .join("lib/codegen-backends");
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Man pages
 447             t!(fs::create_dir_all(image.join("share/man/man1")));
 448             let man_src = build.src.join("src/doc/man");
 449             let man_dst = image.join("share/man/man1");
 450             let date_output = output(Command::new("date").arg("+%B %Y"));
 451             let month_year = date_output.trim();
 452             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 453             // to hardlink, and we don't want to edit the source templates
 454             for entry_result in t!(fs::read_dir(man_src)) {
 455                 let file_entry = t!(entry_result);
 456                 let page_src = file_entry.path();
 457                 let page_dst = man_dst.join(file_entry.file_name());
 458                 t!(fs::copy(&page_src, &page_dst));
 459                 // template in month/year and version number
 460                 replace_in_file(&page_dst,
 461                                 &[("<INSERT DATE HERE>", month_year),
 462                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 463             }
 464
 465             // Debugger scripts
 466             builder.ensure(DebuggerScripts {
 467                 sysroot: INTERNER.intern_path(image.to_owned()),
 468                 host,
 469             });
 470
 471             // Misc license info
 472             let cp = |file: &str| {
 473                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 474             };
 475             cp("COPYRIGHT");
 476             cp("LICENSE-APACHE");
 477             cp("LICENSE-MIT");
 478             cp("README.md");
 479         }
 480     }
 481 }
 482
 483 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 484 pub struct DebuggerScripts {
 485     pub sysroot: Interned<PathBuf>,
 486     pub host: Interned<String>,
 487 }
 488
 489 impl Step for DebuggerScripts {
 490     type Output = ();
 491
 492     fn should_run(run: ShouldRun) -> ShouldRun {
 493         run.path("src/lldb_batchmode.py")
 494     }
 495
 496     fn make_run(run: RunConfig) {
 497         run.builder.ensure(DebuggerScripts {
 498             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 499             host: run.target,
 500         });
 501     }
 502
 503     /// Copies debugger scripts for `target` into the `sysroot` specified.
 504     fn run(self, builder: &Builder) {
 505         let build = builder.build;
 506         let host = self.host;
 507         let sysroot = self.sysroot;
 508         let dst = sysroot.join("lib/rustlib/etc");
 509         t!(fs::create_dir_all(&dst));
 510         let cp_debugger_script = |file: &str| {
 511             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 512         };
 513         if host.contains("windows-msvc") {
 514             // windbg debugger scripts
 515             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 516                 0o755);
 517
 518             cp_debugger_script("natvis/intrinsic.natvis");
 519             cp_debugger_script("natvis/liballoc.natvis");
 520             cp_debugger_script("natvis/libcore.natvis");
 521         } else {
 522             cp_debugger_script("debugger_pretty_printers_common.py");
 523
 524             // gdb debugger scripts
 525             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 526                     0o755);
 527
 528             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 529             cp_debugger_script("gdb_rust_pretty_printing.py");
 530
 531             // lldb debugger scripts
 532             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 533                     0o755);
 534
 535             cp_debugger_script("lldb_rust_formatters.py");
 536         }
 537     }
 538 }
 539
 540 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 541 pub struct Std {
 542     pub compiler: Compiler,
 543     pub target: Interned<String>,
 544 }
 545
 546 impl Step for Std {
 547     type Output = PathBuf;
 548     const DEFAULT: bool = true;
 549     const ONLY_BUILD_TARGETS: bool = true;
 550
 551     fn should_run(run: ShouldRun) -> ShouldRun {
 552         run.path("src/libstd")
 553     }
 554
 555     fn make_run(run: RunConfig) {
 556         run.builder.ensure(Std {
 557             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 558             target: run.target,
 559         });
 560     }
 561
 562     fn run(self, builder: &Builder) -> PathBuf {
 563         let build = builder.build;
 564         let compiler = self.compiler;
 565         let target = self.target;
 566
 567         let name = pkgname(build, "rust-std");
 568         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 569
 570         // The only true set of target libraries came from the build triple, so
 571         // let's reduce redundant work by only producing archives from that host.
 572         if compiler.host != build.build {
 573             println!("\tskipping, not a build host");
 574             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 575         }
 576
 577         // We want to package up as many target libraries as possible
 578         // for the `rust-std` package, so if this is a host target we
 579         // depend on librustc and otherwise we just depend on libtest.
 580         if build.hosts.iter().any(|t| t == target) {
 581             builder.ensure(compile::Rustc { compiler, target });
 582         } else {
 583             builder.ensure(compile::Test { compiler, target });
 584         }
 585
 586         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 587         let _ = fs::remove_dir_all(&image);
 588
 589         let dst = image.join("lib/rustlib").join(target);
 590         t!(fs::create_dir_all(&dst));
 591         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 592         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 593         cp_filtered(&src, &dst, &|path| {
 594             path.file_name().and_then(|s| s.to_str()) != Some("codegen-backends")
 595         });
 596
 597         let mut cmd = rust_installer(builder);
 598         cmd.arg("generate")
 599            .arg("--product-name=Rust")
 600            .arg("--rel-manifest-dir=rustlib")
 601            .arg("--success-message=std-is-standing-at-the-ready.")
 602            .arg("--image-dir").arg(&image)
 603            .arg("--work-dir").arg(&tmpdir(build))
 604            .arg("--output-dir").arg(&distdir(build))
 605            .arg(format!("--package-name={}-{}", name, target))
 606            .arg(format!("--component-name=rust-std-{}", target))
 607            .arg("--legacy-manifest-dirs=rustlib,cargo");
 608         build.run(&mut cmd);
 609         t!(fs::remove_dir_all(&image));
 610         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 611     }
 612 }
 613
 614 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 615 pub struct Analysis {
 616     pub compiler: Compiler,
 617     pub target: Interned<String>,
 618 }
 619
 620 impl Step for Analysis {
 621     type Output = PathBuf;
 622     const DEFAULT: bool = true;
 623     const ONLY_BUILD_TARGETS: bool = true;
 624
 625     fn should_run(run: ShouldRun) -> ShouldRun {
 626         let builder = run.builder;
 627         run.path("analysis").default_condition(builder.build.config.extended)
 628     }
 629
 630     fn make_run(run: RunConfig) {
 631         run.builder.ensure(Analysis {
 632             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 633             target: run.target,
 634         });
 635     }
 636
 637     /// Creates a tarball of save-analysis metadata, if available.
 638     fn run(self, builder: &Builder) -> PathBuf {
 639         let build = builder.build;
 640         let compiler = self.compiler;
 641         let target = self.target;
 642         assert!(build.config.extended);
 643         println!("Dist analysis");
 644         let name = pkgname(build, "rust-analysis");
 645
 646         if &compiler.host != build.build {
 647             println!("\tskipping, not a build host");
 648             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 649         }
 650
 651         builder.ensure(Std { compiler, target });
 652
 653         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 654         // stage2 artifacts is simply copied from stage1 in that case.
 655         let compiler = if build.force_use_stage1(compiler, target) {
 656             builder.compiler(1, compiler.host)
 657         } else {
 658             compiler.clone()
 659         };
 660
 661         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 662
 663         let src = build.stage_out(compiler, Mode::Libstd)
 664             .join(target).join(build.cargo_dir()).join("deps");
 665
 666         let image_src = src.join("save-analysis");
 667         let dst = image.join("lib/rustlib").join(target).join("analysis");
 668         t!(fs::create_dir_all(&dst));
 669         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 670         cp_r(&image_src, &dst);
 671
 672         let mut cmd = rust_installer(builder);
 673         cmd.arg("generate")
 674            .arg("--product-name=Rust")
 675            .arg("--rel-manifest-dir=rustlib")
 676            .arg("--success-message=save-analysis-saved.")
 677            .arg("--image-dir").arg(&image)
 678            .arg("--work-dir").arg(&tmpdir(build))
 679            .arg("--output-dir").arg(&distdir(build))
 680            .arg(format!("--package-name={}-{}", name, target))
 681            .arg(format!("--component-name=rust-analysis-{}", target))
 682            .arg("--legacy-manifest-dirs=rustlib,cargo");
 683         build.run(&mut cmd);
 684         t!(fs::remove_dir_all(&image));
 685         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 686     }
 687 }
 688
 689 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 690     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 691         let spath = match path.to_str() {
 692             Some(path) => path,
 693             None => return false,
 694         };
 695         if spath.ends_with("~") || spath.ends_with(".pyc") {
 696             return false
 697         }
 698         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 699             (spath.ends_with(".ll") ||
 700              spath.ends_with(".td") ||
 701              spath.ends_with(".s")) {
 702             return false
 703         }
 704         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 705             return false
 706         }
 707
 708         let full_path = Path::new(dir).join(path);
 709         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 710             return false;
 711         }
 712
 713         let excludes = [
 714             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 715             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 716             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 717             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 718         ];
 719         !path.iter()
 720              .map(|s| s.to_str().unwrap())
 721              .any(|s| excludes.contains(&s))
 722     }
 723
 724     // Copy the directories using our filter
 725     for item in src_dirs {
 726         let dst = &dst_dir.join(item);
 727         t!(fs::create_dir_all(dst));
 728         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 729     }
 730 }
 731
 732 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 733 pub struct Src;
 734
 735 impl Step for Src {
 736     /// The output path of the src installer tarball
 737     type Output = PathBuf;
 738     const DEFAULT: bool = true;
 739     const ONLY_HOSTS: bool = true;
 740     const ONLY_BUILD_TARGETS: bool = true;
 741     const ONLY_BUILD: bool = true;
 742
 743     fn should_run(run: ShouldRun) -> ShouldRun {
 744         run.path("src")
 745     }
 746
 747     fn make_run(run: RunConfig) {
 748         run.builder.ensure(Src);
 749     }
 750
 751     /// Creates the `rust-src` installer component
 752     fn run(self, builder: &Builder) -> PathBuf {
 753         let build = builder.build;
 754         println!("Dist src");
 755
 756         let name = pkgname(build, "rust-src");
 757         let image = tmpdir(build).join(format!("{}-image", name));
 758         let _ = fs::remove_dir_all(&image);
 759
 760         let dst = image.join("lib/rustlib/src");
 761         let dst_src = dst.join("rust");
 762         t!(fs::create_dir_all(&dst_src));
 763
 764         let src_files = [
 765             "src/Cargo.lock",
 766         ];
 767         // This is the reduced set of paths which will become the rust-src component
 768         // (essentially libstd and all of its path dependencies)
 769         let std_src_dirs = [
 770             "src/build_helper",
 771             "src/dlmalloc",
 772             "src/liballoc",
 773             "src/liballoc_jemalloc",
 774             "src/liballoc_system",
 775             "src/libbacktrace",
 776             "src/libcompiler_builtins",
 777             "src/libcore",
 778             "src/liblibc",
 779             "src/libpanic_abort",
 780             "src/libpanic_unwind",
 781             "src/librustc_asan",
 782             "src/librustc_lsan",
 783             "src/librustc_msan",
 784             "src/librustc_tsan",
 785             "src/libstd",
 786             "src/libstd_unicode",
 787             "src/libunwind",
 788             "src/rustc/compiler_builtins_shim",
 789             "src/rustc/libc_shim",
 790             "src/rustc/dlmalloc_shim",
 791             "src/libtest",
 792             "src/libterm",
 793             "src/jemalloc",
 794             "src/libprofiler_builtins",
 795         ];
 796         let std_src_dirs_exclude = [
 797             "src/libcompiler_builtins/compiler-rt/test",
 798             "src/jemalloc/test/unit",
 799         ];
 800
 801         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 802         for file in src_files.iter() {
 803             copy(&build.src.join(file), &dst_src.join(file));
 804         }
 805
 806         // Create source tarball in rust-installer format
 807         let mut cmd = rust_installer(builder);
 808         cmd.arg("generate")
 809            .arg("--product-name=Rust")
 810            .arg("--rel-manifest-dir=rustlib")
 811            .arg("--success-message=Awesome-Source.")
 812            .arg("--image-dir").arg(&image)
 813            .arg("--work-dir").arg(&tmpdir(build))
 814            .arg("--output-dir").arg(&distdir(build))
 815            .arg(format!("--package-name={}", name))
 816            .arg("--component-name=rust-src")
 817            .arg("--legacy-manifest-dirs=rustlib,cargo");
 818         build.run(&mut cmd);
 819
 820         t!(fs::remove_dir_all(&image));
 821         distdir(build).join(&format!("{}.tar.gz", name))
 822     }
 823 }
 824
 825 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 826
 827 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 828 pub struct PlainSourceTarball;
 829
 830 impl Step for PlainSourceTarball {
 831     /// Produces the location of the tarball generated
 832     type Output = PathBuf;
 833     const DEFAULT: bool = true;
 834     const ONLY_HOSTS: bool = true;
 835     const ONLY_BUILD_TARGETS: bool = true;
 836     const ONLY_BUILD: bool = true;
 837
 838     fn should_run(run: ShouldRun) -> ShouldRun {
 839         let builder = run.builder;
 840         run.path("src").default_condition(builder.config.rust_dist_src)
 841     }
 842
 843     fn make_run(run: RunConfig) {
 844         run.builder.ensure(PlainSourceTarball);
 845     }
 846
 847     /// Creates the plain source tarball
 848     fn run(self, builder: &Builder) -> PathBuf {
 849         let build = builder.build;
 850         println!("Create plain source tarball");
 851
 852         // Make sure that the root folder of tarball has the correct name
 853         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 854         let plain_dst_src = tmpdir(build).join(&plain_name);
 855         let _ = fs::remove_dir_all(&plain_dst_src);
 856         t!(fs::create_dir_all(&plain_dst_src));
 857
 858         // This is the set of root paths which will become part of the source package
 859         let src_files = [
 860             "COPYRIGHT",
 861             "LICENSE-APACHE",
 862             "LICENSE-MIT",
 863             "CONTRIBUTING.md",
 864             "README.md",
 865             "RELEASES.md",
 866             "configure",
 867             "x.py",
 868             "config.toml.example",
 869         ];
 870         let src_dirs = [
 871             "src",
 872         ];
 873
 874         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 875
 876         // Copy the files normally
 877         for item in &src_files {
 878             copy(&build.src.join(item), &plain_dst_src.join(item));
 879         }
 880
 881         // Create the version file
 882         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 883         if let Some(sha) = build.rust_sha() {
 884             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 885         }
 886
 887         // If we're building from git sources, we need to vendor a complete distribution.
 888         if build.rust_info.is_git() {
 889             // Get cargo-vendor installed, if it isn't already.
 890             let mut has_cargo_vendor = false;
 891             let mut cmd = Command::new(&build.initial_cargo);
 892             for line in output(cmd.arg("install").arg("--list")).lines() {
 893                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 894             }
 895             if !has_cargo_vendor {
 896                 let mut cmd = Command::new(&build.initial_cargo);
 897                 cmd.arg("install")
 898                    .arg("--force")
 899                    .arg("--debug")
 900                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 901                    .arg("cargo-vendor")
 902                    .env("RUSTC", &build.initial_rustc);
 903                 build.run(&mut cmd);
 904             }
 905
 906             // Vendor all Cargo dependencies
 907             let mut cmd = Command::new(&build.initial_cargo);
 908             cmd.arg("vendor")
 909                .current_dir(&plain_dst_src.join("src"));
 910             build.run(&mut cmd);
 911         }
 912
 913         // Create plain source tarball
 914         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 915         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 916         tarball.set_extension(""); // strip .gz
 917         tarball.set_extension(""); // strip .tar
 918         if let Some(dir) = tarball.parent() {
 919             t!(fs::create_dir_all(dir));
 920         }
 921         println!("running installer");
 922         let mut cmd = rust_installer(builder);
 923         cmd.arg("tarball")
 924            .arg("--input").arg(&plain_name)
 925            .arg("--output").arg(&tarball)
 926            .arg("--work-dir=.")
 927            .current_dir(tmpdir(build));
 928         build.run(&mut cmd);
 929         distdir(build).join(&format!("{}.tar.gz", plain_name))
 930     }
 931 }
 932
 933 fn install(src: &Path, dstdir: &Path, perms: u32) {
 934     let dst = dstdir.join(src.file_name().unwrap());
 935     t!(fs::create_dir_all(dstdir));
 936     drop(fs::remove_file(&dst));
 937     {
 938         let mut s = t!(fs::File::open(&src));
 939         let mut d = t!(fs::File::create(&dst));
 940         io::copy(&mut s, &mut d).expect("failed to copy");
 941     }
 942     chmod(&dst, perms);
 943 }
 944
 945 #[cfg(unix)]
 946 fn chmod(path: &Path, perms: u32) {
 947     use std::os::unix::fs::*;
 948     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 949 }
 950 #[cfg(windows)]
 951 fn chmod(_path: &Path, _perms: u32) {}
 952
 953 // We have to run a few shell scripts, which choke quite a bit on both `\`
 954 // characters and on `C:\` paths, so normalize both of them away.
 955 pub fn sanitize_sh(path: &Path) -> String {
 956     let path = path.to_str().unwrap().replace("\\", "/");
 957     return change_drive(&path).unwrap_or(path);
 958
 959     fn change_drive(s: &str) -> Option<String> {
 960         let mut ch = s.chars();
 961         let drive = ch.next().unwrap_or('C');
 962         if ch.next() != Some(':') {
 963             return None
 964         }
 965         if ch.next() != Some('/') {
 966             return None
 967         }
 968         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 969     }
 970 }
 971
 972 fn write_file(path: &Path, data: &[u8]) {
 973     let mut vf = t!(fs::File::create(path));
 974     t!(vf.write_all(data));
 975 }
 976
 977 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 978 pub struct Cargo {
 979     pub stage: u32,
 980     pub target: Interned<String>,
 981 }
 982
 983 impl Step for Cargo {
 984     type Output = PathBuf;
 985     const ONLY_BUILD_TARGETS: bool = true;
 986     const ONLY_HOSTS: bool = true;
 987
 988     fn should_run(run: ShouldRun) -> ShouldRun {
 989         run.path("cargo")
 990     }
 991
 992     fn make_run(run: RunConfig) {
 993         run.builder.ensure(Cargo {
 994             stage: run.builder.top_stage,
 995             target: run.target,
 996         });
 997     }
 998
 999     fn run(self, builder: &Builder) -> PathBuf {
1000         let build = builder.build;
1001         let stage = self.stage;
1002         let target = self.target;
1003
1004         println!("Dist cargo stage{} ({})", stage, target);
1005         let src = build.src.join("src/tools/cargo");
1006         let etc = src.join("src/etc");
1007         let release_num = build.release_num("cargo");
1008         let name = pkgname(build, "cargo");
1009         let version = builder.cargo_info.version(build, &release_num);
1010
1011         let tmp = tmpdir(build);
1012         let image = tmp.join("cargo-image");
1013         drop(fs::remove_dir_all(&image));
1014         t!(fs::create_dir_all(&image));
1015
1016         // Prepare the image directory
1017         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1018         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1019         let cargo = builder.ensure(tool::Cargo {
1020             compiler: builder.compiler(stage, build.build),
1021             target
1022         });
1023         install(&cargo, &image.join("bin"), 0o755);
1024         for man in t!(etc.join("man").read_dir()) {
1025             let man = t!(man);
1026             install(&man.path(), &image.join("share/man/man1"), 0o644);
1027         }
1028         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1029         copy(&etc.join("cargo.bashcomp.sh"),
1030              &image.join("etc/bash_completion.d/cargo"));
1031         let doc = image.join("share/doc/cargo");
1032         install(&src.join("README.md"), &doc, 0o644);
1033         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1034         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1035         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1036
1037         // Prepare the overlay
1038         let overlay = tmp.join("cargo-overlay");
1039         drop(fs::remove_dir_all(&overlay));
1040         t!(fs::create_dir_all(&overlay));
1041         install(&src.join("README.md"), &overlay, 0o644);
1042         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1043         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1044         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1045         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1046
1047         // Generate the installer tarball
1048         let mut cmd = rust_installer(builder);
1049         cmd.arg("generate")
1050            .arg("--product-name=Rust")
1051            .arg("--rel-manifest-dir=rustlib")
1052            .arg("--success-message=Rust-is-ready-to-roll.")
1053            .arg("--image-dir").arg(&image)
1054            .arg("--work-dir").arg(&tmpdir(build))
1055            .arg("--output-dir").arg(&distdir(build))
1056            .arg("--non-installed-overlay").arg(&overlay)
1057            .arg(format!("--package-name={}-{}", name, target))
1058            .arg("--component-name=cargo")
1059            .arg("--legacy-manifest-dirs=rustlib,cargo");
1060         build.run(&mut cmd);
1061         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1062     }
1063 }
1064
1065 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1066 pub struct Rls {
1067     pub stage: u32,
1068     pub target: Interned<String>,
1069 }
1070
1071 impl Step for Rls {
1072     type Output = Option<PathBuf>;
1073     const ONLY_BUILD_TARGETS: bool = true;
1074     const ONLY_HOSTS: bool = true;
1075
1076     fn should_run(run: ShouldRun) -> ShouldRun {
1077         run.path("rls")
1078     }
1079
1080     fn make_run(run: RunConfig) {
1081         run.builder.ensure(Rls {
1082             stage: run.builder.top_stage,
1083             target: run.target,
1084         });
1085     }
1086
1087     fn run(self, builder: &Builder) -> Option<PathBuf> {
1088         let build = builder.build;
1089         let stage = self.stage;
1090         let target = self.target;
1091         assert!(build.config.extended);
1092
1093         println!("Dist RLS stage{} ({})", stage, target);
1094         let src = build.src.join("src/tools/rls");
1095         let release_num = build.release_num("rls");
1096         let name = pkgname(build, "rls");
1097         let version = build.rls_info.version(build, &release_num);
1098
1099         let tmp = tmpdir(build);
1100         let image = tmp.join("rls-image");
1101         drop(fs::remove_dir_all(&image));
1102         t!(fs::create_dir_all(&image));
1103
1104         // Prepare the image directory
1105         // We expect RLS to build, because we've exited this step above if tool
1106         // state for RLS isn't testing.
1107         let rls = builder.ensure(tool::Rls {
1108             compiler: builder.compiler(stage, build.build),
1109             target
1110         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1111
1112         install(&rls, &image.join("bin"), 0o755);
1113         let doc = image.join("share/doc/rls");
1114         install(&src.join("README.md"), &doc, 0o644);
1115         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1116         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1117
1118         // Prepare the overlay
1119         let overlay = tmp.join("rls-overlay");
1120         drop(fs::remove_dir_all(&overlay));
1121         t!(fs::create_dir_all(&overlay));
1122         install(&src.join("README.md"), &overlay, 0o644);
1123         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1124         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1125         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1126
1127         // Generate the installer tarball
1128         let mut cmd = rust_installer(builder);
1129         cmd.arg("generate")
1130            .arg("--product-name=Rust")
1131            .arg("--rel-manifest-dir=rustlib")
1132            .arg("--success-message=RLS-ready-to-serve.")
1133            .arg("--image-dir").arg(&image)
1134            .arg("--work-dir").arg(&tmpdir(build))
1135            .arg("--output-dir").arg(&distdir(build))
1136            .arg("--non-installed-overlay").arg(&overlay)
1137            .arg(format!("--package-name={}-{}", name, target))
1138            .arg("--legacy-manifest-dirs=rustlib,cargo")
1139            .arg("--component-name=rls-preview");
1140
1141         build.run(&mut cmd);
1142         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1143     }
1144 }
1145
1146
1147 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1148 pub struct Rustfmt {
1149     pub stage: u32,
1150     pub target: Interned<String>,
1151 }
1152
1153 impl Step for Rustfmt {
1154     type Output = Option<PathBuf>;
1155     const ONLY_BUILD_TARGETS: bool = true;
1156     const ONLY_HOSTS: bool = true;
1157
1158     fn should_run(run: ShouldRun) -> ShouldRun {
1159         run.path("rustfmt")
1160     }
1161
1162     fn make_run(run: RunConfig) {
1163         run.builder.ensure(Rustfmt {
1164             stage: run.builder.top_stage,
1165             target: run.target,
1166         });
1167     }
1168
1169     fn run(self, builder: &Builder) -> Option<PathBuf> {
1170         let build = builder.build;
1171         let stage = self.stage;
1172         let target = self.target;
1173         assert!(build.config.extended);
1174
1175         println!("Dist Rustfmt stage{} ({})", stage, target);
1176         let src = build.src.join("src/tools/rustfmt");
1177         let release_num = build.release_num("rustfmt");
1178         let name = pkgname(build, "rustfmt");
1179         let version = build.rustfmt_info.version(build, &release_num);
1180
1181         let tmp = tmpdir(build);
1182         let image = tmp.join("rustfmt-image");
1183         drop(fs::remove_dir_all(&image));
1184         t!(fs::create_dir_all(&image));
1185
1186         // Prepare the image directory
1187         let rustfmt = builder.ensure(tool::Rustfmt {
1188             compiler: builder.compiler(stage, build.build),
1189             target
1190         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1191         let cargofmt = builder.ensure(tool::Cargofmt {
1192             compiler: builder.compiler(stage, build.build),
1193             target
1194         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1195
1196         install(&rustfmt, &image.join("bin"), 0o755);
1197         install(&cargofmt, &image.join("bin"), 0o755);
1198         let doc = image.join("share/doc/rustfmt");
1199         install(&src.join("README.md"), &doc, 0o644);
1200         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1201         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1202
1203         // Prepare the overlay
1204         let overlay = tmp.join("rustfmt-overlay");
1205         drop(fs::remove_dir_all(&overlay));
1206         t!(fs::create_dir_all(&overlay));
1207         install(&src.join("README.md"), &overlay, 0o644);
1208         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1209         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1210         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1211
1212         // Generate the installer tarball
1213         let mut cmd = rust_installer(builder);
1214         cmd.arg("generate")
1215            .arg("--product-name=Rust")
1216            .arg("--rel-manifest-dir=rustlib")
1217            .arg("--success-message=rustfmt-ready-to-fmt.")
1218            .arg("--image-dir").arg(&image)
1219            .arg("--work-dir").arg(&tmpdir(build))
1220            .arg("--output-dir").arg(&distdir(build))
1221            .arg("--non-installed-overlay").arg(&overlay)
1222            .arg(format!("--package-name={}-{}", name, target))
1223            .arg("--legacy-manifest-dirs=rustlib,cargo")
1224            .arg("--component-name=rustfmt-preview");
1225
1226         build.run(&mut cmd);
1227         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1228     }
1229 }
1230
1231
1232 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1233 pub struct DontDistWithMiriEnabled;
1234
1235 impl Step for DontDistWithMiriEnabled {
1236     type Output = PathBuf;
1237     const DEFAULT: bool = true;
1238
1239     fn should_run(run: ShouldRun) -> ShouldRun {
1240         let build_miri = run.builder.build.config.test_miri;
1241         run.default_condition(build_miri)
1242     }
1243
1244     fn make_run(run: RunConfig) {
1245         run.builder.ensure(DontDistWithMiriEnabled);
1246     }
1247
1248     fn run(self, _: &Builder) -> PathBuf {
1249         panic!("Do not distribute with miri enabled.\n\
1250                 The distributed libraries would include all MIR (increasing binary size).
1251                 The distributed MIR would include validation statements.");
1252     }
1253 }
1254
1255
1256 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1257 pub struct Extended {
1258     stage: u32,
1259     host: Interned<String>,
1260     target: Interned<String>,
1261 }
1262
1263 impl Step for Extended {
1264     type Output = ();
1265     const DEFAULT: bool = true;
1266     const ONLY_BUILD_TARGETS: bool = true;
1267     const ONLY_HOSTS: bool = true;
1268
1269     fn should_run(run: ShouldRun) -> ShouldRun {
1270         let builder = run.builder;
1271         run.path("extended").default_condition(builder.config.extended)
1272     }
1273
1274     fn make_run(run: RunConfig) {
1275         run.builder.ensure(Extended {
1276             stage: run.builder.top_stage,
1277             host: run.host,
1278             target: run.target,
1279         });
1280     }
1281
1282     /// Creates a combined installer for the specified target in the provided stage.
1283     fn run(self, builder: &Builder) {
1284         let build = builder.build;
1285         let stage = self.stage;
1286         let target = self.target;
1287
1288         println!("Dist extended stage{} ({})", stage, target);
1289
1290         let rustc_installer = builder.ensure(Rustc {
1291             compiler: builder.compiler(stage, target),
1292         });
1293         let cargo_installer = builder.ensure(Cargo { stage, target });
1294         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1295         let rls_installer = builder.ensure(Rls { stage, target });
1296         let mingw_installer = builder.ensure(Mingw { host: target });
1297         let analysis_installer = builder.ensure(Analysis {
1298             compiler: builder.compiler(stage, self.host),
1299             target
1300         });
1301
1302         let docs_installer = builder.ensure(Docs { stage, host: target, });
1303         let std_installer = builder.ensure(Std {
1304             compiler: builder.compiler(stage, self.host),
1305             target,
1306         });
1307
1308         let tmp = tmpdir(build);
1309         let overlay = tmp.join("extended-overlay");
1310         let etc = build.src.join("src/etc/installer");
1311         let work = tmp.join("work");
1312
1313         let _ = fs::remove_dir_all(&overlay);
1314         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1315         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1316         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1317         let version = build.rust_version();
1318         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1319         if let Some(sha) = build.rust_sha() {
1320             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1321         }
1322         install(&etc.join("README.md"), &overlay, 0o644);
1323
1324         // When rust-std package split from rustc, we needed to ensure that during
1325         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1326         // the std files during uninstall. To do this ensure that rustc comes
1327         // before rust-std in the list below.
1328         let mut tarballs = Vec::new();
1329         tarballs.push(rustc_installer);
1330         tarballs.push(cargo_installer);
1331         tarballs.extend(rls_installer.clone());
1332         tarballs.extend(rustfmt_installer.clone());
1333         tarballs.push(analysis_installer);
1334         tarballs.push(std_installer);
1335         if build.config.docs {
1336             tarballs.push(docs_installer);
1337         }
1338         if target.contains("pc-windows-gnu") {
1339             tarballs.push(mingw_installer.unwrap());
1340         }
1341         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1342         for tarball in &tarballs[1..] {
1343             input_tarballs.push(",");
1344             input_tarballs.push(tarball);
1345         }
1346
1347         let mut cmd = rust_installer(builder);
1348         cmd.arg("combine")
1349             .arg("--product-name=Rust")
1350             .arg("--rel-manifest-dir=rustlib")
1351             .arg("--success-message=Rust-is-ready-to-roll.")
1352             .arg("--work-dir").arg(&work)
1353             .arg("--output-dir").arg(&distdir(build))
1354             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1355             .arg("--legacy-manifest-dirs=rustlib,cargo")
1356             .arg("--input-tarballs").arg(input_tarballs)
1357             .arg("--non-installed-overlay").arg(&overlay);
1358         build.run(&mut cmd);
1359
1360         let mut license = String::new();
1361         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1362         license.push_str("\n");
1363         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1364         license.push_str("\n");
1365         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1366
1367         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1368         let mut rtf = rtf.to_string();
1369         rtf.push_str("\n");
1370         for line in license.lines() {
1371             rtf.push_str(line);
1372             rtf.push_str("\\line ");
1373         }
1374         rtf.push_str("}");
1375
1376         fn filter(contents: &str, marker: &str) -> String {
1377             let start = format!("tool-{}-start", marker);
1378             let end = format!("tool-{}-end", marker);
1379             let mut lines = Vec::new();
1380             let mut omitted = false;
1381             for line in contents.lines() {
1382                 if line.contains(&start) {
1383                     omitted = true;
1384                 } else if line.contains(&end) {
1385                     omitted = false;
1386                 } else if !omitted {
1387                     lines.push(line);
1388                 }
1389             }
1390
1391             lines.join("\n")
1392         }
1393
1394         let xform = |p: &Path| {
1395             let mut contents = String::new();
1396             t!(t!(File::open(p)).read_to_string(&mut contents));
1397             if rls_installer.is_none() {
1398                 contents = filter(&contents, "rls");
1399             }
1400             if rustfmt_installer.is_none() {
1401                 contents = filter(&contents, "rustfmt");
1402             }
1403             let ret = tmp.join(p.file_name().unwrap());
1404             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1405             return ret
1406         };
1407
1408         if target.contains("apple-darwin") {
1409             let pkg = tmp.join("pkg");
1410             let _ = fs::remove_dir_all(&pkg);
1411
1412             let pkgbuild = |component: &str| {
1413                 let mut cmd = Command::new("pkgbuild");
1414                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1415                     .arg("--scripts").arg(pkg.join(component))
1416                     .arg("--nopayload")
1417                     .arg(pkg.join(component).with_extension("pkg"));
1418                 build.run(&mut cmd);
1419             };
1420
1421             let prepare = |name: &str| {
1422                 t!(fs::create_dir_all(pkg.join(name)));
1423                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1424                         &pkg.join(name));
1425                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1426                 pkgbuild(name);
1427             };
1428             prepare("rustc");
1429             prepare("cargo");
1430             prepare("rust-docs");
1431             prepare("rust-std");
1432             prepare("rust-analysis");
1433
1434             if rls_installer.is_some() {
1435                 prepare("rls");
1436             }
1437
1438             // create an 'uninstall' package
1439             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1440             pkgbuild("uninstall");
1441
1442             t!(fs::create_dir_all(pkg.join("res")));
1443             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1444             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1445             let mut cmd = Command::new("productbuild");
1446             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1447                 .arg("--resources").arg(pkg.join("res"))
1448                 .arg(distdir(build).join(format!("{}-{}.pkg",
1449                                                     pkgname(build, "rust"),
1450                                                     target)))
1451                 .arg("--package-path").arg(&pkg);
1452             build.run(&mut cmd);
1453         }
1454
1455         if target.contains("windows") {
1456             let exe = tmp.join("exe");
1457             let _ = fs::remove_dir_all(&exe);
1458
1459             let prepare = |name: &str| {
1460                 t!(fs::create_dir_all(exe.join(name)));
1461                 let dir = if name == "rust-std" || name == "rust-analysis" {
1462                     format!("{}-{}", name, target)
1463                 } else if name == "rls" {
1464                     "rls-preview".to_string()
1465                 } else {
1466                     name.to_string()
1467                 };
1468                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1469                             .join(dir),
1470                         &exe.join(name));
1471                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1472             };
1473             prepare("rustc");
1474             prepare("cargo");
1475             prepare("rust-analysis");
1476             prepare("rust-docs");
1477             prepare("rust-std");
1478             if rls_installer.is_some() {
1479                 prepare("rls");
1480             }
1481             if target.contains("windows-gnu") {
1482                 prepare("rust-mingw");
1483             }
1484
1485             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1486             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1487             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1488             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1489             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1490
1491             // Generate exe installer
1492             let mut cmd = Command::new("iscc");
1493             cmd.arg("rust.iss")
1494                 .current_dir(&exe);
1495             if target.contains("windows-gnu") {
1496                 cmd.arg("/dMINGW");
1497             }
1498             add_env(build, &mut cmd, target);
1499             build.run(&mut cmd);
1500             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1501                     &distdir(build),
1502                     0o755);
1503
1504             // Generate msi installer
1505             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1506             let heat = wix.join("bin/heat.exe");
1507             let candle = wix.join("bin/candle.exe");
1508             let light = wix.join("bin/light.exe");
1509
1510             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1511             build.run(Command::new(&heat)
1512                             .current_dir(&exe)
1513                             .arg("dir")
1514                             .arg("rustc")
1515                             .args(&heat_flags)
1516                             .arg("-cg").arg("RustcGroup")
1517                             .arg("-dr").arg("Rustc")
1518                             .arg("-var").arg("var.RustcDir")
1519                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1520             build.run(Command::new(&heat)
1521                             .current_dir(&exe)
1522                             .arg("dir")
1523                             .arg("rust-docs")
1524                             .args(&heat_flags)
1525                             .arg("-cg").arg("DocsGroup")
1526                             .arg("-dr").arg("Docs")
1527                             .arg("-var").arg("var.DocsDir")
1528                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1529                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1530             build.run(Command::new(&heat)
1531                             .current_dir(&exe)
1532                             .arg("dir")
1533                             .arg("cargo")
1534                             .args(&heat_flags)
1535                             .arg("-cg").arg("CargoGroup")
1536                             .arg("-dr").arg("Cargo")
1537                             .arg("-var").arg("var.CargoDir")
1538                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1539                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1540             build.run(Command::new(&heat)
1541                             .current_dir(&exe)
1542                             .arg("dir")
1543                             .arg("rust-std")
1544                             .args(&heat_flags)
1545                             .arg("-cg").arg("StdGroup")
1546                             .arg("-dr").arg("Std")
1547                             .arg("-var").arg("var.StdDir")
1548                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1549             if rls_installer.is_some() {
1550                 build.run(Command::new(&heat)
1551                                 .current_dir(&exe)
1552                                 .arg("dir")
1553                                 .arg("rls")
1554                                 .args(&heat_flags)
1555                                 .arg("-cg").arg("RlsGroup")
1556                                 .arg("-dr").arg("Rls")
1557                                 .arg("-var").arg("var.RlsDir")
1558                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1559                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1560             }
1561             build.run(Command::new(&heat)
1562                             .current_dir(&exe)
1563                             .arg("dir")
1564                             .arg("rust-analysis")
1565                             .args(&heat_flags)
1566                             .arg("-cg").arg("AnalysisGroup")
1567                             .arg("-dr").arg("Analysis")
1568                             .arg("-var").arg("var.AnalysisDir")
1569                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1570                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1571             if target.contains("windows-gnu") {
1572                 build.run(Command::new(&heat)
1573                                 .current_dir(&exe)
1574                                 .arg("dir")
1575                                 .arg("rust-mingw")
1576                                 .args(&heat_flags)
1577                                 .arg("-cg").arg("GccGroup")
1578                                 .arg("-dr").arg("Gcc")
1579                                 .arg("-var").arg("var.GccDir")
1580                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1581             }
1582
1583             let candle = |input: &Path| {
1584                 let output = exe.join(input.file_stem().unwrap())
1585                                 .with_extension("wixobj");
1586                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1587                 let mut cmd = Command::new(&candle);
1588                 cmd.current_dir(&exe)
1589                     .arg("-nologo")
1590                     .arg("-dRustcDir=rustc")
1591                     .arg("-dDocsDir=rust-docs")
1592                     .arg("-dCargoDir=cargo")
1593                     .arg("-dStdDir=rust-std")
1594                     .arg("-dAnalysisDir=rust-analysis")
1595                     .arg("-arch").arg(&arch)
1596                     .arg("-out").arg(&output)
1597                     .arg(&input);
1598                 add_env(build, &mut cmd, target);
1599
1600                 if rls_installer.is_some() {
1601                     cmd.arg("-dRlsDir=rls");
1602                 }
1603                 if target.contains("windows-gnu") {
1604                     cmd.arg("-dGccDir=rust-mingw");
1605                 }
1606                 build.run(&mut cmd);
1607             };
1608             candle(&xform(&etc.join("msi/rust.wxs")));
1609             candle(&etc.join("msi/ui.wxs"));
1610             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1611             candle("RustcGroup.wxs".as_ref());
1612             candle("DocsGroup.wxs".as_ref());
1613             candle("CargoGroup.wxs".as_ref());
1614             candle("StdGroup.wxs".as_ref());
1615             if rls_installer.is_some() {
1616                 candle("RlsGroup.wxs".as_ref());
1617             }
1618             candle("AnalysisGroup.wxs".as_ref());
1619
1620             if target.contains("windows-gnu") {
1621                 candle("GccGroup.wxs".as_ref());
1622             }
1623
1624             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1625             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1626             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1627
1628             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1629             let mut cmd = Command::new(&light);
1630             cmd.arg("-nologo")
1631                 .arg("-ext").arg("WixUIExtension")
1632                 .arg("-ext").arg("WixUtilExtension")
1633                 .arg("-out").arg(exe.join(&filename))
1634                 .arg("rust.wixobj")
1635                 .arg("ui.wixobj")
1636                 .arg("rustwelcomedlg.wixobj")
1637                 .arg("RustcGroup.wixobj")
1638                 .arg("DocsGroup.wixobj")
1639                 .arg("CargoGroup.wixobj")
1640                 .arg("StdGroup.wixobj")
1641                 .arg("AnalysisGroup.wixobj")
1642                 .current_dir(&exe);
1643
1644             if rls_installer.is_some() {
1645                 cmd.arg("RlsGroup.wixobj");
1646             }
1647
1648             if target.contains("windows-gnu") {
1649                 cmd.arg("GccGroup.wixobj");
1650             }
1651             // ICE57 wrongly complains about the shortcuts
1652             cmd.arg("-sice:ICE57");
1653
1654             build.run(&mut cmd);
1655
1656             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1657         }
1658     }
1659 }
1660
1661 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1662     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1663     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1664        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1665        .env("CFG_RELEASE", build.rust_release())
1666        .env("CFG_VER_MAJOR", parts.next().unwrap())
1667        .env("CFG_VER_MINOR", parts.next().unwrap())
1668        .env("CFG_VER_PATCH", parts.next().unwrap())
1669        .env("CFG_VER_BUILD", "0") // just needed to build
1670        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1671        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1672        .env("CFG_BUILD", target)
1673        .env("CFG_CHANNEL", &build.config.channel);
1674
1675     if target.contains("windows-gnu") {
1676        cmd.env("CFG_MINGW", "1")
1677           .env("CFG_ABI", "GNU");
1678     } else {
1679        cmd.env("CFG_MINGW", "0")
1680           .env("CFG_ABI", "MSVC");
1681     }
1682
1683     if target.contains("x86_64") {
1684        cmd.env("CFG_PLATFORM", "x64");
1685     } else {
1686        cmd.env("CFG_PLATFORM", "x86");
1687     }
1688 }
1689
1690 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1691 pub struct HashSign;
1692
1693 impl Step for HashSign {
1694     type Output = ();
1695     const ONLY_BUILD_TARGETS: bool = true;
1696     const ONLY_HOSTS: bool = true;
1697     const ONLY_BUILD: bool = true;
1698
1699     fn should_run(run: ShouldRun) -> ShouldRun {
1700         run.path("hash-and-sign")
1701     }
1702
1703     fn make_run(run: RunConfig) {
1704         run.builder.ensure(HashSign);
1705     }
1706
1707     fn run(self, builder: &Builder) {
1708         let build = builder.build;
1709         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1710         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1711             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1712         });
1713         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1714             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1715         });
1716         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1717             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1718         });
1719         let mut pass = String::new();
1720         t!(t!(File::open(&file)).read_to_string(&mut pass));
1721
1722         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1723
1724         cmd.arg(sign);
1725         cmd.arg(distdir(build));
1726         cmd.arg(today.trim());
1727         cmd.arg(build.rust_package_vers());
1728         cmd.arg(build.package_vers(&build.release_num("cargo")));
1729         cmd.arg(build.package_vers(&build.release_num("rls")));
1730         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1731         cmd.arg(addr);
1732
1733         t!(fs::create_dir_all(distdir(build)));
1734
1735         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1736         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1737         let status = t!(child.wait());
1738         assert!(status.success());
1739     }
1740 }