src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Man pages
 447             t!(fs::create_dir_all(image.join("share/man/man1")));
 448             let man_src = build.src.join("src/doc/man");
 449             let man_dst = image.join("share/man/man1");
 450             let month_year = t!(time::strftime("%B %Y", &time::now()));
 451             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 452             // to hardlink, and we don't want to edit the source templates
 453             for entry_result in t!(fs::read_dir(man_src)) {
 454                 let file_entry = t!(entry_result);
 455                 let page_src = file_entry.path();
 456                 let page_dst = man_dst.join(file_entry.file_name());
 457                 t!(fs::copy(&page_src, &page_dst));
 458                 // template in month/year and version number
 459                 replace_in_file(&page_dst,
 460                                 &[("<INSERT DATE HERE>", &month_year),
 461                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 462             }
 463
 464             // Debugger scripts
 465             builder.ensure(DebuggerScripts {
 466                 sysroot: INTERNER.intern_path(image.to_owned()),
 467                 host,
 468             });
 469
 470             // Misc license info
 471             let cp = |file: &str| {
 472                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 473             };
 474             cp("COPYRIGHT");
 475             cp("LICENSE-APACHE");
 476             cp("LICENSE-MIT");
 477             cp("README.md");
 478         }
 479     }
 480 }
 481
 482 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 483 pub struct DebuggerScripts {
 484     pub sysroot: Interned<PathBuf>,
 485     pub host: Interned<String>,
 486 }
 487
 488 impl Step for DebuggerScripts {
 489     type Output = ();
 490
 491     fn should_run(run: ShouldRun) -> ShouldRun {
 492         run.path("src/lldb_batchmode.py")
 493     }
 494
 495     fn make_run(run: RunConfig) {
 496         run.builder.ensure(DebuggerScripts {
 497             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 498             host: run.target,
 499         });
 500     }
 501
 502     /// Copies debugger scripts for `target` into the `sysroot` specified.
 503     fn run(self, builder: &Builder) {
 504         let build = builder.build;
 505         let host = self.host;
 506         let sysroot = self.sysroot;
 507         let dst = sysroot.join("lib/rustlib/etc");
 508         t!(fs::create_dir_all(&dst));
 509         let cp_debugger_script = |file: &str| {
 510             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 511         };
 512         if host.contains("windows-msvc") {
 513             // windbg debugger scripts
 514             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 515                 0o755);
 516
 517             cp_debugger_script("natvis/intrinsic.natvis");
 518             cp_debugger_script("natvis/liballoc.natvis");
 519             cp_debugger_script("natvis/libcore.natvis");
 520         } else {
 521             cp_debugger_script("debugger_pretty_printers_common.py");
 522
 523             // gdb debugger scripts
 524             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 525                     0o755);
 526
 527             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 528             cp_debugger_script("gdb_rust_pretty_printing.py");
 529
 530             // lldb debugger scripts
 531             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 532                     0o755);
 533
 534             cp_debugger_script("lldb_rust_formatters.py");
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: Interned<String>,
 543 }
 544
 545 impl Step for Std {
 546     type Output = PathBuf;
 547     const DEFAULT: bool = true;
 548     const ONLY_BUILD_TARGETS: bool = true;
 549
 550     fn should_run(run: ShouldRun) -> ShouldRun {
 551         run.path("src/libstd")
 552     }
 553
 554     fn make_run(run: RunConfig) {
 555         run.builder.ensure(Std {
 556             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 557             target: run.target,
 558         });
 559     }
 560
 561     fn run(self, builder: &Builder) -> PathBuf {
 562         let build = builder.build;
 563         let compiler = self.compiler;
 564         let target = self.target;
 565
 566         let name = pkgname(build, "rust-std");
 567         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 568
 569         // The only true set of target libraries came from the build triple, so
 570         // let's reduce redundant work by only producing archives from that host.
 571         if compiler.host != build.build {
 572             println!("\tskipping, not a build host");
 573             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 574         }
 575
 576         // We want to package up as many target libraries as possible
 577         // for the `rust-std` package, so if this is a host target we
 578         // depend on librustc and otherwise we just depend on libtest.
 579         if build.hosts.iter().any(|t| t == target) {
 580             builder.ensure(compile::Rustc { compiler, target });
 581         } else {
 582             builder.ensure(compile::Test { compiler, target });
 583         }
 584
 585         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 586         let _ = fs::remove_dir_all(&image);
 587
 588         let dst = image.join("lib/rustlib").join(target);
 589         t!(fs::create_dir_all(&dst));
 590         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 591         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 592         cp_filtered(&src, &dst, &|path| {
 593             path.file_name().and_then(|s| s.to_str()) != Some("codegen-backends")
 594         });
 595
 596         let mut cmd = rust_installer(builder);
 597         cmd.arg("generate")
 598            .arg("--product-name=Rust")
 599            .arg("--rel-manifest-dir=rustlib")
 600            .arg("--success-message=std-is-standing-at-the-ready.")
 601            .arg("--image-dir").arg(&image)
 602            .arg("--work-dir").arg(&tmpdir(build))
 603            .arg("--output-dir").arg(&distdir(build))
 604            .arg(format!("--package-name={}-{}", name, target))
 605            .arg(format!("--component-name=rust-std-{}", target))
 606            .arg("--legacy-manifest-dirs=rustlib,cargo");
 607         build.run(&mut cmd);
 608         t!(fs::remove_dir_all(&image));
 609         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 610     }
 611 }
 612
 613 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 614 pub struct Analysis {
 615     pub compiler: Compiler,
 616     pub target: Interned<String>,
 617 }
 618
 619 impl Step for Analysis {
 620     type Output = PathBuf;
 621     const DEFAULT: bool = true;
 622     const ONLY_BUILD_TARGETS: bool = true;
 623
 624     fn should_run(run: ShouldRun) -> ShouldRun {
 625         let builder = run.builder;
 626         run.path("analysis").default_condition(builder.build.config.extended)
 627     }
 628
 629     fn make_run(run: RunConfig) {
 630         run.builder.ensure(Analysis {
 631             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 632             target: run.target,
 633         });
 634     }
 635
 636     /// Creates a tarball of save-analysis metadata, if available.
 637     fn run(self, builder: &Builder) -> PathBuf {
 638         let build = builder.build;
 639         let compiler = self.compiler;
 640         let target = self.target;
 641         assert!(build.config.extended);
 642         println!("Dist analysis");
 643         let name = pkgname(build, "rust-analysis");
 644
 645         if &compiler.host != build.build {
 646             println!("\tskipping, not a build host");
 647             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 648         }
 649
 650         builder.ensure(Std { compiler, target });
 651
 652         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 653         // stage2 artifacts is simply copied from stage1 in that case.
 654         let compiler = if build.force_use_stage1(compiler, target) {
 655             builder.compiler(1, compiler.host)
 656         } else {
 657             compiler.clone()
 658         };
 659
 660         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 661
 662         let src = build.stage_out(compiler, Mode::Libstd)
 663             .join(target).join(build.cargo_dir()).join("deps");
 664
 665         let image_src = src.join("save-analysis");
 666         let dst = image.join("lib/rustlib").join(target).join("analysis");
 667         t!(fs::create_dir_all(&dst));
 668         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 669         cp_r(&image_src, &dst);
 670
 671         let mut cmd = rust_installer(builder);
 672         cmd.arg("generate")
 673            .arg("--product-name=Rust")
 674            .arg("--rel-manifest-dir=rustlib")
 675            .arg("--success-message=save-analysis-saved.")
 676            .arg("--image-dir").arg(&image)
 677            .arg("--work-dir").arg(&tmpdir(build))
 678            .arg("--output-dir").arg(&distdir(build))
 679            .arg(format!("--package-name={}-{}", name, target))
 680            .arg(format!("--component-name=rust-analysis-{}", target))
 681            .arg("--legacy-manifest-dirs=rustlib,cargo");
 682         build.run(&mut cmd);
 683         t!(fs::remove_dir_all(&image));
 684         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 685     }
 686 }
 687
 688 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 689     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 690         let spath = match path.to_str() {
 691             Some(path) => path,
 692             None => return false,
 693         };
 694         if spath.ends_with("~") || spath.ends_with(".pyc") {
 695             return false
 696         }
 697         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 698             (spath.ends_with(".ll") ||
 699              spath.ends_with(".td") ||
 700              spath.ends_with(".s")) {
 701             return false
 702         }
 703         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 704             return false
 705         }
 706
 707         let full_path = Path::new(dir).join(path);
 708         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 709             return false;
 710         }
 711
 712         let excludes = [
 713             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 714             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 715             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 716             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 717         ];
 718         !path.iter()
 719              .map(|s| s.to_str().unwrap())
 720              .any(|s| excludes.contains(&s))
 721     }
 722
 723     // Copy the directories using our filter
 724     for item in src_dirs {
 725         let dst = &dst_dir.join(item);
 726         t!(fs::create_dir_all(dst));
 727         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 728     }
 729 }
 730
 731 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 732 pub struct Src;
 733
 734 impl Step for Src {
 735     /// The output path of the src installer tarball
 736     type Output = PathBuf;
 737     const DEFAULT: bool = true;
 738     const ONLY_HOSTS: bool = true;
 739     const ONLY_BUILD_TARGETS: bool = true;
 740     const ONLY_BUILD: bool = true;
 741
 742     fn should_run(run: ShouldRun) -> ShouldRun {
 743         run.path("src")
 744     }
 745
 746     fn make_run(run: RunConfig) {
 747         run.builder.ensure(Src);
 748     }
 749
 750     /// Creates the `rust-src` installer component
 751     fn run(self, builder: &Builder) -> PathBuf {
 752         let build = builder.build;
 753         println!("Dist src");
 754
 755         let name = pkgname(build, "rust-src");
 756         let image = tmpdir(build).join(format!("{}-image", name));
 757         let _ = fs::remove_dir_all(&image);
 758
 759         let dst = image.join("lib/rustlib/src");
 760         let dst_src = dst.join("rust");
 761         t!(fs::create_dir_all(&dst_src));
 762
 763         let src_files = [
 764             "src/Cargo.lock",
 765         ];
 766         // This is the reduced set of paths which will become the rust-src component
 767         // (essentially libstd and all of its path dependencies)
 768         let std_src_dirs = [
 769             "src/build_helper",
 770             "src/dlmalloc",
 771             "src/liballoc",
 772             "src/liballoc_jemalloc",
 773             "src/liballoc_system",
 774             "src/libbacktrace",
 775             "src/libcompiler_builtins",
 776             "src/libcore",
 777             "src/liblibc",
 778             "src/libpanic_abort",
 779             "src/libpanic_unwind",
 780             "src/librustc_asan",
 781             "src/librustc_lsan",
 782             "src/librustc_msan",
 783             "src/librustc_tsan",
 784             "src/libstd",
 785             "src/libstd_unicode",
 786             "src/libunwind",
 787             "src/rustc/compiler_builtins_shim",
 788             "src/rustc/libc_shim",
 789             "src/rustc/dlmalloc_shim",
 790             "src/libtest",
 791             "src/libterm",
 792             "src/jemalloc",
 793             "src/libprofiler_builtins",
 794         ];
 795         let std_src_dirs_exclude = [
 796             "src/libcompiler_builtins/compiler-rt/test",
 797             "src/jemalloc/test/unit",
 798         ];
 799
 800         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 801         for file in src_files.iter() {
 802             copy(&build.src.join(file), &dst_src.join(file));
 803         }
 804
 805         // Create source tarball in rust-installer format
 806         let mut cmd = rust_installer(builder);
 807         cmd.arg("generate")
 808            .arg("--product-name=Rust")
 809            .arg("--rel-manifest-dir=rustlib")
 810            .arg("--success-message=Awesome-Source.")
 811            .arg("--image-dir").arg(&image)
 812            .arg("--work-dir").arg(&tmpdir(build))
 813            .arg("--output-dir").arg(&distdir(build))
 814            .arg(format!("--package-name={}", name))
 815            .arg("--component-name=rust-src")
 816            .arg("--legacy-manifest-dirs=rustlib,cargo");
 817         build.run(&mut cmd);
 818
 819         t!(fs::remove_dir_all(&image));
 820         distdir(build).join(&format!("{}.tar.gz", name))
 821     }
 822 }
 823
 824 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 825
 826 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 827 pub struct PlainSourceTarball;
 828
 829 impl Step for PlainSourceTarball {
 830     /// Produces the location of the tarball generated
 831     type Output = PathBuf;
 832     const DEFAULT: bool = true;
 833     const ONLY_HOSTS: bool = true;
 834     const ONLY_BUILD_TARGETS: bool = true;
 835     const ONLY_BUILD: bool = true;
 836
 837     fn should_run(run: ShouldRun) -> ShouldRun {
 838         let builder = run.builder;
 839         run.path("src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder) -> PathBuf {
 848         let build = builder.build;
 849         println!("Create plain source tarball");
 850
 851         // Make sure that the root folder of tarball has the correct name
 852         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 853         let plain_dst_src = tmpdir(build).join(&plain_name);
 854         let _ = fs::remove_dir_all(&plain_dst_src);
 855         t!(fs::create_dir_all(&plain_dst_src));
 856
 857         // This is the set of root paths which will become part of the source package
 858         let src_files = [
 859             "COPYRIGHT",
 860             "LICENSE-APACHE",
 861             "LICENSE-MIT",
 862             "CONTRIBUTING.md",
 863             "README.md",
 864             "RELEASES.md",
 865             "configure",
 866             "x.py",
 867             "config.toml.example",
 868         ];
 869         let src_dirs = [
 870             "src",
 871         ];
 872
 873         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 874
 875         // Copy the files normally
 876         for item in &src_files {
 877             copy(&build.src.join(item), &plain_dst_src.join(item));
 878         }
 879
 880         // Create the version file
 881         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 882         if let Some(sha) = build.rust_sha() {
 883             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 884         }
 885
 886         // If we're building from git sources, we need to vendor a complete distribution.
 887         if build.rust_info.is_git() {
 888             // Get cargo-vendor installed, if it isn't already.
 889             let mut has_cargo_vendor = false;
 890             let mut cmd = Command::new(&build.initial_cargo);
 891             for line in output(cmd.arg("install").arg("--list")).lines() {
 892                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 893             }
 894             if !has_cargo_vendor {
 895                 let mut cmd = Command::new(&build.initial_cargo);
 896                 cmd.arg("install")
 897                    .arg("--force")
 898                    .arg("--debug")
 899                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 900                    .arg("cargo-vendor")
 901                    .env("RUSTC", &build.initial_rustc);
 902                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 903                     builder.ensure(native::Openssl {
 904                         target: build.config.build,
 905                     });
 906                     cmd.env("OPENSSL_DIR", dir);
 907                 }
 908                 build.run(&mut cmd);
 909             }
 910
 911             // Vendor all Cargo dependencies
 912             let mut cmd = Command::new(&build.initial_cargo);
 913             cmd.arg("vendor")
 914                .current_dir(&plain_dst_src.join("src"));
 915             build.run(&mut cmd);
 916         }
 917
 918         // Create plain source tarball
 919         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 920         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 921         tarball.set_extension(""); // strip .gz
 922         tarball.set_extension(""); // strip .tar
 923         if let Some(dir) = tarball.parent() {
 924             t!(fs::create_dir_all(dir));
 925         }
 926         println!("running installer");
 927         let mut cmd = rust_installer(builder);
 928         cmd.arg("tarball")
 929            .arg("--input").arg(&plain_name)
 930            .arg("--output").arg(&tarball)
 931            .arg("--work-dir=.")
 932            .current_dir(tmpdir(build));
 933         build.run(&mut cmd);
 934         distdir(build).join(&format!("{}.tar.gz", plain_name))
 935     }
 936 }
 937
 938 fn install(src: &Path, dstdir: &Path, perms: u32) {
 939     let dst = dstdir.join(src.file_name().unwrap());
 940     t!(fs::create_dir_all(dstdir));
 941     drop(fs::remove_file(&dst));
 942     {
 943         let mut s = t!(fs::File::open(&src));
 944         let mut d = t!(fs::File::create(&dst));
 945         io::copy(&mut s, &mut d).expect("failed to copy");
 946     }
 947     chmod(&dst, perms);
 948 }
 949
 950 #[cfg(unix)]
 951 fn chmod(path: &Path, perms: u32) {
 952     use std::os::unix::fs::*;
 953     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 954 }
 955 #[cfg(windows)]
 956 fn chmod(_path: &Path, _perms: u32) {}
 957
 958 // We have to run a few shell scripts, which choke quite a bit on both `\`
 959 // characters and on `C:\` paths, so normalize both of them away.
 960 pub fn sanitize_sh(path: &Path) -> String {
 961     let path = path.to_str().unwrap().replace("\\", "/");
 962     return change_drive(&path).unwrap_or(path);
 963
 964     fn change_drive(s: &str) -> Option<String> {
 965         let mut ch = s.chars();
 966         let drive = ch.next().unwrap_or('C');
 967         if ch.next() != Some(':') {
 968             return None
 969         }
 970         if ch.next() != Some('/') {
 971             return None
 972         }
 973         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 974     }
 975 }
 976
 977 fn write_file(path: &Path, data: &[u8]) {
 978     let mut vf = t!(fs::File::create(path));
 979     t!(vf.write_all(data));
 980 }
 981
 982 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 983 pub struct Cargo {
 984     pub stage: u32,
 985     pub target: Interned<String>,
 986 }
 987
 988 impl Step for Cargo {
 989     type Output = PathBuf;
 990     const ONLY_BUILD_TARGETS: bool = true;
 991     const ONLY_HOSTS: bool = true;
 992
 993     fn should_run(run: ShouldRun) -> ShouldRun {
 994         run.path("cargo")
 995     }
 996
 997     fn make_run(run: RunConfig) {
 998         run.builder.ensure(Cargo {
 999             stage: run.builder.top_stage,
1000             target: run.target,
1001         });
1002     }
1003
1004     fn run(self, builder: &Builder) -> PathBuf {
1005         let build = builder.build;
1006         let stage = self.stage;
1007         let target = self.target;
1008
1009         println!("Dist cargo stage{} ({})", stage, target);
1010         let src = build.src.join("src/tools/cargo");
1011         let etc = src.join("src/etc");
1012         let release_num = build.release_num("cargo");
1013         let name = pkgname(build, "cargo");
1014         let version = builder.cargo_info.version(build, &release_num);
1015
1016         let tmp = tmpdir(build);
1017         let image = tmp.join("cargo-image");
1018         drop(fs::remove_dir_all(&image));
1019         t!(fs::create_dir_all(&image));
1020
1021         // Prepare the image directory
1022         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1023         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1024         let cargo = builder.ensure(tool::Cargo {
1025             compiler: builder.compiler(stage, build.build),
1026             target
1027         });
1028         install(&cargo, &image.join("bin"), 0o755);
1029         for man in t!(etc.join("man").read_dir()) {
1030             let man = t!(man);
1031             install(&man.path(), &image.join("share/man/man1"), 0o644);
1032         }
1033         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1034         copy(&etc.join("cargo.bashcomp.sh"),
1035              &image.join("etc/bash_completion.d/cargo"));
1036         let doc = image.join("share/doc/cargo");
1037         install(&src.join("README.md"), &doc, 0o644);
1038         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1039         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1040         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1041
1042         // Prepare the overlay
1043         let overlay = tmp.join("cargo-overlay");
1044         drop(fs::remove_dir_all(&overlay));
1045         t!(fs::create_dir_all(&overlay));
1046         install(&src.join("README.md"), &overlay, 0o644);
1047         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1048         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1049         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1050         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1051
1052         // Generate the installer tarball
1053         let mut cmd = rust_installer(builder);
1054         cmd.arg("generate")
1055            .arg("--product-name=Rust")
1056            .arg("--rel-manifest-dir=rustlib")
1057            .arg("--success-message=Rust-is-ready-to-roll.")
1058            .arg("--image-dir").arg(&image)
1059            .arg("--work-dir").arg(&tmpdir(build))
1060            .arg("--output-dir").arg(&distdir(build))
1061            .arg("--non-installed-overlay").arg(&overlay)
1062            .arg(format!("--package-name={}-{}", name, target))
1063            .arg("--component-name=cargo")
1064            .arg("--legacy-manifest-dirs=rustlib,cargo");
1065         build.run(&mut cmd);
1066         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1067     }
1068 }
1069
1070 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1071 pub struct Rls {
1072     pub stage: u32,
1073     pub target: Interned<String>,
1074 }
1075
1076 impl Step for Rls {
1077     type Output = Option<PathBuf>;
1078     const ONLY_BUILD_TARGETS: bool = true;
1079     const ONLY_HOSTS: bool = true;
1080
1081     fn should_run(run: ShouldRun) -> ShouldRun {
1082         run.path("rls")
1083     }
1084
1085     fn make_run(run: RunConfig) {
1086         run.builder.ensure(Rls {
1087             stage: run.builder.top_stage,
1088             target: run.target,
1089         });
1090     }
1091
1092     fn run(self, builder: &Builder) -> Option<PathBuf> {
1093         let build = builder.build;
1094         let stage = self.stage;
1095         let target = self.target;
1096         assert!(build.config.extended);
1097
1098         println!("Dist RLS stage{} ({})", stage, target);
1099         let src = build.src.join("src/tools/rls");
1100         let release_num = build.release_num("rls");
1101         let name = pkgname(build, "rls");
1102         let version = build.rls_info.version(build, &release_num);
1103
1104         let tmp = tmpdir(build);
1105         let image = tmp.join("rls-image");
1106         drop(fs::remove_dir_all(&image));
1107         t!(fs::create_dir_all(&image));
1108
1109         // Prepare the image directory
1110         // We expect RLS to build, because we've exited this step above if tool
1111         // state for RLS isn't testing.
1112         let rls = builder.ensure(tool::Rls {
1113             compiler: builder.compiler(stage, build.build),
1114             target
1115         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1116
1117         install(&rls, &image.join("bin"), 0o755);
1118         let doc = image.join("share/doc/rls");
1119         install(&src.join("README.md"), &doc, 0o644);
1120         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1121         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1122
1123         // Prepare the overlay
1124         let overlay = tmp.join("rls-overlay");
1125         drop(fs::remove_dir_all(&overlay));
1126         t!(fs::create_dir_all(&overlay));
1127         install(&src.join("README.md"), &overlay, 0o644);
1128         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1129         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1130         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1131
1132         // Generate the installer tarball
1133         let mut cmd = rust_installer(builder);
1134         cmd.arg("generate")
1135            .arg("--product-name=Rust")
1136            .arg("--rel-manifest-dir=rustlib")
1137            .arg("--success-message=RLS-ready-to-serve.")
1138            .arg("--image-dir").arg(&image)
1139            .arg("--work-dir").arg(&tmpdir(build))
1140            .arg("--output-dir").arg(&distdir(build))
1141            .arg("--non-installed-overlay").arg(&overlay)
1142            .arg(format!("--package-name={}-{}", name, target))
1143            .arg("--legacy-manifest-dirs=rustlib,cargo")
1144            .arg("--component-name=rls-preview");
1145
1146         build.run(&mut cmd);
1147         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1148     }
1149 }
1150
1151
1152 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1153 pub struct Rustfmt {
1154     pub stage: u32,
1155     pub target: Interned<String>,
1156 }
1157
1158 impl Step for Rustfmt {
1159     type Output = Option<PathBuf>;
1160     const ONLY_BUILD_TARGETS: bool = true;
1161     const ONLY_HOSTS: bool = true;
1162
1163     fn should_run(run: ShouldRun) -> ShouldRun {
1164         run.path("rustfmt")
1165     }
1166
1167     fn make_run(run: RunConfig) {
1168         run.builder.ensure(Rustfmt {
1169             stage: run.builder.top_stage,
1170             target: run.target,
1171         });
1172     }
1173
1174     fn run(self, builder: &Builder) -> Option<PathBuf> {
1175         let build = builder.build;
1176         let stage = self.stage;
1177         let target = self.target;
1178         assert!(build.config.extended);
1179
1180         println!("Dist Rustfmt stage{} ({})", stage, target);
1181         let src = build.src.join("src/tools/rustfmt");
1182         let release_num = build.release_num("rustfmt");
1183         let name = pkgname(build, "rustfmt");
1184         let version = build.rustfmt_info.version(build, &release_num);
1185
1186         let tmp = tmpdir(build);
1187         let image = tmp.join("rustfmt-image");
1188         drop(fs::remove_dir_all(&image));
1189         t!(fs::create_dir_all(&image));
1190
1191         // Prepare the image directory
1192         let rustfmt = builder.ensure(tool::Rustfmt {
1193             compiler: builder.compiler(stage, build.build),
1194             target
1195         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1196         let cargofmt = builder.ensure(tool::Cargofmt {
1197             compiler: builder.compiler(stage, build.build),
1198             target
1199         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1200
1201         install(&rustfmt, &image.join("bin"), 0o755);
1202         install(&cargofmt, &image.join("bin"), 0o755);
1203         let doc = image.join("share/doc/rustfmt");
1204         install(&src.join("README.md"), &doc, 0o644);
1205         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1206         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1207
1208         // Prepare the overlay
1209         let overlay = tmp.join("rustfmt-overlay");
1210         drop(fs::remove_dir_all(&overlay));
1211         t!(fs::create_dir_all(&overlay));
1212         install(&src.join("README.md"), &overlay, 0o644);
1213         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1214         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1215         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1216
1217         // Generate the installer tarball
1218         let mut cmd = rust_installer(builder);
1219         cmd.arg("generate")
1220            .arg("--product-name=Rust")
1221            .arg("--rel-manifest-dir=rustlib")
1222            .arg("--success-message=rustfmt-ready-to-fmt.")
1223            .arg("--image-dir").arg(&image)
1224            .arg("--work-dir").arg(&tmpdir(build))
1225            .arg("--output-dir").arg(&distdir(build))
1226            .arg("--non-installed-overlay").arg(&overlay)
1227            .arg(format!("--package-name={}-{}", name, target))
1228            .arg("--legacy-manifest-dirs=rustlib,cargo")
1229            .arg("--component-name=rustfmt-preview");
1230
1231         build.run(&mut cmd);
1232         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1233     }
1234 }
1235
1236
1237 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1238 pub struct DontDistWithMiriEnabled;
1239
1240 impl Step for DontDistWithMiriEnabled {
1241     type Output = PathBuf;
1242     const DEFAULT: bool = true;
1243
1244     fn should_run(run: ShouldRun) -> ShouldRun {
1245         let build_miri = run.builder.build.config.test_miri;
1246         run.default_condition(build_miri)
1247     }
1248
1249     fn make_run(run: RunConfig) {
1250         run.builder.ensure(DontDistWithMiriEnabled);
1251     }
1252
1253     fn run(self, _: &Builder) -> PathBuf {
1254         panic!("Do not distribute with miri enabled.\n\
1255                 The distributed libraries would include all MIR (increasing binary size).
1256                 The distributed MIR would include validation statements.");
1257     }
1258 }
1259
1260
1261 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1262 pub struct Extended {
1263     stage: u32,
1264     host: Interned<String>,
1265     target: Interned<String>,
1266 }
1267
1268 impl Step for Extended {
1269     type Output = ();
1270     const DEFAULT: bool = true;
1271     const ONLY_BUILD_TARGETS: bool = true;
1272     const ONLY_HOSTS: bool = true;
1273
1274     fn should_run(run: ShouldRun) -> ShouldRun {
1275         let builder = run.builder;
1276         run.path("extended").default_condition(builder.config.extended)
1277     }
1278
1279     fn make_run(run: RunConfig) {
1280         run.builder.ensure(Extended {
1281             stage: run.builder.top_stage,
1282             host: run.host,
1283             target: run.target,
1284         });
1285     }
1286
1287     /// Creates a combined installer for the specified target in the provided stage.
1288     fn run(self, builder: &Builder) {
1289         let build = builder.build;
1290         let stage = self.stage;
1291         let target = self.target;
1292
1293         println!("Dist extended stage{} ({})", stage, target);
1294
1295         let rustc_installer = builder.ensure(Rustc {
1296             compiler: builder.compiler(stage, target),
1297         });
1298         let cargo_installer = builder.ensure(Cargo { stage, target });
1299         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1300         let rls_installer = builder.ensure(Rls { stage, target });
1301         let mingw_installer = builder.ensure(Mingw { host: target });
1302         let analysis_installer = builder.ensure(Analysis {
1303             compiler: builder.compiler(stage, self.host),
1304             target
1305         });
1306
1307         let docs_installer = builder.ensure(Docs { stage, host: target, });
1308         let std_installer = builder.ensure(Std {
1309             compiler: builder.compiler(stage, self.host),
1310             target,
1311         });
1312
1313         let tmp = tmpdir(build);
1314         let overlay = tmp.join("extended-overlay");
1315         let etc = build.src.join("src/etc/installer");
1316         let work = tmp.join("work");
1317
1318         let _ = fs::remove_dir_all(&overlay);
1319         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1320         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1321         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1322         let version = build.rust_version();
1323         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1324         if let Some(sha) = build.rust_sha() {
1325             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1326         }
1327         install(&etc.join("README.md"), &overlay, 0o644);
1328
1329         // When rust-std package split from rustc, we needed to ensure that during
1330         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1331         // the std files during uninstall. To do this ensure that rustc comes
1332         // before rust-std in the list below.
1333         let mut tarballs = Vec::new();
1334         tarballs.push(rustc_installer);
1335         tarballs.push(cargo_installer);
1336         tarballs.extend(rls_installer.clone());
1337         tarballs.extend(rustfmt_installer.clone());
1338         tarballs.push(analysis_installer);
1339         tarballs.push(std_installer);
1340         if build.config.docs {
1341             tarballs.push(docs_installer);
1342         }
1343         if target.contains("pc-windows-gnu") {
1344             tarballs.push(mingw_installer.unwrap());
1345         }
1346         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1347         for tarball in &tarballs[1..] {
1348             input_tarballs.push(",");
1349             input_tarballs.push(tarball);
1350         }
1351
1352         let mut cmd = rust_installer(builder);
1353         cmd.arg("combine")
1354             .arg("--product-name=Rust")
1355             .arg("--rel-manifest-dir=rustlib")
1356             .arg("--success-message=Rust-is-ready-to-roll.")
1357             .arg("--work-dir").arg(&work)
1358             .arg("--output-dir").arg(&distdir(build))
1359             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1360             .arg("--legacy-manifest-dirs=rustlib,cargo")
1361             .arg("--input-tarballs").arg(input_tarballs)
1362             .arg("--non-installed-overlay").arg(&overlay);
1363         build.run(&mut cmd);
1364
1365         let mut license = String::new();
1366         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1367         license.push_str("\n");
1368         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1369         license.push_str("\n");
1370         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1371
1372         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1373         let mut rtf = rtf.to_string();
1374         rtf.push_str("\n");
1375         for line in license.lines() {
1376             rtf.push_str(line);
1377             rtf.push_str("\\line ");
1378         }
1379         rtf.push_str("}");
1380
1381         fn filter(contents: &str, marker: &str) -> String {
1382             let start = format!("tool-{}-start", marker);
1383             let end = format!("tool-{}-end", marker);
1384             let mut lines = Vec::new();
1385             let mut omitted = false;
1386             for line in contents.lines() {
1387                 if line.contains(&start) {
1388                     omitted = true;
1389                 } else if line.contains(&end) {
1390                     omitted = false;
1391                 } else if !omitted {
1392                     lines.push(line);
1393                 }
1394             }
1395
1396             lines.join("\n")
1397         }
1398
1399         let xform = |p: &Path| {
1400             let mut contents = String::new();
1401             t!(t!(File::open(p)).read_to_string(&mut contents));
1402             if rls_installer.is_none() {
1403                 contents = filter(&contents, "rls");
1404             }
1405             if rustfmt_installer.is_none() {
1406                 contents = filter(&contents, "rustfmt");
1407             }
1408             let ret = tmp.join(p.file_name().unwrap());
1409             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1410             return ret
1411         };
1412
1413         if target.contains("apple-darwin") {
1414             let pkg = tmp.join("pkg");
1415             let _ = fs::remove_dir_all(&pkg);
1416
1417             let pkgbuild = |component: &str| {
1418                 let mut cmd = Command::new("pkgbuild");
1419                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1420                     .arg("--scripts").arg(pkg.join(component))
1421                     .arg("--nopayload")
1422                     .arg(pkg.join(component).with_extension("pkg"));
1423                 build.run(&mut cmd);
1424             };
1425
1426             let prepare = |name: &str| {
1427                 t!(fs::create_dir_all(pkg.join(name)));
1428                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1429                         &pkg.join(name));
1430                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1431                 pkgbuild(name);
1432             };
1433             prepare("rustc");
1434             prepare("cargo");
1435             prepare("rust-docs");
1436             prepare("rust-std");
1437             prepare("rust-analysis");
1438
1439             if rls_installer.is_some() {
1440                 prepare("rls");
1441             }
1442
1443             // create an 'uninstall' package
1444             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1445             pkgbuild("uninstall");
1446
1447             t!(fs::create_dir_all(pkg.join("res")));
1448             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1449             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1450             let mut cmd = Command::new("productbuild");
1451             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1452                 .arg("--resources").arg(pkg.join("res"))
1453                 .arg(distdir(build).join(format!("{}-{}.pkg",
1454                                                     pkgname(build, "rust"),
1455                                                     target)))
1456                 .arg("--package-path").arg(&pkg);
1457             build.run(&mut cmd);
1458         }
1459
1460         if target.contains("windows") {
1461             let exe = tmp.join("exe");
1462             let _ = fs::remove_dir_all(&exe);
1463
1464             let prepare = |name: &str| {
1465                 t!(fs::create_dir_all(exe.join(name)));
1466                 let dir = if name == "rust-std" || name == "rust-analysis" {
1467                     format!("{}-{}", name, target)
1468                 } else if name == "rls" {
1469                     "rls-preview".to_string()
1470                 } else {
1471                     name.to_string()
1472                 };
1473                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1474                             .join(dir),
1475                         &exe.join(name));
1476                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1477             };
1478             prepare("rustc");
1479             prepare("cargo");
1480             prepare("rust-analysis");
1481             prepare("rust-docs");
1482             prepare("rust-std");
1483             if rls_installer.is_some() {
1484                 prepare("rls");
1485             }
1486             if target.contains("windows-gnu") {
1487                 prepare("rust-mingw");
1488             }
1489
1490             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1491             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1492             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1493             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1494             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1495
1496             // Generate exe installer
1497             let mut cmd = Command::new("iscc");
1498             cmd.arg("rust.iss")
1499                 .current_dir(&exe);
1500             if target.contains("windows-gnu") {
1501                 cmd.arg("/dMINGW");
1502             }
1503             add_env(build, &mut cmd, target);
1504             build.run(&mut cmd);
1505             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1506                     &distdir(build),
1507                     0o755);
1508
1509             // Generate msi installer
1510             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1511             let heat = wix.join("bin/heat.exe");
1512             let candle = wix.join("bin/candle.exe");
1513             let light = wix.join("bin/light.exe");
1514
1515             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1516             build.run(Command::new(&heat)
1517                             .current_dir(&exe)
1518                             .arg("dir")
1519                             .arg("rustc")
1520                             .args(&heat_flags)
1521                             .arg("-cg").arg("RustcGroup")
1522                             .arg("-dr").arg("Rustc")
1523                             .arg("-var").arg("var.RustcDir")
1524                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1525             build.run(Command::new(&heat)
1526                             .current_dir(&exe)
1527                             .arg("dir")
1528                             .arg("rust-docs")
1529                             .args(&heat_flags)
1530                             .arg("-cg").arg("DocsGroup")
1531                             .arg("-dr").arg("Docs")
1532                             .arg("-var").arg("var.DocsDir")
1533                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1534                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1535             build.run(Command::new(&heat)
1536                             .current_dir(&exe)
1537                             .arg("dir")
1538                             .arg("cargo")
1539                             .args(&heat_flags)
1540                             .arg("-cg").arg("CargoGroup")
1541                             .arg("-dr").arg("Cargo")
1542                             .arg("-var").arg("var.CargoDir")
1543                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1544                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1545             build.run(Command::new(&heat)
1546                             .current_dir(&exe)
1547                             .arg("dir")
1548                             .arg("rust-std")
1549                             .args(&heat_flags)
1550                             .arg("-cg").arg("StdGroup")
1551                             .arg("-dr").arg("Std")
1552                             .arg("-var").arg("var.StdDir")
1553                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1554             if rls_installer.is_some() {
1555                 build.run(Command::new(&heat)
1556                                 .current_dir(&exe)
1557                                 .arg("dir")
1558                                 .arg("rls")
1559                                 .args(&heat_flags)
1560                                 .arg("-cg").arg("RlsGroup")
1561                                 .arg("-dr").arg("Rls")
1562                                 .arg("-var").arg("var.RlsDir")
1563                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1564                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1565             }
1566             build.run(Command::new(&heat)
1567                             .current_dir(&exe)
1568                             .arg("dir")
1569                             .arg("rust-analysis")
1570                             .args(&heat_flags)
1571                             .arg("-cg").arg("AnalysisGroup")
1572                             .arg("-dr").arg("Analysis")
1573                             .arg("-var").arg("var.AnalysisDir")
1574                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1575                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1576             if target.contains("windows-gnu") {
1577                 build.run(Command::new(&heat)
1578                                 .current_dir(&exe)
1579                                 .arg("dir")
1580                                 .arg("rust-mingw")
1581                                 .args(&heat_flags)
1582                                 .arg("-cg").arg("GccGroup")
1583                                 .arg("-dr").arg("Gcc")
1584                                 .arg("-var").arg("var.GccDir")
1585                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1586             }
1587
1588             let candle = |input: &Path| {
1589                 let output = exe.join(input.file_stem().unwrap())
1590                                 .with_extension("wixobj");
1591                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1592                 let mut cmd = Command::new(&candle);
1593                 cmd.current_dir(&exe)
1594                     .arg("-nologo")
1595                     .arg("-dRustcDir=rustc")
1596                     .arg("-dDocsDir=rust-docs")
1597                     .arg("-dCargoDir=cargo")
1598                     .arg("-dStdDir=rust-std")
1599                     .arg("-dAnalysisDir=rust-analysis")
1600                     .arg("-arch").arg(&arch)
1601                     .arg("-out").arg(&output)
1602                     .arg(&input);
1603                 add_env(build, &mut cmd, target);
1604
1605                 if rls_installer.is_some() {
1606                     cmd.arg("-dRlsDir=rls");
1607                 }
1608                 if target.contains("windows-gnu") {
1609                     cmd.arg("-dGccDir=rust-mingw");
1610                 }
1611                 build.run(&mut cmd);
1612             };
1613             candle(&xform(&etc.join("msi/rust.wxs")));
1614             candle(&etc.join("msi/ui.wxs"));
1615             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1616             candle("RustcGroup.wxs".as_ref());
1617             candle("DocsGroup.wxs".as_ref());
1618             candle("CargoGroup.wxs".as_ref());
1619             candle("StdGroup.wxs".as_ref());
1620             if rls_installer.is_some() {
1621                 candle("RlsGroup.wxs".as_ref());
1622             }
1623             candle("AnalysisGroup.wxs".as_ref());
1624
1625             if target.contains("windows-gnu") {
1626                 candle("GccGroup.wxs".as_ref());
1627             }
1628
1629             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1630             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1631             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1632
1633             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1634             let mut cmd = Command::new(&light);
1635             cmd.arg("-nologo")
1636                 .arg("-ext").arg("WixUIExtension")
1637                 .arg("-ext").arg("WixUtilExtension")
1638                 .arg("-out").arg(exe.join(&filename))
1639                 .arg("rust.wixobj")
1640                 .arg("ui.wixobj")
1641                 .arg("rustwelcomedlg.wixobj")
1642                 .arg("RustcGroup.wixobj")
1643                 .arg("DocsGroup.wixobj")
1644                 .arg("CargoGroup.wixobj")
1645                 .arg("StdGroup.wixobj")
1646                 .arg("AnalysisGroup.wixobj")
1647                 .current_dir(&exe);
1648
1649             if rls_installer.is_some() {
1650                 cmd.arg("RlsGroup.wixobj");
1651             }
1652
1653             if target.contains("windows-gnu") {
1654                 cmd.arg("GccGroup.wixobj");
1655             }
1656             // ICE57 wrongly complains about the shortcuts
1657             cmd.arg("-sice:ICE57");
1658
1659             build.run(&mut cmd);
1660
1661             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1662         }
1663     }
1664 }
1665
1666 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1667     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1668     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1669        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1670        .env("CFG_RELEASE", build.rust_release())
1671        .env("CFG_VER_MAJOR", parts.next().unwrap())
1672        .env("CFG_VER_MINOR", parts.next().unwrap())
1673        .env("CFG_VER_PATCH", parts.next().unwrap())
1674        .env("CFG_VER_BUILD", "0") // just needed to build
1675        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1676        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1677        .env("CFG_BUILD", target)
1678        .env("CFG_CHANNEL", &build.config.channel);
1679
1680     if target.contains("windows-gnu") {
1681        cmd.env("CFG_MINGW", "1")
1682           .env("CFG_ABI", "GNU");
1683     } else {
1684        cmd.env("CFG_MINGW", "0")
1685           .env("CFG_ABI", "MSVC");
1686     }
1687
1688     if target.contains("x86_64") {
1689        cmd.env("CFG_PLATFORM", "x64");
1690     } else {
1691        cmd.env("CFG_PLATFORM", "x86");
1692     }
1693 }
1694
1695 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1696 pub struct HashSign;
1697
1698 impl Step for HashSign {
1699     type Output = ();
1700     const ONLY_BUILD_TARGETS: bool = true;
1701     const ONLY_HOSTS: bool = true;
1702     const ONLY_BUILD: bool = true;
1703
1704     fn should_run(run: ShouldRun) -> ShouldRun {
1705         run.path("hash-and-sign")
1706     }
1707
1708     fn make_run(run: RunConfig) {
1709         run.builder.ensure(HashSign);
1710     }
1711
1712     fn run(self, builder: &Builder) {
1713         let build = builder.build;
1714         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1715         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1716             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1717         });
1718         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1719             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1720         });
1721         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1722             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1723         });
1724         let mut pass = String::new();
1725         t!(t!(File::open(&file)).read_to_string(&mut pass));
1726
1727         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1728
1729         cmd.arg(sign);
1730         cmd.arg(distdir(build));
1731         cmd.arg(today.trim());
1732         cmd.arg(build.rust_package_vers());
1733         cmd.arg(build.package_vers(&build.release_num("cargo")));
1734         cmd.arg(build.package_vers(&build.release_num("rls")));
1735         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1736         cmd.arg(addr);
1737
1738         t!(fs::create_dir_all(distdir(build)));
1739
1740         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1741         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1742         let status = t!(child.wait());
1743         assert!(status.success());
1744     }
1745 }