src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Man pages
 447             t!(fs::create_dir_all(image.join("share/man/man1")));
 448             let man_src = build.src.join("src/doc/man");
 449             let man_dst = image.join("share/man/man1");
 450             let month_year = t!(time::strftime("%B %Y", &time::now()));
 451             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 452             // to hardlink, and we don't want to edit the source templates
 453             for entry_result in t!(fs::read_dir(man_src)) {
 454                 let file_entry = t!(entry_result);
 455                 let page_src = file_entry.path();
 456                 let page_dst = man_dst.join(file_entry.file_name());
 457                 t!(fs::copy(&page_src, &page_dst));
 458                 // template in month/year and version number
 459                 replace_in_file(&page_dst,
 460                                 &[("<INSERT DATE HERE>", &month_year),
 461                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 462             }
 463
 464             // Debugger scripts
 465             builder.ensure(DebuggerScripts {
 466                 sysroot: INTERNER.intern_path(image.to_owned()),
 467                 host,
 468             });
 469
 470             // Misc license info
 471             let cp = |file: &str| {
 472                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 473             };
 474             cp("COPYRIGHT");
 475             cp("LICENSE-APACHE");
 476             cp("LICENSE-MIT");
 477             cp("README.md");
 478         }
 479     }
 480 }
 481
 482 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 483 pub struct DebuggerScripts {
 484     pub sysroot: Interned<PathBuf>,
 485     pub host: Interned<String>,
 486 }
 487
 488 impl Step for DebuggerScripts {
 489     type Output = ();
 490
 491     fn should_run(run: ShouldRun) -> ShouldRun {
 492         run.path("src/lldb_batchmode.py")
 493     }
 494
 495     fn make_run(run: RunConfig) {
 496         run.builder.ensure(DebuggerScripts {
 497             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 498             host: run.target,
 499         });
 500     }
 501
 502     /// Copies debugger scripts for `target` into the `sysroot` specified.
 503     fn run(self, builder: &Builder) {
 504         let build = builder.build;
 505         let host = self.host;
 506         let sysroot = self.sysroot;
 507         let dst = sysroot.join("lib/rustlib/etc");
 508         t!(fs::create_dir_all(&dst));
 509         let cp_debugger_script = |file: &str| {
 510             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 511         };
 512         if host.contains("windows-msvc") {
 513             // windbg debugger scripts
 514             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 515                 0o755);
 516
 517             cp_debugger_script("natvis/intrinsic.natvis");
 518             cp_debugger_script("natvis/liballoc.natvis");
 519             cp_debugger_script("natvis/libcore.natvis");
 520         } else {
 521             cp_debugger_script("debugger_pretty_printers_common.py");
 522
 523             // gdb debugger scripts
 524             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 525                     0o755);
 526
 527             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 528             cp_debugger_script("gdb_rust_pretty_printing.py");
 529
 530             // lldb debugger scripts
 531             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 532                     0o755);
 533
 534             cp_debugger_script("lldb_rust_formatters.py");
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: Interned<String>,
 543 }
 544
 545 impl Step for Std {
 546     type Output = PathBuf;
 547     const DEFAULT: bool = true;
 548     const ONLY_BUILD_TARGETS: bool = true;
 549
 550     fn should_run(run: ShouldRun) -> ShouldRun {
 551         run.path("src/libstd")
 552     }
 553
 554     fn make_run(run: RunConfig) {
 555         run.builder.ensure(Std {
 556             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 557             target: run.target,
 558         });
 559     }
 560
 561     fn run(self, builder: &Builder) -> PathBuf {
 562         let build = builder.build;
 563         let compiler = self.compiler;
 564         let target = self.target;
 565
 566         let name = pkgname(build, "rust-std");
 567         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 568
 569         // The only true set of target libraries came from the build triple, so
 570         // let's reduce redundant work by only producing archives from that host.
 571         if compiler.host != build.build {
 572             println!("\tskipping, not a build host");
 573             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 574         }
 575
 576         // We want to package up as many target libraries as possible
 577         // for the `rust-std` package, so if this is a host target we
 578         // depend on librustc and otherwise we just depend on libtest.
 579         if build.hosts.iter().any(|t| t == target) {
 580             builder.ensure(compile::Rustc { compiler, target });
 581         } else {
 582             builder.ensure(compile::Test { compiler, target });
 583         }
 584
 585         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 586         let _ = fs::remove_dir_all(&image);
 587
 588         let dst = image.join("lib/rustlib").join(target);
 589         t!(fs::create_dir_all(&dst));
 590         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 591         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 592         cp_filtered(&src, &dst, &|path| {
 593             path.file_name().and_then(|s| s.to_str()) != Some("codegen-backends")
 594         });
 595
 596         let mut cmd = rust_installer(builder);
 597         cmd.arg("generate")
 598            .arg("--product-name=Rust")
 599            .arg("--rel-manifest-dir=rustlib")
 600            .arg("--success-message=std-is-standing-at-the-ready.")
 601            .arg("--image-dir").arg(&image)
 602            .arg("--work-dir").arg(&tmpdir(build))
 603            .arg("--output-dir").arg(&distdir(build))
 604            .arg(format!("--package-name={}-{}", name, target))
 605            .arg(format!("--component-name=rust-std-{}", target))
 606            .arg("--legacy-manifest-dirs=rustlib,cargo");
 607         build.run(&mut cmd);
 608         t!(fs::remove_dir_all(&image));
 609         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 610     }
 611 }
 612
 613 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 614 pub struct Analysis {
 615     pub compiler: Compiler,
 616     pub target: Interned<String>,
 617 }
 618
 619 impl Step for Analysis {
 620     type Output = PathBuf;
 621     const DEFAULT: bool = true;
 622     const ONLY_BUILD_TARGETS: bool = true;
 623
 624     fn should_run(run: ShouldRun) -> ShouldRun {
 625         let builder = run.builder;
 626         run.path("analysis").default_condition(builder.build.config.extended)
 627     }
 628
 629     fn make_run(run: RunConfig) {
 630         run.builder.ensure(Analysis {
 631             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 632             target: run.target,
 633         });
 634     }
 635
 636     /// Creates a tarball of save-analysis metadata, if available.
 637     fn run(self, builder: &Builder) -> PathBuf {
 638         let build = builder.build;
 639         let compiler = self.compiler;
 640         let target = self.target;
 641         assert!(build.config.extended);
 642         println!("Dist analysis");
 643         let name = pkgname(build, "rust-analysis");
 644
 645         if &compiler.host != build.build {
 646             println!("\tskipping, not a build host");
 647             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 648         }
 649
 650         builder.ensure(Std { compiler, target });
 651
 652         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 653         // stage2 artifacts is simply copied from stage1 in that case.
 654         let compiler = if build.force_use_stage1(compiler, target) {
 655             builder.compiler(1, compiler.host)
 656         } else {
 657             compiler.clone()
 658         };
 659
 660         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 661
 662         let src = build.stage_out(compiler, Mode::Libstd)
 663             .join(target).join(build.cargo_dir()).join("deps");
 664
 665         let image_src = src.join("save-analysis");
 666         let dst = image.join("lib/rustlib").join(target).join("analysis");
 667         t!(fs::create_dir_all(&dst));
 668         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 669         cp_r(&image_src, &dst);
 670
 671         let mut cmd = rust_installer(builder);
 672         cmd.arg("generate")
 673            .arg("--product-name=Rust")
 674            .arg("--rel-manifest-dir=rustlib")
 675            .arg("--success-message=save-analysis-saved.")
 676            .arg("--image-dir").arg(&image)
 677            .arg("--work-dir").arg(&tmpdir(build))
 678            .arg("--output-dir").arg(&distdir(build))
 679            .arg(format!("--package-name={}-{}", name, target))
 680            .arg(format!("--component-name=rust-analysis-{}", target))
 681            .arg("--legacy-manifest-dirs=rustlib,cargo");
 682         build.run(&mut cmd);
 683         t!(fs::remove_dir_all(&image));
 684         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 685     }
 686 }
 687
 688 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 689     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 690         let spath = match path.to_str() {
 691             Some(path) => path,
 692             None => return false,
 693         };
 694         if spath.ends_with("~") || spath.ends_with(".pyc") {
 695             return false
 696         }
 697         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 698             (spath.ends_with(".ll") ||
 699              spath.ends_with(".td") ||
 700              spath.ends_with(".s")) {
 701             return false
 702         }
 703         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 704             return false
 705         }
 706
 707         let full_path = Path::new(dir).join(path);
 708         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 709             return false;
 710         }
 711
 712         let excludes = [
 713             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 714             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 715             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 716             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 717         ];
 718         !path.iter()
 719              .map(|s| s.to_str().unwrap())
 720              .any(|s| excludes.contains(&s))
 721     }
 722
 723     // Copy the directories using our filter
 724     for item in src_dirs {
 725         let dst = &dst_dir.join(item);
 726         t!(fs::create_dir_all(dst));
 727         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 728     }
 729 }
 730
 731 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 732 pub struct Src;
 733
 734 impl Step for Src {
 735     /// The output path of the src installer tarball
 736     type Output = PathBuf;
 737     const DEFAULT: bool = true;
 738     const ONLY_HOSTS: bool = true;
 739     const ONLY_BUILD_TARGETS: bool = true;
 740     const ONLY_BUILD: bool = true;
 741
 742     fn should_run(run: ShouldRun) -> ShouldRun {
 743         run.path("src")
 744     }
 745
 746     fn make_run(run: RunConfig) {
 747         run.builder.ensure(Src);
 748     }
 749
 750     /// Creates the `rust-src` installer component
 751     fn run(self, builder: &Builder) -> PathBuf {
 752         let build = builder.build;
 753         println!("Dist src");
 754
 755         let name = pkgname(build, "rust-src");
 756         let image = tmpdir(build).join(format!("{}-image", name));
 757         let _ = fs::remove_dir_all(&image);
 758
 759         let dst = image.join("lib/rustlib/src");
 760         let dst_src = dst.join("rust");
 761         t!(fs::create_dir_all(&dst_src));
 762
 763         let src_files = [
 764             "src/Cargo.lock",
 765         ];
 766         // This is the reduced set of paths which will become the rust-src component
 767         // (essentially libstd and all of its path dependencies)
 768         let std_src_dirs = [
 769             "src/build_helper",
 770             "src/dlmalloc",
 771             "src/liballoc",
 772             "src/liballoc_jemalloc",
 773             "src/liballoc_system",
 774             "src/libbacktrace",
 775             "src/libcompiler_builtins",
 776             "src/libcore",
 777             "src/liblibc",
 778             "src/libpanic_abort",
 779             "src/libpanic_unwind",
 780             "src/librustc_asan",
 781             "src/librustc_lsan",
 782             "src/librustc_msan",
 783             "src/librustc_tsan",
 784             "src/libstd",
 785             "src/libstd_unicode",
 786             "src/libunwind",
 787             "src/rustc/compiler_builtins_shim",
 788             "src/rustc/libc_shim",
 789             "src/rustc/dlmalloc_shim",
 790             "src/libtest",
 791             "src/libterm",
 792             "src/jemalloc",
 793             "src/libprofiler_builtins",
 794         ];
 795         let std_src_dirs_exclude = [
 796             "src/libcompiler_builtins/compiler-rt/test",
 797             "src/jemalloc/test/unit",
 798         ];
 799
 800         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 801         for file in src_files.iter() {
 802             copy(&build.src.join(file), &dst_src.join(file));
 803         }
 804
 805         // Create source tarball in rust-installer format
 806         let mut cmd = rust_installer(builder);
 807         cmd.arg("generate")
 808            .arg("--product-name=Rust")
 809            .arg("--rel-manifest-dir=rustlib")
 810            .arg("--success-message=Awesome-Source.")
 811            .arg("--image-dir").arg(&image)
 812            .arg("--work-dir").arg(&tmpdir(build))
 813            .arg("--output-dir").arg(&distdir(build))
 814            .arg(format!("--package-name={}", name))
 815            .arg("--component-name=rust-src")
 816            .arg("--legacy-manifest-dirs=rustlib,cargo");
 817         build.run(&mut cmd);
 818
 819         t!(fs::remove_dir_all(&image));
 820         distdir(build).join(&format!("{}.tar.gz", name))
 821     }
 822 }
 823
 824 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 825
 826 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 827 pub struct PlainSourceTarball;
 828
 829 impl Step for PlainSourceTarball {
 830     /// Produces the location of the tarball generated
 831     type Output = PathBuf;
 832     const DEFAULT: bool = true;
 833     const ONLY_HOSTS: bool = true;
 834     const ONLY_BUILD_TARGETS: bool = true;
 835     const ONLY_BUILD: bool = true;
 836
 837     fn should_run(run: ShouldRun) -> ShouldRun {
 838         let builder = run.builder;
 839         run.path("src").default_condition(builder.config.rust_dist_src)
 840     }
 841
 842     fn make_run(run: RunConfig) {
 843         run.builder.ensure(PlainSourceTarball);
 844     }
 845
 846     /// Creates the plain source tarball
 847     fn run(self, builder: &Builder) -> PathBuf {
 848         let build = builder.build;
 849         println!("Create plain source tarball");
 850
 851         // Make sure that the root folder of tarball has the correct name
 852         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 853         let plain_dst_src = tmpdir(build).join(&plain_name);
 854         let _ = fs::remove_dir_all(&plain_dst_src);
 855         t!(fs::create_dir_all(&plain_dst_src));
 856
 857         // This is the set of root paths which will become part of the source package
 858         let src_files = [
 859             "COPYRIGHT",
 860             "LICENSE-APACHE",
 861             "LICENSE-MIT",
 862             "CONTRIBUTING.md",
 863             "README.md",
 864             "RELEASES.md",
 865             "configure",
 866             "x.py",
 867             "config.toml.example",
 868         ];
 869         let src_dirs = [
 870             "src",
 871         ];
 872
 873         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 874
 875         // Copy the files normally
 876         for item in &src_files {
 877             copy(&build.src.join(item), &plain_dst_src.join(item));
 878         }
 879
 880         // Create the version file
 881         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 882         if let Some(sha) = build.rust_sha() {
 883             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 884         }
 885
 886         // If we're building from git sources, we need to vendor a complete distribution.
 887         if build.rust_info.is_git() {
 888             // Get cargo-vendor installed, if it isn't already.
 889             let mut has_cargo_vendor = false;
 890             let mut cmd = Command::new(&build.initial_cargo);
 891             for line in output(cmd.arg("install").arg("--list")).lines() {
 892                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 893             }
 894             if !has_cargo_vendor {
 895                 let mut cmd = Command::new(&build.initial_cargo);
 896                 cmd.arg("install")
 897                    .arg("--force")
 898                    .arg("--debug")
 899                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 900                    .arg("cargo-vendor")
 901                    .env("RUSTC", &build.initial_rustc);
 902                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 903                     builder.ensure(native::Openssl {
 904                         target: build.config.build,
 905                     });
 906                     cmd.env("OPENSSL_DIR", dir);
 907                 }
 908                 build.run(&mut cmd);
 909             }
 910
 911             // Vendor all Cargo dependencies
 912             let mut cmd = Command::new(&build.initial_cargo);
 913             cmd.arg("vendor")
 914                .current_dir(&plain_dst_src.join("src"));
 915             build.run(&mut cmd);
 916         }
 917
 918         // Create plain source tarball
 919         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 920         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 921         tarball.set_extension(""); // strip .gz
 922         tarball.set_extension(""); // strip .tar
 923         if let Some(dir) = tarball.parent() {
 924             t!(fs::create_dir_all(dir));
 925         }
 926         println!("running installer");
 927         let mut cmd = rust_installer(builder);
 928         cmd.arg("tarball")
 929            .arg("--input").arg(&plain_name)
 930            .arg("--output").arg(&tarball)
 931            .arg("--work-dir=.")
 932            .current_dir(tmpdir(build));
 933         build.run(&mut cmd);
 934         distdir(build).join(&format!("{}.tar.gz", plain_name))
 935     }
 936 }
 937
 938 fn install(src: &Path, dstdir: &Path, perms: u32) {
 939     let dst = dstdir.join(src.file_name().unwrap());
 940     t!(fs::create_dir_all(dstdir));
 941     drop(fs::remove_file(&dst));
 942     {
 943         let mut s = t!(fs::File::open(&src));
 944         let mut d = t!(fs::File::create(&dst));
 945         io::copy(&mut s, &mut d).expect("failed to copy");
 946     }
 947     chmod(&dst, perms);
 948 }
 949
 950 #[cfg(unix)]
 951 fn chmod(path: &Path, perms: u32) {
 952     use std::os::unix::fs::*;
 953     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 954 }
 955 #[cfg(windows)]
 956 fn chmod(_path: &Path, _perms: u32) {}
 957
 958 // We have to run a few shell scripts, which choke quite a bit on both `\`
 959 // characters and on `C:\` paths, so normalize both of them away.
 960 pub fn sanitize_sh(path: &Path) -> String {
 961     let path = path.to_str().unwrap().replace("\\", "/");
 962     return change_drive(&path).unwrap_or(path);
 963
 964     fn change_drive(s: &str) -> Option<String> {
 965         let mut ch = s.chars();
 966         let drive = ch.next().unwrap_or('C');
 967         if ch.next() != Some(':') {
 968             return None
 969         }
 970         if ch.next() != Some('/') {
 971             return None
 972         }
 973         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 974     }
 975 }
 976
 977 fn write_file(path: &Path, data: &[u8]) {
 978     let mut vf = t!(fs::File::create(path));
 979     t!(vf.write_all(data));
 980 }
 981
 982 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 983 pub struct Cargo {
 984     pub stage: u32,
 985     pub target: Interned<String>,
 986 }
 987
 988 impl Step for Cargo {
 989     type Output = PathBuf;
 990     const ONLY_BUILD_TARGETS: bool = true;
 991     const ONLY_HOSTS: bool = true;
 992
 993     fn should_run(run: ShouldRun) -> ShouldRun {
 994         run.path("cargo")
 995     }
 996
 997     fn make_run(run: RunConfig) {
 998         run.builder.ensure(Cargo {
 999             stage: run.builder.top_stage,
1000             target: run.target,
1001         });
1002     }
1003
1004     fn run(self, builder: &Builder) -> PathBuf {
1005         let build = builder.build;
1006         let stage = self.stage;
1007         let target = self.target;
1008
1009         println!("Dist cargo stage{} ({})", stage, target);
1010         let src = build.src.join("src/tools/cargo");
1011         let etc = src.join("src/etc");
1012         let release_num = build.release_num("cargo");
1013         let name = pkgname(build, "cargo");
1014         let version = builder.cargo_info.version(build, &release_num);
1015
1016         let tmp = tmpdir(build);
1017         let image = tmp.join("cargo-image");
1018         drop(fs::remove_dir_all(&image));
1019         t!(fs::create_dir_all(&image));
1020
1021         // Prepare the image directory
1022         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1023         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1024         let cargo = builder.ensure(tool::Cargo {
1025             compiler: builder.compiler(stage, build.build),
1026             target
1027         });
1028         install(&cargo, &image.join("bin"), 0o755);
1029         for man in t!(etc.join("man").read_dir()) {
1030             let man = t!(man);
1031             install(&man.path(), &image.join("share/man/man1"), 0o644);
1032         }
1033         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1034         copy(&etc.join("cargo.bashcomp.sh"),
1035              &image.join("etc/bash_completion.d/cargo"));
1036         let doc = image.join("share/doc/cargo");
1037         install(&src.join("README.md"), &doc, 0o644);
1038         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1039         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1040         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1041
1042         // Prepare the overlay
1043         let overlay = tmp.join("cargo-overlay");
1044         drop(fs::remove_dir_all(&overlay));
1045         t!(fs::create_dir_all(&overlay));
1046         install(&src.join("README.md"), &overlay, 0o644);
1047         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1048         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1049         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1050         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1051
1052         // Generate the installer tarball
1053         let mut cmd = rust_installer(builder);
1054         cmd.arg("generate")
1055            .arg("--product-name=Rust")
1056            .arg("--rel-manifest-dir=rustlib")
1057            .arg("--success-message=Rust-is-ready-to-roll.")
1058            .arg("--image-dir").arg(&image)
1059            .arg("--work-dir").arg(&tmpdir(build))
1060            .arg("--output-dir").arg(&distdir(build))
1061            .arg("--non-installed-overlay").arg(&overlay)
1062            .arg(format!("--package-name={}-{}", name, target))
1063            .arg("--component-name=cargo")
1064            .arg("--legacy-manifest-dirs=rustlib,cargo");
1065         build.run(&mut cmd);
1066         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1067     }
1068 }
1069
1070 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1071 pub struct Rls {
1072     pub stage: u32,
1073     pub target: Interned<String>,
1074 }
1075
1076 impl Step for Rls {
1077     type Output = Option<PathBuf>;
1078     const ONLY_BUILD_TARGETS: bool = true;
1079     const ONLY_HOSTS: bool = true;
1080
1081     fn should_run(run: ShouldRun) -> ShouldRun {
1082         run.path("rls")
1083     }
1084
1085     fn make_run(run: RunConfig) {
1086         run.builder.ensure(Rls {
1087             stage: run.builder.top_stage,
1088             target: run.target,
1089         });
1090     }
1091
1092     fn run(self, builder: &Builder) -> Option<PathBuf> {
1093         let build = builder.build;
1094         let stage = self.stage;
1095         let target = self.target;
1096         assert!(build.config.extended);
1097
1098         println!("Dist RLS stage{} ({})", stage, target);
1099         let src = build.src.join("src/tools/rls");
1100         let release_num = build.release_num("rls");
1101         let name = pkgname(build, "rls");
1102         let version = build.rls_info.version(build, &release_num);
1103
1104         let tmp = tmpdir(build);
1105         let image = tmp.join("rls-image");
1106         drop(fs::remove_dir_all(&image));
1107         t!(fs::create_dir_all(&image));
1108
1109         // Prepare the image directory
1110         // We expect RLS to build, because we've exited this step above if tool
1111         // state for RLS isn't testing.
1112         let rls = builder.ensure(tool::Rls {
1113             compiler: builder.compiler(stage, build.build),
1114             target
1115         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1116
1117         install(&rls, &image.join("bin"), 0o755);
1118         let doc = image.join("share/doc/rls");
1119         install(&src.join("README.md"), &doc, 0o644);
1120         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1121         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1122
1123         // Prepare the overlay
1124         let overlay = tmp.join("rls-overlay");
1125         drop(fs::remove_dir_all(&overlay));
1126         t!(fs::create_dir_all(&overlay));
1127         install(&src.join("README.md"), &overlay, 0o644);
1128         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1129         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1130         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1131
1132         // Generate the installer tarball
1133         let mut cmd = rust_installer(builder);
1134         cmd.arg("generate")
1135            .arg("--product-name=Rust")
1136            .arg("--rel-manifest-dir=rustlib")
1137            .arg("--success-message=RLS-ready-to-serve.")
1138            .arg("--image-dir").arg(&image)
1139            .arg("--work-dir").arg(&tmpdir(build))
1140            .arg("--output-dir").arg(&distdir(build))
1141            .arg("--non-installed-overlay").arg(&overlay)
1142            .arg(format!("--package-name={}-{}", name, target))
1143            .arg("--legacy-manifest-dirs=rustlib,cargo")
1144            .arg("--component-name=rls-preview");
1145
1146         build.run(&mut cmd);
1147         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1148     }
1149 }
1150
1151
1152 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1153 pub struct Rustfmt {
1154     pub stage: u32,
1155     pub target: Interned<String>,
1156 }
1157
1158 impl Step for Rustfmt {
1159     type Output = Option<PathBuf>;
1160     const ONLY_BUILD_TARGETS: bool = true;
1161     const ONLY_HOSTS: bool = true;
1162
1163     fn should_run(run: ShouldRun) -> ShouldRun {
1164         run.path("rustfmt")
1165     }
1166
1167     fn make_run(run: RunConfig) {
1168         run.builder.ensure(Rustfmt {
1169             stage: run.builder.top_stage,
1170             target: run.target,
1171         });
1172     }
1173
1174     fn run(self, builder: &Builder) -> Option<PathBuf> {
1175         let build = builder.build;
1176         let stage = self.stage;
1177         let target = self.target;
1178         assert!(build.config.extended);
1179
1180         println!("Dist Rustfmt stage{} ({})", stage, target);
1181         let src = build.src.join("src/tools/rustfmt");
1182         let release_num = build.release_num("rustfmt");
1183         let name = pkgname(build, "rustfmt");
1184         let version = build.rustfmt_info.version(build, &release_num);
1185
1186         let tmp = tmpdir(build);
1187         let image = tmp.join("rustfmt-image");
1188         drop(fs::remove_dir_all(&image));
1189         t!(fs::create_dir_all(&image));
1190
1191         // Prepare the image directory
1192         let rustfmt = builder.ensure(tool::Rustfmt {
1193             compiler: builder.compiler(stage, build.build),
1194             target
1195         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1196         let cargofmt = builder.ensure(tool::Cargofmt {
1197             compiler: builder.compiler(stage, build.build),
1198             target
1199         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1200
1201         install(&rustfmt, &image.join("bin"), 0o755);
1202         install(&cargofmt, &image.join("bin"), 0o755);
1203         let doc = image.join("share/doc/rustfmt");
1204         install(&src.join("README.md"), &doc, 0o644);
1205         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1206         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1207
1208         // Prepare the overlay
1209         let overlay = tmp.join("rustfmt-overlay");
1210         drop(fs::remove_dir_all(&overlay));
1211         t!(fs::create_dir_all(&overlay));
1212         install(&src.join("README.md"), &overlay, 0o644);
1213         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1214         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1215         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1216
1217         // Generate the installer tarball
1218         let mut cmd = rust_installer(builder);
1219         cmd.arg("generate")
1220            .arg("--product-name=Rust")
1221            .arg("--rel-manifest-dir=rustlib")
1222            .arg("--success-message=rustfmt-ready-to-fmt.")
1223            .arg("--image-dir").arg(&image)
1224            .arg("--work-dir").arg(&tmpdir(build))
1225            .arg("--output-dir").arg(&distdir(build))
1226            .arg("--non-installed-overlay").arg(&overlay)
1227            .arg(format!("--package-name={}-{}", name, target))
1228            .arg("--legacy-manifest-dirs=rustlib,cargo")
1229            .arg("--component-name=rustfmt-preview");
1230
1231         build.run(&mut cmd);
1232         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1233     }
1234 }
1235
1236 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1237 pub struct Extended {
1238     stage: u32,
1239     host: Interned<String>,
1240     target: Interned<String>,
1241 }
1242
1243 impl Step for Extended {
1244     type Output = ();
1245     const DEFAULT: bool = true;
1246     const ONLY_BUILD_TARGETS: bool = true;
1247     const ONLY_HOSTS: bool = true;
1248
1249     fn should_run(run: ShouldRun) -> ShouldRun {
1250         let builder = run.builder;
1251         run.path("extended").default_condition(builder.config.extended)
1252     }
1253
1254     fn make_run(run: RunConfig) {
1255         run.builder.ensure(Extended {
1256             stage: run.builder.top_stage,
1257             host: run.host,
1258             target: run.target,
1259         });
1260     }
1261
1262     /// Creates a combined installer for the specified target in the provided stage.
1263     fn run(self, builder: &Builder) {
1264         let build = builder.build;
1265         let stage = self.stage;
1266         let target = self.target;
1267
1268         println!("Dist extended stage{} ({})", stage, target);
1269
1270         let rustc_installer = builder.ensure(Rustc {
1271             compiler: builder.compiler(stage, target),
1272         });
1273         let cargo_installer = builder.ensure(Cargo { stage, target });
1274         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1275         let rls_installer = builder.ensure(Rls { stage, target });
1276         let mingw_installer = builder.ensure(Mingw { host: target });
1277         let analysis_installer = builder.ensure(Analysis {
1278             compiler: builder.compiler(stage, self.host),
1279             target
1280         });
1281
1282         let docs_installer = builder.ensure(Docs { stage, host: target, });
1283         let std_installer = builder.ensure(Std {
1284             compiler: builder.compiler(stage, self.host),
1285             target,
1286         });
1287
1288         let tmp = tmpdir(build);
1289         let overlay = tmp.join("extended-overlay");
1290         let etc = build.src.join("src/etc/installer");
1291         let work = tmp.join("work");
1292
1293         let _ = fs::remove_dir_all(&overlay);
1294         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1295         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1296         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1297         let version = build.rust_version();
1298         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1299         if let Some(sha) = build.rust_sha() {
1300             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1301         }
1302         install(&etc.join("README.md"), &overlay, 0o644);
1303
1304         // When rust-std package split from rustc, we needed to ensure that during
1305         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1306         // the std files during uninstall. To do this ensure that rustc comes
1307         // before rust-std in the list below.
1308         let mut tarballs = Vec::new();
1309         tarballs.push(rustc_installer);
1310         tarballs.push(cargo_installer);
1311         tarballs.extend(rls_installer.clone());
1312         tarballs.extend(rustfmt_installer.clone());
1313         tarballs.push(analysis_installer);
1314         tarballs.push(std_installer);
1315         if build.config.docs {
1316             tarballs.push(docs_installer);
1317         }
1318         if target.contains("pc-windows-gnu") {
1319             tarballs.push(mingw_installer.unwrap());
1320         }
1321         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1322         for tarball in &tarballs[1..] {
1323             input_tarballs.push(",");
1324             input_tarballs.push(tarball);
1325         }
1326
1327         let mut cmd = rust_installer(builder);
1328         cmd.arg("combine")
1329             .arg("--product-name=Rust")
1330             .arg("--rel-manifest-dir=rustlib")
1331             .arg("--success-message=Rust-is-ready-to-roll.")
1332             .arg("--work-dir").arg(&work)
1333             .arg("--output-dir").arg(&distdir(build))
1334             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1335             .arg("--legacy-manifest-dirs=rustlib,cargo")
1336             .arg("--input-tarballs").arg(input_tarballs)
1337             .arg("--non-installed-overlay").arg(&overlay);
1338         build.run(&mut cmd);
1339
1340         let mut license = String::new();
1341         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1342         license.push_str("\n");
1343         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1344         license.push_str("\n");
1345         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1346
1347         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1348         let mut rtf = rtf.to_string();
1349         rtf.push_str("\n");
1350         for line in license.lines() {
1351             rtf.push_str(line);
1352             rtf.push_str("\\line ");
1353         }
1354         rtf.push_str("}");
1355
1356         fn filter(contents: &str, marker: &str) -> String {
1357             let start = format!("tool-{}-start", marker);
1358             let end = format!("tool-{}-end", marker);
1359             let mut lines = Vec::new();
1360             let mut omitted = false;
1361             for line in contents.lines() {
1362                 if line.contains(&start) {
1363                     omitted = true;
1364                 } else if line.contains(&end) {
1365                     omitted = false;
1366                 } else if !omitted {
1367                     lines.push(line);
1368                 }
1369             }
1370
1371             lines.join("\n")
1372         }
1373
1374         let xform = |p: &Path| {
1375             let mut contents = String::new();
1376             t!(t!(File::open(p)).read_to_string(&mut contents));
1377             if rls_installer.is_none() {
1378                 contents = filter(&contents, "rls");
1379             }
1380             if rustfmt_installer.is_none() {
1381                 contents = filter(&contents, "rustfmt");
1382             }
1383             let ret = tmp.join(p.file_name().unwrap());
1384             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1385             return ret
1386         };
1387
1388         if target.contains("apple-darwin") {
1389             let pkg = tmp.join("pkg");
1390             let _ = fs::remove_dir_all(&pkg);
1391
1392             let pkgbuild = |component: &str| {
1393                 let mut cmd = Command::new("pkgbuild");
1394                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1395                     .arg("--scripts").arg(pkg.join(component))
1396                     .arg("--nopayload")
1397                     .arg(pkg.join(component).with_extension("pkg"));
1398                 build.run(&mut cmd);
1399             };
1400
1401             let prepare = |name: &str| {
1402                 t!(fs::create_dir_all(pkg.join(name)));
1403                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1404                         &pkg.join(name));
1405                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1406                 pkgbuild(name);
1407             };
1408             prepare("rustc");
1409             prepare("cargo");
1410             prepare("rust-docs");
1411             prepare("rust-std");
1412             prepare("rust-analysis");
1413
1414             if rls_installer.is_some() {
1415                 prepare("rls");
1416             }
1417
1418             // create an 'uninstall' package
1419             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1420             pkgbuild("uninstall");
1421
1422             t!(fs::create_dir_all(pkg.join("res")));
1423             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1424             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1425             let mut cmd = Command::new("productbuild");
1426             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1427                 .arg("--resources").arg(pkg.join("res"))
1428                 .arg(distdir(build).join(format!("{}-{}.pkg",
1429                                                     pkgname(build, "rust"),
1430                                                     target)))
1431                 .arg("--package-path").arg(&pkg);
1432             build.run(&mut cmd);
1433         }
1434
1435         if target.contains("windows") {
1436             let exe = tmp.join("exe");
1437             let _ = fs::remove_dir_all(&exe);
1438
1439             let prepare = |name: &str| {
1440                 t!(fs::create_dir_all(exe.join(name)));
1441                 let dir = if name == "rust-std" || name == "rust-analysis" {
1442                     format!("{}-{}", name, target)
1443                 } else if name == "rls" {
1444                     "rls-preview".to_string()
1445                 } else {
1446                     name.to_string()
1447                 };
1448                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1449                             .join(dir),
1450                         &exe.join(name));
1451                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1452             };
1453             prepare("rustc");
1454             prepare("cargo");
1455             prepare("rust-analysis");
1456             prepare("rust-docs");
1457             prepare("rust-std");
1458             if rls_installer.is_some() {
1459                 prepare("rls");
1460             }
1461             if target.contains("windows-gnu") {
1462                 prepare("rust-mingw");
1463             }
1464
1465             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1466             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1467             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1468             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1469             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1470
1471             // Generate exe installer
1472             let mut cmd = Command::new("iscc");
1473             cmd.arg("rust.iss")
1474                 .current_dir(&exe);
1475             if target.contains("windows-gnu") {
1476                 cmd.arg("/dMINGW");
1477             }
1478             add_env(build, &mut cmd, target);
1479             build.run(&mut cmd);
1480             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1481                     &distdir(build),
1482                     0o755);
1483
1484             // Generate msi installer
1485             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1486             let heat = wix.join("bin/heat.exe");
1487             let candle = wix.join("bin/candle.exe");
1488             let light = wix.join("bin/light.exe");
1489
1490             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1491             build.run(Command::new(&heat)
1492                             .current_dir(&exe)
1493                             .arg("dir")
1494                             .arg("rustc")
1495                             .args(&heat_flags)
1496                             .arg("-cg").arg("RustcGroup")
1497                             .arg("-dr").arg("Rustc")
1498                             .arg("-var").arg("var.RustcDir")
1499                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1500             build.run(Command::new(&heat)
1501                             .current_dir(&exe)
1502                             .arg("dir")
1503                             .arg("rust-docs")
1504                             .args(&heat_flags)
1505                             .arg("-cg").arg("DocsGroup")
1506                             .arg("-dr").arg("Docs")
1507                             .arg("-var").arg("var.DocsDir")
1508                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1509                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1510             build.run(Command::new(&heat)
1511                             .current_dir(&exe)
1512                             .arg("dir")
1513                             .arg("cargo")
1514                             .args(&heat_flags)
1515                             .arg("-cg").arg("CargoGroup")
1516                             .arg("-dr").arg("Cargo")
1517                             .arg("-var").arg("var.CargoDir")
1518                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1519                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1520             build.run(Command::new(&heat)
1521                             .current_dir(&exe)
1522                             .arg("dir")
1523                             .arg("rust-std")
1524                             .args(&heat_flags)
1525                             .arg("-cg").arg("StdGroup")
1526                             .arg("-dr").arg("Std")
1527                             .arg("-var").arg("var.StdDir")
1528                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1529             if rls_installer.is_some() {
1530                 build.run(Command::new(&heat)
1531                                 .current_dir(&exe)
1532                                 .arg("dir")
1533                                 .arg("rls")
1534                                 .args(&heat_flags)
1535                                 .arg("-cg").arg("RlsGroup")
1536                                 .arg("-dr").arg("Rls")
1537                                 .arg("-var").arg("var.RlsDir")
1538                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1539                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1540             }
1541             build.run(Command::new(&heat)
1542                             .current_dir(&exe)
1543                             .arg("dir")
1544                             .arg("rust-analysis")
1545                             .args(&heat_flags)
1546                             .arg("-cg").arg("AnalysisGroup")
1547                             .arg("-dr").arg("Analysis")
1548                             .arg("-var").arg("var.AnalysisDir")
1549                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1550                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1551             if target.contains("windows-gnu") {
1552                 build.run(Command::new(&heat)
1553                                 .current_dir(&exe)
1554                                 .arg("dir")
1555                                 .arg("rust-mingw")
1556                                 .args(&heat_flags)
1557                                 .arg("-cg").arg("GccGroup")
1558                                 .arg("-dr").arg("Gcc")
1559                                 .arg("-var").arg("var.GccDir")
1560                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1561             }
1562
1563             let candle = |input: &Path| {
1564                 let output = exe.join(input.file_stem().unwrap())
1565                                 .with_extension("wixobj");
1566                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1567                 let mut cmd = Command::new(&candle);
1568                 cmd.current_dir(&exe)
1569                     .arg("-nologo")
1570                     .arg("-dRustcDir=rustc")
1571                     .arg("-dDocsDir=rust-docs")
1572                     .arg("-dCargoDir=cargo")
1573                     .arg("-dStdDir=rust-std")
1574                     .arg("-dAnalysisDir=rust-analysis")
1575                     .arg("-arch").arg(&arch)
1576                     .arg("-out").arg(&output)
1577                     .arg(&input);
1578                 add_env(build, &mut cmd, target);
1579
1580                 if rls_installer.is_some() {
1581                     cmd.arg("-dRlsDir=rls");
1582                 }
1583                 if target.contains("windows-gnu") {
1584                     cmd.arg("-dGccDir=rust-mingw");
1585                 }
1586                 build.run(&mut cmd);
1587             };
1588             candle(&xform(&etc.join("msi/rust.wxs")));
1589             candle(&etc.join("msi/ui.wxs"));
1590             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1591             candle("RustcGroup.wxs".as_ref());
1592             candle("DocsGroup.wxs".as_ref());
1593             candle("CargoGroup.wxs".as_ref());
1594             candle("StdGroup.wxs".as_ref());
1595             if rls_installer.is_some() {
1596                 candle("RlsGroup.wxs".as_ref());
1597             }
1598             candle("AnalysisGroup.wxs".as_ref());
1599
1600             if target.contains("windows-gnu") {
1601                 candle("GccGroup.wxs".as_ref());
1602             }
1603
1604             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1605             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1606             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1607
1608             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1609             let mut cmd = Command::new(&light);
1610             cmd.arg("-nologo")
1611                 .arg("-ext").arg("WixUIExtension")
1612                 .arg("-ext").arg("WixUtilExtension")
1613                 .arg("-out").arg(exe.join(&filename))
1614                 .arg("rust.wixobj")
1615                 .arg("ui.wixobj")
1616                 .arg("rustwelcomedlg.wixobj")
1617                 .arg("RustcGroup.wixobj")
1618                 .arg("DocsGroup.wixobj")
1619                 .arg("CargoGroup.wixobj")
1620                 .arg("StdGroup.wixobj")
1621                 .arg("AnalysisGroup.wixobj")
1622                 .current_dir(&exe);
1623
1624             if rls_installer.is_some() {
1625                 cmd.arg("RlsGroup.wixobj");
1626             }
1627
1628             if target.contains("windows-gnu") {
1629                 cmd.arg("GccGroup.wixobj");
1630             }
1631             // ICE57 wrongly complains about the shortcuts
1632             cmd.arg("-sice:ICE57");
1633
1634             build.run(&mut cmd);
1635
1636             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1637         }
1638     }
1639 }
1640
1641 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1642     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1643     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1644        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1645        .env("CFG_RELEASE", build.rust_release())
1646        .env("CFG_VER_MAJOR", parts.next().unwrap())
1647        .env("CFG_VER_MINOR", parts.next().unwrap())
1648        .env("CFG_VER_PATCH", parts.next().unwrap())
1649        .env("CFG_VER_BUILD", "0") // just needed to build
1650        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1651        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1652        .env("CFG_BUILD", target)
1653        .env("CFG_CHANNEL", &build.config.channel);
1654
1655     if target.contains("windows-gnu") {
1656        cmd.env("CFG_MINGW", "1")
1657           .env("CFG_ABI", "GNU");
1658     } else {
1659        cmd.env("CFG_MINGW", "0")
1660           .env("CFG_ABI", "MSVC");
1661     }
1662
1663     if target.contains("x86_64") {
1664        cmd.env("CFG_PLATFORM", "x64");
1665     } else {
1666        cmd.env("CFG_PLATFORM", "x86");
1667     }
1668 }
1669
1670 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1671 pub struct HashSign;
1672
1673 impl Step for HashSign {
1674     type Output = ();
1675     const ONLY_BUILD_TARGETS: bool = true;
1676     const ONLY_HOSTS: bool = true;
1677     const ONLY_BUILD: bool = true;
1678
1679     fn should_run(run: ShouldRun) -> ShouldRun {
1680         run.path("hash-and-sign")
1681     }
1682
1683     fn make_run(run: RunConfig) {
1684         run.builder.ensure(HashSign);
1685     }
1686
1687     fn run(self, builder: &Builder) {
1688         let build = builder.build;
1689         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1690         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1691             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1692         });
1693         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1694             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1695         });
1696         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1697             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1698         });
1699         let mut pass = String::new();
1700         t!(t!(File::open(&file)).read_to_string(&mut pass));
1701
1702         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1703
1704         cmd.arg(sign);
1705         cmd.arg(distdir(build));
1706         cmd.arg(today.trim());
1707         cmd.arg(build.rust_package_vers());
1708         cmd.arg(build.package_vers(&build.release_num("cargo")));
1709         cmd.arg(build.package_vers(&build.release_num("rls")));
1710         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1711         cmd.arg(addr);
1712
1713         t!(fs::create_dir_all(distdir(build)));
1714
1715         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1716         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1717         let status = t!(child.wait());
1718         assert!(status.success());
1719     }
1720 }