src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Copy over lld if it's there
 447             if builder.config.lld_enabled {
 448                 let exe = exe("lld", &compiler.host);
 449                 let src = builder.sysroot_libdir(compiler, host)
 450                     .parent()
 451                     .unwrap()
 452                     .join("bin")
 453                     .join(&exe);
 454                 let dst = image.join("lib/rustlib")
 455                     .join(&*host)
 456                     .join("bin")
 457                     .join(&exe);
 458                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 459                 copy(&src, &dst);
 460             }
 461
 462             // Man pages
 463             t!(fs::create_dir_all(image.join("share/man/man1")));
 464             let man_src = build.src.join("src/doc/man");
 465             let man_dst = image.join("share/man/man1");
 466             let month_year = t!(time::strftime("%B %Y", &time::now()));
 467             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 468             // to hardlink, and we don't want to edit the source templates
 469             for entry_result in t!(fs::read_dir(man_src)) {
 470                 let file_entry = t!(entry_result);
 471                 let page_src = file_entry.path();
 472                 let page_dst = man_dst.join(file_entry.file_name());
 473                 t!(fs::copy(&page_src, &page_dst));
 474                 // template in month/year and version number
 475                 replace_in_file(&page_dst,
 476                                 &[("<INSERT DATE HERE>", &month_year),
 477                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 478             }
 479
 480             // Debugger scripts
 481             builder.ensure(DebuggerScripts {
 482                 sysroot: INTERNER.intern_path(image.to_owned()),
 483                 host,
 484             });
 485
 486             // Misc license info
 487             let cp = |file: &str| {
 488                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 489             };
 490             cp("COPYRIGHT");
 491             cp("LICENSE-APACHE");
 492             cp("LICENSE-MIT");
 493             cp("README.md");
 494         }
 495     }
 496 }
 497
 498 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 499 pub struct DebuggerScripts {
 500     pub sysroot: Interned<PathBuf>,
 501     pub host: Interned<String>,
 502 }
 503
 504 impl Step for DebuggerScripts {
 505     type Output = ();
 506
 507     fn should_run(run: ShouldRun) -> ShouldRun {
 508         run.path("src/lldb_batchmode.py")
 509     }
 510
 511     fn make_run(run: RunConfig) {
 512         run.builder.ensure(DebuggerScripts {
 513             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 514             host: run.target,
 515         });
 516     }
 517
 518     /// Copies debugger scripts for `target` into the `sysroot` specified.
 519     fn run(self, builder: &Builder) {
 520         let build = builder.build;
 521         let host = self.host;
 522         let sysroot = self.sysroot;
 523         let dst = sysroot.join("lib/rustlib/etc");
 524         t!(fs::create_dir_all(&dst));
 525         let cp_debugger_script = |file: &str| {
 526             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 527         };
 528         if host.contains("windows-msvc") {
 529             // windbg debugger scripts
 530             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 531                 0o755);
 532
 533             cp_debugger_script("natvis/intrinsic.natvis");
 534             cp_debugger_script("natvis/liballoc.natvis");
 535             cp_debugger_script("natvis/libcore.natvis");
 536         } else {
 537             cp_debugger_script("debugger_pretty_printers_common.py");
 538
 539             // gdb debugger scripts
 540             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 541                     0o755);
 542
 543             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 544             cp_debugger_script("gdb_rust_pretty_printing.py");
 545
 546             // lldb debugger scripts
 547             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 548                     0o755);
 549
 550             cp_debugger_script("lldb_rust_formatters.py");
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: Interned<String>,
 559 }
 560
 561 impl Step for Std {
 562     type Output = PathBuf;
 563     const DEFAULT: bool = true;
 564     const ONLY_BUILD_TARGETS: bool = true;
 565
 566     fn should_run(run: ShouldRun) -> ShouldRun {
 567         run.path("src/libstd")
 568     }
 569
 570     fn make_run(run: RunConfig) {
 571         run.builder.ensure(Std {
 572             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 573             target: run.target,
 574         });
 575     }
 576
 577     fn run(self, builder: &Builder) -> PathBuf {
 578         let build = builder.build;
 579         let compiler = self.compiler;
 580         let target = self.target;
 581
 582         let name = pkgname(build, "rust-std");
 583         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 584
 585         // The only true set of target libraries came from the build triple, so
 586         // let's reduce redundant work by only producing archives from that host.
 587         if compiler.host != build.build {
 588             println!("\tskipping, not a build host");
 589             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 590         }
 591
 592         // We want to package up as many target libraries as possible
 593         // for the `rust-std` package, so if this is a host target we
 594         // depend on librustc and otherwise we just depend on libtest.
 595         if build.hosts.iter().any(|t| t == target) {
 596             builder.ensure(compile::Rustc { compiler, target });
 597         } else {
 598             builder.ensure(compile::Test { compiler, target });
 599         }
 600
 601         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 602         let _ = fs::remove_dir_all(&image);
 603
 604         let dst = image.join("lib/rustlib").join(target);
 605         t!(fs::create_dir_all(&dst));
 606         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 607         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 608         cp_filtered(&src, &dst, &|path| {
 609             path.file_name().and_then(|s| s.to_str()) !=
 610                 Some(build.config.rust_codegen_backends_dir.as_str())
 611         });
 612
 613         let mut cmd = rust_installer(builder);
 614         cmd.arg("generate")
 615            .arg("--product-name=Rust")
 616            .arg("--rel-manifest-dir=rustlib")
 617            .arg("--success-message=std-is-standing-at-the-ready.")
 618            .arg("--image-dir").arg(&image)
 619            .arg("--work-dir").arg(&tmpdir(build))
 620            .arg("--output-dir").arg(&distdir(build))
 621            .arg(format!("--package-name={}-{}", name, target))
 622            .arg(format!("--component-name=rust-std-{}", target))
 623            .arg("--legacy-manifest-dirs=rustlib,cargo");
 624         build.run(&mut cmd);
 625         t!(fs::remove_dir_all(&image));
 626         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 627     }
 628 }
 629
 630 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 631 pub struct Analysis {
 632     pub compiler: Compiler,
 633     pub target: Interned<String>,
 634 }
 635
 636 impl Step for Analysis {
 637     type Output = PathBuf;
 638     const DEFAULT: bool = true;
 639     const ONLY_BUILD_TARGETS: bool = true;
 640
 641     fn should_run(run: ShouldRun) -> ShouldRun {
 642         let builder = run.builder;
 643         run.path("analysis").default_condition(builder.build.config.extended)
 644     }
 645
 646     fn make_run(run: RunConfig) {
 647         run.builder.ensure(Analysis {
 648             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 649             target: run.target,
 650         });
 651     }
 652
 653     /// Creates a tarball of save-analysis metadata, if available.
 654     fn run(self, builder: &Builder) -> PathBuf {
 655         let build = builder.build;
 656         let compiler = self.compiler;
 657         let target = self.target;
 658         assert!(build.config.extended);
 659         println!("Dist analysis");
 660         let name = pkgname(build, "rust-analysis");
 661
 662         if &compiler.host != build.build {
 663             println!("\tskipping, not a build host");
 664             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 665         }
 666
 667         builder.ensure(Std { compiler, target });
 668
 669         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 670         // stage2 artifacts is simply copied from stage1 in that case.
 671         let compiler = if build.force_use_stage1(compiler, target) {
 672             builder.compiler(1, compiler.host)
 673         } else {
 674             compiler.clone()
 675         };
 676
 677         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 678
 679         let src = build.stage_out(compiler, Mode::Libstd)
 680             .join(target).join(build.cargo_dir()).join("deps");
 681
 682         let image_src = src.join("save-analysis");
 683         let dst = image.join("lib/rustlib").join(target).join("analysis");
 684         t!(fs::create_dir_all(&dst));
 685         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 686         cp_r(&image_src, &dst);
 687
 688         let mut cmd = rust_installer(builder);
 689         cmd.arg("generate")
 690            .arg("--product-name=Rust")
 691            .arg("--rel-manifest-dir=rustlib")
 692            .arg("--success-message=save-analysis-saved.")
 693            .arg("--image-dir").arg(&image)
 694            .arg("--work-dir").arg(&tmpdir(build))
 695            .arg("--output-dir").arg(&distdir(build))
 696            .arg(format!("--package-name={}-{}", name, target))
 697            .arg(format!("--component-name=rust-analysis-{}", target))
 698            .arg("--legacy-manifest-dirs=rustlib,cargo");
 699         build.run(&mut cmd);
 700         t!(fs::remove_dir_all(&image));
 701         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 702     }
 703 }
 704
 705 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 706     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 707         let spath = match path.to_str() {
 708             Some(path) => path,
 709             None => return false,
 710         };
 711         if spath.ends_with("~") || spath.ends_with(".pyc") {
 712             return false
 713         }
 714         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 715             (spath.ends_with(".ll") ||
 716              spath.ends_with(".td") ||
 717              spath.ends_with(".s")) {
 718             return false
 719         }
 720         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 721             return false
 722         }
 723
 724         let full_path = Path::new(dir).join(path);
 725         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 726             return false;
 727         }
 728
 729         let excludes = [
 730             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 731             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 732             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 733             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 734         ];
 735         !path.iter()
 736              .map(|s| s.to_str().unwrap())
 737              .any(|s| excludes.contains(&s))
 738     }
 739
 740     // Copy the directories using our filter
 741     for item in src_dirs {
 742         let dst = &dst_dir.join(item);
 743         t!(fs::create_dir_all(dst));
 744         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 745     }
 746 }
 747
 748 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 749 pub struct Src;
 750
 751 impl Step for Src {
 752     /// The output path of the src installer tarball
 753     type Output = PathBuf;
 754     const DEFAULT: bool = true;
 755     const ONLY_HOSTS: bool = true;
 756     const ONLY_BUILD_TARGETS: bool = true;
 757     const ONLY_BUILD: bool = true;
 758
 759     fn should_run(run: ShouldRun) -> ShouldRun {
 760         run.path("src")
 761     }
 762
 763     fn make_run(run: RunConfig) {
 764         run.builder.ensure(Src);
 765     }
 766
 767     /// Creates the `rust-src` installer component
 768     fn run(self, builder: &Builder) -> PathBuf {
 769         let build = builder.build;
 770         println!("Dist src");
 771
 772         let name = pkgname(build, "rust-src");
 773         let image = tmpdir(build).join(format!("{}-image", name));
 774         let _ = fs::remove_dir_all(&image);
 775
 776         let dst = image.join("lib/rustlib/src");
 777         let dst_src = dst.join("rust");
 778         t!(fs::create_dir_all(&dst_src));
 779
 780         let src_files = [
 781             "src/Cargo.lock",
 782         ];
 783         // This is the reduced set of paths which will become the rust-src component
 784         // (essentially libstd and all of its path dependencies)
 785         let std_src_dirs = [
 786             "src/build_helper",
 787             "src/dlmalloc",
 788             "src/liballoc",
 789             "src/liballoc_jemalloc",
 790             "src/liballoc_system",
 791             "src/libbacktrace",
 792             "src/libcompiler_builtins",
 793             "src/libcore",
 794             "src/liblibc",
 795             "src/libpanic_abort",
 796             "src/libpanic_unwind",
 797             "src/librustc_asan",
 798             "src/librustc_lsan",
 799             "src/librustc_msan",
 800             "src/librustc_tsan",
 801             "src/libstd",
 802             "src/libstd_unicode",
 803             "src/libunwind",
 804             "src/rustc/compiler_builtins_shim",
 805             "src/rustc/libc_shim",
 806             "src/rustc/dlmalloc_shim",
 807             "src/libtest",
 808             "src/libterm",
 809             "src/jemalloc",
 810             "src/libprofiler_builtins",
 811         ];
 812         let std_src_dirs_exclude = [
 813             "src/libcompiler_builtins/compiler-rt/test",
 814             "src/jemalloc/test/unit",
 815         ];
 816
 817         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 818         for file in src_files.iter() {
 819             copy(&build.src.join(file), &dst_src.join(file));
 820         }
 821
 822         // Create source tarball in rust-installer format
 823         let mut cmd = rust_installer(builder);
 824         cmd.arg("generate")
 825            .arg("--product-name=Rust")
 826            .arg("--rel-manifest-dir=rustlib")
 827            .arg("--success-message=Awesome-Source.")
 828            .arg("--image-dir").arg(&image)
 829            .arg("--work-dir").arg(&tmpdir(build))
 830            .arg("--output-dir").arg(&distdir(build))
 831            .arg(format!("--package-name={}", name))
 832            .arg("--component-name=rust-src")
 833            .arg("--legacy-manifest-dirs=rustlib,cargo");
 834         build.run(&mut cmd);
 835
 836         t!(fs::remove_dir_all(&image));
 837         distdir(build).join(&format!("{}.tar.gz", name))
 838     }
 839 }
 840
 841 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 842
 843 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 844 pub struct PlainSourceTarball;
 845
 846 impl Step for PlainSourceTarball {
 847     /// Produces the location of the tarball generated
 848     type Output = PathBuf;
 849     const DEFAULT: bool = true;
 850     const ONLY_HOSTS: bool = true;
 851     const ONLY_BUILD_TARGETS: bool = true;
 852     const ONLY_BUILD: bool = true;
 853
 854     fn should_run(run: ShouldRun) -> ShouldRun {
 855         let builder = run.builder;
 856         run.path("src").default_condition(builder.config.rust_dist_src)
 857     }
 858
 859     fn make_run(run: RunConfig) {
 860         run.builder.ensure(PlainSourceTarball);
 861     }
 862
 863     /// Creates the plain source tarball
 864     fn run(self, builder: &Builder) -> PathBuf {
 865         let build = builder.build;
 866         println!("Create plain source tarball");
 867
 868         // Make sure that the root folder of tarball has the correct name
 869         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 870         let plain_dst_src = tmpdir(build).join(&plain_name);
 871         let _ = fs::remove_dir_all(&plain_dst_src);
 872         t!(fs::create_dir_all(&plain_dst_src));
 873
 874         // This is the set of root paths which will become part of the source package
 875         let src_files = [
 876             "COPYRIGHT",
 877             "LICENSE-APACHE",
 878             "LICENSE-MIT",
 879             "CONTRIBUTING.md",
 880             "README.md",
 881             "RELEASES.md",
 882             "configure",
 883             "x.py",
 884             "config.toml.example",
 885         ];
 886         let src_dirs = [
 887             "src",
 888         ];
 889
 890         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 891
 892         // Copy the files normally
 893         for item in &src_files {
 894             copy(&build.src.join(item), &plain_dst_src.join(item));
 895         }
 896
 897         // Create the version file
 898         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 899         if let Some(sha) = build.rust_sha() {
 900             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 901         }
 902
 903         // If we're building from git sources, we need to vendor a complete distribution.
 904         if build.rust_info.is_git() {
 905             // Get cargo-vendor installed, if it isn't already.
 906             let mut has_cargo_vendor = false;
 907             let mut cmd = Command::new(&build.initial_cargo);
 908             for line in output(cmd.arg("install").arg("--list")).lines() {
 909                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 910             }
 911             if !has_cargo_vendor {
 912                 let mut cmd = Command::new(&build.initial_cargo);
 913                 cmd.arg("install")
 914                    .arg("--force")
 915                    .arg("--debug")
 916                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 917                    .arg("cargo-vendor")
 918                    .env("RUSTC", &build.initial_rustc);
 919                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 920                     builder.ensure(native::Openssl {
 921                         target: build.config.build,
 922                     });
 923                     cmd.env("OPENSSL_DIR", dir);
 924                 }
 925                 build.run(&mut cmd);
 926             }
 927
 928             // Vendor all Cargo dependencies
 929             let mut cmd = Command::new(&build.initial_cargo);
 930             cmd.arg("vendor")
 931                .current_dir(&plain_dst_src.join("src"));
 932             build.run(&mut cmd);
 933         }
 934
 935         // Create plain source tarball
 936         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 937         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 938         tarball.set_extension(""); // strip .gz
 939         tarball.set_extension(""); // strip .tar
 940         if let Some(dir) = tarball.parent() {
 941             t!(fs::create_dir_all(dir));
 942         }
 943         println!("running installer");
 944         let mut cmd = rust_installer(builder);
 945         cmd.arg("tarball")
 946            .arg("--input").arg(&plain_name)
 947            .arg("--output").arg(&tarball)
 948            .arg("--work-dir=.")
 949            .current_dir(tmpdir(build));
 950         build.run(&mut cmd);
 951         distdir(build).join(&format!("{}.tar.gz", plain_name))
 952     }
 953 }
 954
 955 fn install(src: &Path, dstdir: &Path, perms: u32) {
 956     let dst = dstdir.join(src.file_name().unwrap());
 957     t!(fs::create_dir_all(dstdir));
 958     drop(fs::remove_file(&dst));
 959     {
 960         let mut s = t!(fs::File::open(&src));
 961         let mut d = t!(fs::File::create(&dst));
 962         io::copy(&mut s, &mut d).expect("failed to copy");
 963     }
 964     chmod(&dst, perms);
 965 }
 966
 967 #[cfg(unix)]
 968 fn chmod(path: &Path, perms: u32) {
 969     use std::os::unix::fs::*;
 970     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 971 }
 972 #[cfg(windows)]
 973 fn chmod(_path: &Path, _perms: u32) {}
 974
 975 // We have to run a few shell scripts, which choke quite a bit on both `\`
 976 // characters and on `C:\` paths, so normalize both of them away.
 977 pub fn sanitize_sh(path: &Path) -> String {
 978     let path = path.to_str().unwrap().replace("\\", "/");
 979     return change_drive(&path).unwrap_or(path);
 980
 981     fn change_drive(s: &str) -> Option<String> {
 982         let mut ch = s.chars();
 983         let drive = ch.next().unwrap_or('C');
 984         if ch.next() != Some(':') {
 985             return None
 986         }
 987         if ch.next() != Some('/') {
 988             return None
 989         }
 990         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 991     }
 992 }
 993
 994 fn write_file(path: &Path, data: &[u8]) {
 995     let mut vf = t!(fs::File::create(path));
 996     t!(vf.write_all(data));
 997 }
 998
 999 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1000 pub struct Cargo {
1001     pub stage: u32,
1002     pub target: Interned<String>,
1003 }
1004
1005 impl Step for Cargo {
1006     type Output = PathBuf;
1007     const ONLY_BUILD_TARGETS: bool = true;
1008     const ONLY_HOSTS: bool = true;
1009
1010     fn should_run(run: ShouldRun) -> ShouldRun {
1011         run.path("cargo")
1012     }
1013
1014     fn make_run(run: RunConfig) {
1015         run.builder.ensure(Cargo {
1016             stage: run.builder.top_stage,
1017             target: run.target,
1018         });
1019     }
1020
1021     fn run(self, builder: &Builder) -> PathBuf {
1022         let build = builder.build;
1023         let stage = self.stage;
1024         let target = self.target;
1025
1026         println!("Dist cargo stage{} ({})", stage, target);
1027         let src = build.src.join("src/tools/cargo");
1028         let etc = src.join("src/etc");
1029         let release_num = build.release_num("cargo");
1030         let name = pkgname(build, "cargo");
1031         let version = builder.cargo_info.version(build, &release_num);
1032
1033         let tmp = tmpdir(build);
1034         let image = tmp.join("cargo-image");
1035         drop(fs::remove_dir_all(&image));
1036         t!(fs::create_dir_all(&image));
1037
1038         // Prepare the image directory
1039         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1040         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1041         let cargo = builder.ensure(tool::Cargo {
1042             compiler: builder.compiler(stage, build.build),
1043             target
1044         });
1045         install(&cargo, &image.join("bin"), 0o755);
1046         for man in t!(etc.join("man").read_dir()) {
1047             let man = t!(man);
1048             install(&man.path(), &image.join("share/man/man1"), 0o644);
1049         }
1050         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1051         copy(&etc.join("cargo.bashcomp.sh"),
1052              &image.join("etc/bash_completion.d/cargo"));
1053         let doc = image.join("share/doc/cargo");
1054         install(&src.join("README.md"), &doc, 0o644);
1055         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1056         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1057         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1058
1059         // Prepare the overlay
1060         let overlay = tmp.join("cargo-overlay");
1061         drop(fs::remove_dir_all(&overlay));
1062         t!(fs::create_dir_all(&overlay));
1063         install(&src.join("README.md"), &overlay, 0o644);
1064         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1065         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1066         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1067         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1068
1069         // Generate the installer tarball
1070         let mut cmd = rust_installer(builder);
1071         cmd.arg("generate")
1072            .arg("--product-name=Rust")
1073            .arg("--rel-manifest-dir=rustlib")
1074            .arg("--success-message=Rust-is-ready-to-roll.")
1075            .arg("--image-dir").arg(&image)
1076            .arg("--work-dir").arg(&tmpdir(build))
1077            .arg("--output-dir").arg(&distdir(build))
1078            .arg("--non-installed-overlay").arg(&overlay)
1079            .arg(format!("--package-name={}-{}", name, target))
1080            .arg("--component-name=cargo")
1081            .arg("--legacy-manifest-dirs=rustlib,cargo");
1082         build.run(&mut cmd);
1083         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1084     }
1085 }
1086
1087 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1088 pub struct Rls {
1089     pub stage: u32,
1090     pub target: Interned<String>,
1091 }
1092
1093 impl Step for Rls {
1094     type Output = Option<PathBuf>;
1095     const ONLY_BUILD_TARGETS: bool = true;
1096     const ONLY_HOSTS: bool = true;
1097
1098     fn should_run(run: ShouldRun) -> ShouldRun {
1099         run.path("rls")
1100     }
1101
1102     fn make_run(run: RunConfig) {
1103         run.builder.ensure(Rls {
1104             stage: run.builder.top_stage,
1105             target: run.target,
1106         });
1107     }
1108
1109     fn run(self, builder: &Builder) -> Option<PathBuf> {
1110         let build = builder.build;
1111         let stage = self.stage;
1112         let target = self.target;
1113         assert!(build.config.extended);
1114
1115         println!("Dist RLS stage{} ({})", stage, target);
1116         let src = build.src.join("src/tools/rls");
1117         let release_num = build.release_num("rls");
1118         let name = pkgname(build, "rls");
1119         let version = build.rls_info.version(build, &release_num);
1120
1121         let tmp = tmpdir(build);
1122         let image = tmp.join("rls-image");
1123         drop(fs::remove_dir_all(&image));
1124         t!(fs::create_dir_all(&image));
1125
1126         // Prepare the image directory
1127         // We expect RLS to build, because we've exited this step above if tool
1128         // state for RLS isn't testing.
1129         let rls = builder.ensure(tool::Rls {
1130             compiler: builder.compiler(stage, build.build),
1131             target
1132         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1133
1134         install(&rls, &image.join("bin"), 0o755);
1135         let doc = image.join("share/doc/rls");
1136         install(&src.join("README.md"), &doc, 0o644);
1137         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1138         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1139
1140         // Prepare the overlay
1141         let overlay = tmp.join("rls-overlay");
1142         drop(fs::remove_dir_all(&overlay));
1143         t!(fs::create_dir_all(&overlay));
1144         install(&src.join("README.md"), &overlay, 0o644);
1145         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1146         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1147         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1148
1149         // Generate the installer tarball
1150         let mut cmd = rust_installer(builder);
1151         cmd.arg("generate")
1152            .arg("--product-name=Rust")
1153            .arg("--rel-manifest-dir=rustlib")
1154            .arg("--success-message=RLS-ready-to-serve.")
1155            .arg("--image-dir").arg(&image)
1156            .arg("--work-dir").arg(&tmpdir(build))
1157            .arg("--output-dir").arg(&distdir(build))
1158            .arg("--non-installed-overlay").arg(&overlay)
1159            .arg(format!("--package-name={}-{}", name, target))
1160            .arg("--legacy-manifest-dirs=rustlib,cargo")
1161            .arg("--component-name=rls-preview");
1162
1163         build.run(&mut cmd);
1164         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1165     }
1166 }
1167
1168
1169 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1170 pub struct Rustfmt {
1171     pub stage: u32,
1172     pub target: Interned<String>,
1173 }
1174
1175 impl Step for Rustfmt {
1176     type Output = Option<PathBuf>;
1177     const ONLY_BUILD_TARGETS: bool = true;
1178     const ONLY_HOSTS: bool = true;
1179
1180     fn should_run(run: ShouldRun) -> ShouldRun {
1181         run.path("rustfmt")
1182     }
1183
1184     fn make_run(run: RunConfig) {
1185         run.builder.ensure(Rustfmt {
1186             stage: run.builder.top_stage,
1187             target: run.target,
1188         });
1189     }
1190
1191     fn run(self, builder: &Builder) -> Option<PathBuf> {
1192         let build = builder.build;
1193         let stage = self.stage;
1194         let target = self.target;
1195         assert!(build.config.extended);
1196
1197         println!("Dist Rustfmt stage{} ({})", stage, target);
1198         let src = build.src.join("src/tools/rustfmt");
1199         let release_num = build.release_num("rustfmt");
1200         let name = pkgname(build, "rustfmt");
1201         let version = build.rustfmt_info.version(build, &release_num);
1202
1203         let tmp = tmpdir(build);
1204         let image = tmp.join("rustfmt-image");
1205         drop(fs::remove_dir_all(&image));
1206         t!(fs::create_dir_all(&image));
1207
1208         // Prepare the image directory
1209         let rustfmt = builder.ensure(tool::Rustfmt {
1210             compiler: builder.compiler(stage, build.build),
1211             target
1212         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1213         let cargofmt = builder.ensure(tool::Cargofmt {
1214             compiler: builder.compiler(stage, build.build),
1215             target
1216         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1217
1218         install(&rustfmt, &image.join("bin"), 0o755);
1219         install(&cargofmt, &image.join("bin"), 0o755);
1220         let doc = image.join("share/doc/rustfmt");
1221         install(&src.join("README.md"), &doc, 0o644);
1222         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1223         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1224
1225         // Prepare the overlay
1226         let overlay = tmp.join("rustfmt-overlay");
1227         drop(fs::remove_dir_all(&overlay));
1228         t!(fs::create_dir_all(&overlay));
1229         install(&src.join("README.md"), &overlay, 0o644);
1230         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1231         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1232         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1233
1234         // Generate the installer tarball
1235         let mut cmd = rust_installer(builder);
1236         cmd.arg("generate")
1237            .arg("--product-name=Rust")
1238            .arg("--rel-manifest-dir=rustlib")
1239            .arg("--success-message=rustfmt-ready-to-fmt.")
1240            .arg("--image-dir").arg(&image)
1241            .arg("--work-dir").arg(&tmpdir(build))
1242            .arg("--output-dir").arg(&distdir(build))
1243            .arg("--non-installed-overlay").arg(&overlay)
1244            .arg(format!("--package-name={}-{}", name, target))
1245            .arg("--legacy-manifest-dirs=rustlib,cargo")
1246            .arg("--component-name=rustfmt-preview");
1247
1248         build.run(&mut cmd);
1249         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1250     }
1251 }
1252
1253 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1254 pub struct Extended {
1255     stage: u32,
1256     host: Interned<String>,
1257     target: Interned<String>,
1258 }
1259
1260 impl Step for Extended {
1261     type Output = ();
1262     const DEFAULT: bool = true;
1263     const ONLY_BUILD_TARGETS: bool = true;
1264     const ONLY_HOSTS: bool = true;
1265
1266     fn should_run(run: ShouldRun) -> ShouldRun {
1267         let builder = run.builder;
1268         run.path("extended").default_condition(builder.config.extended)
1269     }
1270
1271     fn make_run(run: RunConfig) {
1272         run.builder.ensure(Extended {
1273             stage: run.builder.top_stage,
1274             host: run.host,
1275             target: run.target,
1276         });
1277     }
1278
1279     /// Creates a combined installer for the specified target in the provided stage.
1280     fn run(self, builder: &Builder) {
1281         let build = builder.build;
1282         let stage = self.stage;
1283         let target = self.target;
1284
1285         println!("Dist extended stage{} ({})", stage, target);
1286
1287         let rustc_installer = builder.ensure(Rustc {
1288             compiler: builder.compiler(stage, target),
1289         });
1290         let cargo_installer = builder.ensure(Cargo { stage, target });
1291         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1292         let rls_installer = builder.ensure(Rls { stage, target });
1293         let mingw_installer = builder.ensure(Mingw { host: target });
1294         let analysis_installer = builder.ensure(Analysis {
1295             compiler: builder.compiler(stage, self.host),
1296             target
1297         });
1298
1299         let docs_installer = builder.ensure(Docs { stage, host: target, });
1300         let std_installer = builder.ensure(Std {
1301             compiler: builder.compiler(stage, self.host),
1302             target,
1303         });
1304
1305         let tmp = tmpdir(build);
1306         let overlay = tmp.join("extended-overlay");
1307         let etc = build.src.join("src/etc/installer");
1308         let work = tmp.join("work");
1309
1310         let _ = fs::remove_dir_all(&overlay);
1311         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1312         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1313         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1314         let version = build.rust_version();
1315         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1316         if let Some(sha) = build.rust_sha() {
1317             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1318         }
1319         install(&etc.join("README.md"), &overlay, 0o644);
1320
1321         // When rust-std package split from rustc, we needed to ensure that during
1322         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1323         // the std files during uninstall. To do this ensure that rustc comes
1324         // before rust-std in the list below.
1325         let mut tarballs = Vec::new();
1326         tarballs.push(rustc_installer);
1327         tarballs.push(cargo_installer);
1328         tarballs.extend(rls_installer.clone());
1329         tarballs.extend(rustfmt_installer.clone());
1330         tarballs.push(analysis_installer);
1331         tarballs.push(std_installer);
1332         if build.config.docs {
1333             tarballs.push(docs_installer);
1334         }
1335         if target.contains("pc-windows-gnu") {
1336             tarballs.push(mingw_installer.unwrap());
1337         }
1338         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1339         for tarball in &tarballs[1..] {
1340             input_tarballs.push(",");
1341             input_tarballs.push(tarball);
1342         }
1343
1344         let mut cmd = rust_installer(builder);
1345         cmd.arg("combine")
1346             .arg("--product-name=Rust")
1347             .arg("--rel-manifest-dir=rustlib")
1348             .arg("--success-message=Rust-is-ready-to-roll.")
1349             .arg("--work-dir").arg(&work)
1350             .arg("--output-dir").arg(&distdir(build))
1351             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1352             .arg("--legacy-manifest-dirs=rustlib,cargo")
1353             .arg("--input-tarballs").arg(input_tarballs)
1354             .arg("--non-installed-overlay").arg(&overlay);
1355         build.run(&mut cmd);
1356
1357         let mut license = String::new();
1358         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1359         license.push_str("\n");
1360         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1361         license.push_str("\n");
1362         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1363
1364         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1365         let mut rtf = rtf.to_string();
1366         rtf.push_str("\n");
1367         for line in license.lines() {
1368             rtf.push_str(line);
1369             rtf.push_str("\\line ");
1370         }
1371         rtf.push_str("}");
1372
1373         fn filter(contents: &str, marker: &str) -> String {
1374             let start = format!("tool-{}-start", marker);
1375             let end = format!("tool-{}-end", marker);
1376             let mut lines = Vec::new();
1377             let mut omitted = false;
1378             for line in contents.lines() {
1379                 if line.contains(&start) {
1380                     omitted = true;
1381                 } else if line.contains(&end) {
1382                     omitted = false;
1383                 } else if !omitted {
1384                     lines.push(line);
1385                 }
1386             }
1387
1388             lines.join("\n")
1389         }
1390
1391         let xform = |p: &Path| {
1392             let mut contents = String::new();
1393             t!(t!(File::open(p)).read_to_string(&mut contents));
1394             if rls_installer.is_none() {
1395                 contents = filter(&contents, "rls");
1396             }
1397             if rustfmt_installer.is_none() {
1398                 contents = filter(&contents, "rustfmt");
1399             }
1400             let ret = tmp.join(p.file_name().unwrap());
1401             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1402             return ret
1403         };
1404
1405         if target.contains("apple-darwin") {
1406             let pkg = tmp.join("pkg");
1407             let _ = fs::remove_dir_all(&pkg);
1408
1409             let pkgbuild = |component: &str| {
1410                 let mut cmd = Command::new("pkgbuild");
1411                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1412                     .arg("--scripts").arg(pkg.join(component))
1413                     .arg("--nopayload")
1414                     .arg(pkg.join(component).with_extension("pkg"));
1415                 build.run(&mut cmd);
1416             };
1417
1418             let prepare = |name: &str| {
1419                 t!(fs::create_dir_all(pkg.join(name)));
1420                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1421                         &pkg.join(name));
1422                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1423                 pkgbuild(name);
1424             };
1425             prepare("rustc");
1426             prepare("cargo");
1427             prepare("rust-docs");
1428             prepare("rust-std");
1429             prepare("rust-analysis");
1430
1431             if rls_installer.is_some() {
1432                 prepare("rls");
1433             }
1434
1435             // create an 'uninstall' package
1436             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1437             pkgbuild("uninstall");
1438
1439             t!(fs::create_dir_all(pkg.join("res")));
1440             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1441             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1442             let mut cmd = Command::new("productbuild");
1443             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1444                 .arg("--resources").arg(pkg.join("res"))
1445                 .arg(distdir(build).join(format!("{}-{}.pkg",
1446                                                     pkgname(build, "rust"),
1447                                                     target)))
1448                 .arg("--package-path").arg(&pkg);
1449             build.run(&mut cmd);
1450         }
1451
1452         if target.contains("windows") {
1453             let exe = tmp.join("exe");
1454             let _ = fs::remove_dir_all(&exe);
1455
1456             let prepare = |name: &str| {
1457                 t!(fs::create_dir_all(exe.join(name)));
1458                 let dir = if name == "rust-std" || name == "rust-analysis" {
1459                     format!("{}-{}", name, target)
1460                 } else if name == "rls" {
1461                     "rls-preview".to_string()
1462                 } else {
1463                     name.to_string()
1464                 };
1465                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1466                             .join(dir),
1467                         &exe.join(name));
1468                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1469             };
1470             prepare("rustc");
1471             prepare("cargo");
1472             prepare("rust-analysis");
1473             prepare("rust-docs");
1474             prepare("rust-std");
1475             if rls_installer.is_some() {
1476                 prepare("rls");
1477             }
1478             if target.contains("windows-gnu") {
1479                 prepare("rust-mingw");
1480             }
1481
1482             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1483             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1484             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1485             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1486             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1487
1488             // Generate exe installer
1489             let mut cmd = Command::new("iscc");
1490             cmd.arg("rust.iss")
1491                 .current_dir(&exe);
1492             if target.contains("windows-gnu") {
1493                 cmd.arg("/dMINGW");
1494             }
1495             add_env(build, &mut cmd, target);
1496             build.run(&mut cmd);
1497             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1498                     &distdir(build),
1499                     0o755);
1500
1501             // Generate msi installer
1502             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1503             let heat = wix.join("bin/heat.exe");
1504             let candle = wix.join("bin/candle.exe");
1505             let light = wix.join("bin/light.exe");
1506
1507             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1508             build.run(Command::new(&heat)
1509                             .current_dir(&exe)
1510                             .arg("dir")
1511                             .arg("rustc")
1512                             .args(&heat_flags)
1513                             .arg("-cg").arg("RustcGroup")
1514                             .arg("-dr").arg("Rustc")
1515                             .arg("-var").arg("var.RustcDir")
1516                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1517             build.run(Command::new(&heat)
1518                             .current_dir(&exe)
1519                             .arg("dir")
1520                             .arg("rust-docs")
1521                             .args(&heat_flags)
1522                             .arg("-cg").arg("DocsGroup")
1523                             .arg("-dr").arg("Docs")
1524                             .arg("-var").arg("var.DocsDir")
1525                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1526                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1527             build.run(Command::new(&heat)
1528                             .current_dir(&exe)
1529                             .arg("dir")
1530                             .arg("cargo")
1531                             .args(&heat_flags)
1532                             .arg("-cg").arg("CargoGroup")
1533                             .arg("-dr").arg("Cargo")
1534                             .arg("-var").arg("var.CargoDir")
1535                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1536                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1537             build.run(Command::new(&heat)
1538                             .current_dir(&exe)
1539                             .arg("dir")
1540                             .arg("rust-std")
1541                             .args(&heat_flags)
1542                             .arg("-cg").arg("StdGroup")
1543                             .arg("-dr").arg("Std")
1544                             .arg("-var").arg("var.StdDir")
1545                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1546             if rls_installer.is_some() {
1547                 build.run(Command::new(&heat)
1548                                 .current_dir(&exe)
1549                                 .arg("dir")
1550                                 .arg("rls")
1551                                 .args(&heat_flags)
1552                                 .arg("-cg").arg("RlsGroup")
1553                                 .arg("-dr").arg("Rls")
1554                                 .arg("-var").arg("var.RlsDir")
1555                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1556                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1557             }
1558             build.run(Command::new(&heat)
1559                             .current_dir(&exe)
1560                             .arg("dir")
1561                             .arg("rust-analysis")
1562                             .args(&heat_flags)
1563                             .arg("-cg").arg("AnalysisGroup")
1564                             .arg("-dr").arg("Analysis")
1565                             .arg("-var").arg("var.AnalysisDir")
1566                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1567                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1568             if target.contains("windows-gnu") {
1569                 build.run(Command::new(&heat)
1570                                 .current_dir(&exe)
1571                                 .arg("dir")
1572                                 .arg("rust-mingw")
1573                                 .args(&heat_flags)
1574                                 .arg("-cg").arg("GccGroup")
1575                                 .arg("-dr").arg("Gcc")
1576                                 .arg("-var").arg("var.GccDir")
1577                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1578             }
1579
1580             let candle = |input: &Path| {
1581                 let output = exe.join(input.file_stem().unwrap())
1582                                 .with_extension("wixobj");
1583                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1584                 let mut cmd = Command::new(&candle);
1585                 cmd.current_dir(&exe)
1586                     .arg("-nologo")
1587                     .arg("-dRustcDir=rustc")
1588                     .arg("-dDocsDir=rust-docs")
1589                     .arg("-dCargoDir=cargo")
1590                     .arg("-dStdDir=rust-std")
1591                     .arg("-dAnalysisDir=rust-analysis")
1592                     .arg("-arch").arg(&arch)
1593                     .arg("-out").arg(&output)
1594                     .arg(&input);
1595                 add_env(build, &mut cmd, target);
1596
1597                 if rls_installer.is_some() {
1598                     cmd.arg("-dRlsDir=rls");
1599                 }
1600                 if target.contains("windows-gnu") {
1601                     cmd.arg("-dGccDir=rust-mingw");
1602                 }
1603                 build.run(&mut cmd);
1604             };
1605             candle(&xform(&etc.join("msi/rust.wxs")));
1606             candle(&etc.join("msi/ui.wxs"));
1607             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1608             candle("RustcGroup.wxs".as_ref());
1609             candle("DocsGroup.wxs".as_ref());
1610             candle("CargoGroup.wxs".as_ref());
1611             candle("StdGroup.wxs".as_ref());
1612             if rls_installer.is_some() {
1613                 candle("RlsGroup.wxs".as_ref());
1614             }
1615             candle("AnalysisGroup.wxs".as_ref());
1616
1617             if target.contains("windows-gnu") {
1618                 candle("GccGroup.wxs".as_ref());
1619             }
1620
1621             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1622             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1623             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1624
1625             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1626             let mut cmd = Command::new(&light);
1627             cmd.arg("-nologo")
1628                 .arg("-ext").arg("WixUIExtension")
1629                 .arg("-ext").arg("WixUtilExtension")
1630                 .arg("-out").arg(exe.join(&filename))
1631                 .arg("rust.wixobj")
1632                 .arg("ui.wixobj")
1633                 .arg("rustwelcomedlg.wixobj")
1634                 .arg("RustcGroup.wixobj")
1635                 .arg("DocsGroup.wixobj")
1636                 .arg("CargoGroup.wixobj")
1637                 .arg("StdGroup.wixobj")
1638                 .arg("AnalysisGroup.wixobj")
1639                 .current_dir(&exe);
1640
1641             if rls_installer.is_some() {
1642                 cmd.arg("RlsGroup.wixobj");
1643             }
1644
1645             if target.contains("windows-gnu") {
1646                 cmd.arg("GccGroup.wixobj");
1647             }
1648             // ICE57 wrongly complains about the shortcuts
1649             cmd.arg("-sice:ICE57");
1650
1651             build.run(&mut cmd);
1652
1653             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1654         }
1655     }
1656 }
1657
1658 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1659     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1660     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1661        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1662        .env("CFG_RELEASE", build.rust_release())
1663        .env("CFG_VER_MAJOR", parts.next().unwrap())
1664        .env("CFG_VER_MINOR", parts.next().unwrap())
1665        .env("CFG_VER_PATCH", parts.next().unwrap())
1666        .env("CFG_VER_BUILD", "0") // just needed to build
1667        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1668        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1669        .env("CFG_BUILD", target)
1670        .env("CFG_CHANNEL", &build.config.channel);
1671
1672     if target.contains("windows-gnu") {
1673        cmd.env("CFG_MINGW", "1")
1674           .env("CFG_ABI", "GNU");
1675     } else {
1676        cmd.env("CFG_MINGW", "0")
1677           .env("CFG_ABI", "MSVC");
1678     }
1679
1680     if target.contains("x86_64") {
1681        cmd.env("CFG_PLATFORM", "x64");
1682     } else {
1683        cmd.env("CFG_PLATFORM", "x86");
1684     }
1685 }
1686
1687 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1688 pub struct HashSign;
1689
1690 impl Step for HashSign {
1691     type Output = ();
1692     const ONLY_BUILD_TARGETS: bool = true;
1693     const ONLY_HOSTS: bool = true;
1694     const ONLY_BUILD: bool = true;
1695
1696     fn should_run(run: ShouldRun) -> ShouldRun {
1697         run.path("hash-and-sign")
1698     }
1699
1700     fn make_run(run: RunConfig) {
1701         run.builder.ensure(HashSign);
1702     }
1703
1704     fn run(self, builder: &Builder) {
1705         let build = builder.build;
1706         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1707         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1708             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1709         });
1710         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1711             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1712         });
1713         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1714             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1715         });
1716         let mut pass = String::new();
1717         t!(t!(File::open(&file)).read_to_string(&mut pass));
1718
1719         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1720
1721         cmd.arg(sign);
1722         cmd.arg(distdir(build));
1723         cmd.arg(today.trim());
1724         cmd.arg(build.rust_package_vers());
1725         cmd.arg(build.package_vers(&build.release_num("cargo")));
1726         cmd.arg(build.package_vers(&build.release_num("rls")));
1727         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1728         cmd.arg(addr);
1729
1730         t!(fs::create_dir_all(distdir(build)));
1731
1732         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1733         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1734         let status = t!(child.wait());
1735         assert!(status.success());
1736     }
1737 }