src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Man pages
 447             t!(fs::create_dir_all(image.join("share/man/man1")));
 448             let man_src = build.src.join("src/doc/man");
 449             let man_dst = image.join("share/man/man1");
 450             let month_year = t!(time::strftime("%B %Y", &time::now()));
 451             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 452             // to hardlink, and we don't want to edit the source templates
 453             for entry_result in t!(fs::read_dir(man_src)) {
 454                 let file_entry = t!(entry_result);
 455                 let page_src = file_entry.path();
 456                 let page_dst = man_dst.join(file_entry.file_name());
 457                 t!(fs::copy(&page_src, &page_dst));
 458                 // template in month/year and version number
 459                 replace_in_file(&page_dst,
 460                                 &[("<INSERT DATE HERE>", &month_year),
 461                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 462             }
 463
 464             // Debugger scripts
 465             builder.ensure(DebuggerScripts {
 466                 sysroot: INTERNER.intern_path(image.to_owned()),
 467                 host,
 468             });
 469
 470             // Misc license info
 471             let cp = |file: &str| {
 472                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 473             };
 474             cp("COPYRIGHT");
 475             cp("LICENSE-APACHE");
 476             cp("LICENSE-MIT");
 477             cp("README.md");
 478         }
 479     }
 480 }
 481
 482 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 483 pub struct DebuggerScripts {
 484     pub sysroot: Interned<PathBuf>,
 485     pub host: Interned<String>,
 486 }
 487
 488 impl Step for DebuggerScripts {
 489     type Output = ();
 490
 491     fn should_run(run: ShouldRun) -> ShouldRun {
 492         run.path("src/lldb_batchmode.py")
 493     }
 494
 495     fn make_run(run: RunConfig) {
 496         run.builder.ensure(DebuggerScripts {
 497             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 498             host: run.target,
 499         });
 500     }
 501
 502     /// Copies debugger scripts for `target` into the `sysroot` specified.
 503     fn run(self, builder: &Builder) {
 504         let build = builder.build;
 505         let host = self.host;
 506         let sysroot = self.sysroot;
 507         let dst = sysroot.join("lib/rustlib/etc");
 508         t!(fs::create_dir_all(&dst));
 509         let cp_debugger_script = |file: &str| {
 510             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 511         };
 512         if host.contains("windows-msvc") {
 513             // windbg debugger scripts
 514             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 515                 0o755);
 516
 517             cp_debugger_script("natvis/intrinsic.natvis");
 518             cp_debugger_script("natvis/liballoc.natvis");
 519             cp_debugger_script("natvis/libcore.natvis");
 520         } else {
 521             cp_debugger_script("debugger_pretty_printers_common.py");
 522
 523             // gdb debugger scripts
 524             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 525                     0o755);
 526
 527             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 528             cp_debugger_script("gdb_rust_pretty_printing.py");
 529
 530             // lldb debugger scripts
 531             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 532                     0o755);
 533
 534             cp_debugger_script("lldb_rust_formatters.py");
 535         }
 536     }
 537 }
 538
 539 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 540 pub struct Std {
 541     pub compiler: Compiler,
 542     pub target: Interned<String>,
 543 }
 544
 545 impl Step for Std {
 546     type Output = PathBuf;
 547     const DEFAULT: bool = true;
 548     const ONLY_BUILD_TARGETS: bool = true;
 549
 550     fn should_run(run: ShouldRun) -> ShouldRun {
 551         run.path("src/libstd")
 552     }
 553
 554     fn make_run(run: RunConfig) {
 555         run.builder.ensure(Std {
 556             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 557             target: run.target,
 558         });
 559     }
 560
 561     fn run(self, builder: &Builder) -> PathBuf {
 562         let build = builder.build;
 563         let compiler = self.compiler;
 564         let target = self.target;
 565
 566         let name = pkgname(build, "rust-std");
 567         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 568
 569         // The only true set of target libraries came from the build triple, so
 570         // let's reduce redundant work by only producing archives from that host.
 571         if compiler.host != build.build {
 572             println!("\tskipping, not a build host");
 573             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 574         }
 575
 576         // We want to package up as many target libraries as possible
 577         // for the `rust-std` package, so if this is a host target we
 578         // depend on librustc and otherwise we just depend on libtest.
 579         if build.hosts.iter().any(|t| t == target) {
 580             builder.ensure(compile::Rustc { compiler, target });
 581         } else {
 582             builder.ensure(compile::Test { compiler, target });
 583         }
 584
 585         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 586         let _ = fs::remove_dir_all(&image);
 587
 588         let dst = image.join("lib/rustlib").join(target);
 589         t!(fs::create_dir_all(&dst));
 590         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 591         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 592         cp_filtered(&src, &dst, &|path| {
 593             path.file_name().and_then(|s| s.to_str()) !=
 594                 Some(build.config.rust_codegen_backends_dir.as_str())
 595         });
 596
 597         let mut cmd = rust_installer(builder);
 598         cmd.arg("generate")
 599            .arg("--product-name=Rust")
 600            .arg("--rel-manifest-dir=rustlib")
 601            .arg("--success-message=std-is-standing-at-the-ready.")
 602            .arg("--image-dir").arg(&image)
 603            .arg("--work-dir").arg(&tmpdir(build))
 604            .arg("--output-dir").arg(&distdir(build))
 605            .arg(format!("--package-name={}-{}", name, target))
 606            .arg(format!("--component-name=rust-std-{}", target))
 607            .arg("--legacy-manifest-dirs=rustlib,cargo");
 608         build.run(&mut cmd);
 609         t!(fs::remove_dir_all(&image));
 610         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 611     }
 612 }
 613
 614 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 615 pub struct Analysis {
 616     pub compiler: Compiler,
 617     pub target: Interned<String>,
 618 }
 619
 620 impl Step for Analysis {
 621     type Output = PathBuf;
 622     const DEFAULT: bool = true;
 623     const ONLY_BUILD_TARGETS: bool = true;
 624
 625     fn should_run(run: ShouldRun) -> ShouldRun {
 626         let builder = run.builder;
 627         run.path("analysis").default_condition(builder.build.config.extended)
 628     }
 629
 630     fn make_run(run: RunConfig) {
 631         run.builder.ensure(Analysis {
 632             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 633             target: run.target,
 634         });
 635     }
 636
 637     /// Creates a tarball of save-analysis metadata, if available.
 638     fn run(self, builder: &Builder) -> PathBuf {
 639         let build = builder.build;
 640         let compiler = self.compiler;
 641         let target = self.target;
 642         assert!(build.config.extended);
 643         println!("Dist analysis");
 644         let name = pkgname(build, "rust-analysis");
 645
 646         if &compiler.host != build.build {
 647             println!("\tskipping, not a build host");
 648             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 649         }
 650
 651         builder.ensure(Std { compiler, target });
 652
 653         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 654         // stage2 artifacts is simply copied from stage1 in that case.
 655         let compiler = if build.force_use_stage1(compiler, target) {
 656             builder.compiler(1, compiler.host)
 657         } else {
 658             compiler.clone()
 659         };
 660
 661         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 662
 663         let src = build.stage_out(compiler, Mode::Libstd)
 664             .join(target).join(build.cargo_dir()).join("deps");
 665
 666         let image_src = src.join("save-analysis");
 667         let dst = image.join("lib/rustlib").join(target).join("analysis");
 668         t!(fs::create_dir_all(&dst));
 669         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 670         cp_r(&image_src, &dst);
 671
 672         let mut cmd = rust_installer(builder);
 673         cmd.arg("generate")
 674            .arg("--product-name=Rust")
 675            .arg("--rel-manifest-dir=rustlib")
 676            .arg("--success-message=save-analysis-saved.")
 677            .arg("--image-dir").arg(&image)
 678            .arg("--work-dir").arg(&tmpdir(build))
 679            .arg("--output-dir").arg(&distdir(build))
 680            .arg(format!("--package-name={}-{}", name, target))
 681            .arg(format!("--component-name=rust-analysis-{}", target))
 682            .arg("--legacy-manifest-dirs=rustlib,cargo");
 683         build.run(&mut cmd);
 684         t!(fs::remove_dir_all(&image));
 685         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 686     }
 687 }
 688
 689 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 690     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 691         let spath = match path.to_str() {
 692             Some(path) => path,
 693             None => return false,
 694         };
 695         if spath.ends_with("~") || spath.ends_with(".pyc") {
 696             return false
 697         }
 698         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 699             (spath.ends_with(".ll") ||
 700              spath.ends_with(".td") ||
 701              spath.ends_with(".s")) {
 702             return false
 703         }
 704         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 705             return false
 706         }
 707
 708         let full_path = Path::new(dir).join(path);
 709         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 710             return false;
 711         }
 712
 713         let excludes = [
 714             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 715             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 716             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 717             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 718         ];
 719         !path.iter()
 720              .map(|s| s.to_str().unwrap())
 721              .any(|s| excludes.contains(&s))
 722     }
 723
 724     // Copy the directories using our filter
 725     for item in src_dirs {
 726         let dst = &dst_dir.join(item);
 727         t!(fs::create_dir_all(dst));
 728         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 729     }
 730 }
 731
 732 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 733 pub struct Src;
 734
 735 impl Step for Src {
 736     /// The output path of the src installer tarball
 737     type Output = PathBuf;
 738     const DEFAULT: bool = true;
 739     const ONLY_HOSTS: bool = true;
 740     const ONLY_BUILD_TARGETS: bool = true;
 741     const ONLY_BUILD: bool = true;
 742
 743     fn should_run(run: ShouldRun) -> ShouldRun {
 744         run.path("src")
 745     }
 746
 747     fn make_run(run: RunConfig) {
 748         run.builder.ensure(Src);
 749     }
 750
 751     /// Creates the `rust-src` installer component
 752     fn run(self, builder: &Builder) -> PathBuf {
 753         let build = builder.build;
 754         println!("Dist src");
 755
 756         let name = pkgname(build, "rust-src");
 757         let image = tmpdir(build).join(format!("{}-image", name));
 758         let _ = fs::remove_dir_all(&image);
 759
 760         let dst = image.join("lib/rustlib/src");
 761         let dst_src = dst.join("rust");
 762         t!(fs::create_dir_all(&dst_src));
 763
 764         let src_files = [
 765             "src/Cargo.lock",
 766         ];
 767         // This is the reduced set of paths which will become the rust-src component
 768         // (essentially libstd and all of its path dependencies)
 769         let std_src_dirs = [
 770             "src/build_helper",
 771             "src/dlmalloc",
 772             "src/liballoc",
 773             "src/liballoc_jemalloc",
 774             "src/liballoc_system",
 775             "src/libbacktrace",
 776             "src/libcompiler_builtins",
 777             "src/libcore",
 778             "src/liblibc",
 779             "src/libpanic_abort",
 780             "src/libpanic_unwind",
 781             "src/librustc_asan",
 782             "src/librustc_lsan",
 783             "src/librustc_msan",
 784             "src/librustc_tsan",
 785             "src/libstd",
 786             "src/libstd_unicode",
 787             "src/libunwind",
 788             "src/rustc/compiler_builtins_shim",
 789             "src/rustc/libc_shim",
 790             "src/rustc/dlmalloc_shim",
 791             "src/libtest",
 792             "src/libterm",
 793             "src/jemalloc",
 794             "src/libprofiler_builtins",
 795         ];
 796         let std_src_dirs_exclude = [
 797             "src/libcompiler_builtins/compiler-rt/test",
 798             "src/jemalloc/test/unit",
 799         ];
 800
 801         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 802         for file in src_files.iter() {
 803             copy(&build.src.join(file), &dst_src.join(file));
 804         }
 805
 806         // Create source tarball in rust-installer format
 807         let mut cmd = rust_installer(builder);
 808         cmd.arg("generate")
 809            .arg("--product-name=Rust")
 810            .arg("--rel-manifest-dir=rustlib")
 811            .arg("--success-message=Awesome-Source.")
 812            .arg("--image-dir").arg(&image)
 813            .arg("--work-dir").arg(&tmpdir(build))
 814            .arg("--output-dir").arg(&distdir(build))
 815            .arg(format!("--package-name={}", name))
 816            .arg("--component-name=rust-src")
 817            .arg("--legacy-manifest-dirs=rustlib,cargo");
 818         build.run(&mut cmd);
 819
 820         t!(fs::remove_dir_all(&image));
 821         distdir(build).join(&format!("{}.tar.gz", name))
 822     }
 823 }
 824
 825 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 826
 827 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 828 pub struct PlainSourceTarball;
 829
 830 impl Step for PlainSourceTarball {
 831     /// Produces the location of the tarball generated
 832     type Output = PathBuf;
 833     const DEFAULT: bool = true;
 834     const ONLY_HOSTS: bool = true;
 835     const ONLY_BUILD_TARGETS: bool = true;
 836     const ONLY_BUILD: bool = true;
 837
 838     fn should_run(run: ShouldRun) -> ShouldRun {
 839         let builder = run.builder;
 840         run.path("src").default_condition(builder.config.rust_dist_src)
 841     }
 842
 843     fn make_run(run: RunConfig) {
 844         run.builder.ensure(PlainSourceTarball);
 845     }
 846
 847     /// Creates the plain source tarball
 848     fn run(self, builder: &Builder) -> PathBuf {
 849         let build = builder.build;
 850         println!("Create plain source tarball");
 851
 852         // Make sure that the root folder of tarball has the correct name
 853         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 854         let plain_dst_src = tmpdir(build).join(&plain_name);
 855         let _ = fs::remove_dir_all(&plain_dst_src);
 856         t!(fs::create_dir_all(&plain_dst_src));
 857
 858         // This is the set of root paths which will become part of the source package
 859         let src_files = [
 860             "COPYRIGHT",
 861             "LICENSE-APACHE",
 862             "LICENSE-MIT",
 863             "CONTRIBUTING.md",
 864             "README.md",
 865             "RELEASES.md",
 866             "configure",
 867             "x.py",
 868             "config.toml.example",
 869         ];
 870         let src_dirs = [
 871             "src",
 872         ];
 873
 874         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 875
 876         // Copy the files normally
 877         for item in &src_files {
 878             copy(&build.src.join(item), &plain_dst_src.join(item));
 879         }
 880
 881         // Create the version file
 882         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 883         if let Some(sha) = build.rust_sha() {
 884             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 885         }
 886
 887         // If we're building from git sources, we need to vendor a complete distribution.
 888         if build.rust_info.is_git() {
 889             // Get cargo-vendor installed, if it isn't already.
 890             let mut has_cargo_vendor = false;
 891             let mut cmd = Command::new(&build.initial_cargo);
 892             for line in output(cmd.arg("install").arg("--list")).lines() {
 893                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 894             }
 895             if !has_cargo_vendor {
 896                 let mut cmd = Command::new(&build.initial_cargo);
 897                 cmd.arg("install")
 898                    .arg("--force")
 899                    .arg("--debug")
 900                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 901                    .arg("cargo-vendor")
 902                    .env("RUSTC", &build.initial_rustc);
 903                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 904                     builder.ensure(native::Openssl {
 905                         target: build.config.build,
 906                     });
 907                     cmd.env("OPENSSL_DIR", dir);
 908                 }
 909                 build.run(&mut cmd);
 910             }
 911
 912             // Vendor all Cargo dependencies
 913             let mut cmd = Command::new(&build.initial_cargo);
 914             cmd.arg("vendor")
 915                .current_dir(&plain_dst_src.join("src"));
 916             build.run(&mut cmd);
 917         }
 918
 919         // Create plain source tarball
 920         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 921         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 922         tarball.set_extension(""); // strip .gz
 923         tarball.set_extension(""); // strip .tar
 924         if let Some(dir) = tarball.parent() {
 925             t!(fs::create_dir_all(dir));
 926         }
 927         println!("running installer");
 928         let mut cmd = rust_installer(builder);
 929         cmd.arg("tarball")
 930            .arg("--input").arg(&plain_name)
 931            .arg("--output").arg(&tarball)
 932            .arg("--work-dir=.")
 933            .current_dir(tmpdir(build));
 934         build.run(&mut cmd);
 935         distdir(build).join(&format!("{}.tar.gz", plain_name))
 936     }
 937 }
 938
 939 fn install(src: &Path, dstdir: &Path, perms: u32) {
 940     let dst = dstdir.join(src.file_name().unwrap());
 941     t!(fs::create_dir_all(dstdir));
 942     drop(fs::remove_file(&dst));
 943     {
 944         let mut s = t!(fs::File::open(&src));
 945         let mut d = t!(fs::File::create(&dst));
 946         io::copy(&mut s, &mut d).expect("failed to copy");
 947     }
 948     chmod(&dst, perms);
 949 }
 950
 951 #[cfg(unix)]
 952 fn chmod(path: &Path, perms: u32) {
 953     use std::os::unix::fs::*;
 954     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 955 }
 956 #[cfg(windows)]
 957 fn chmod(_path: &Path, _perms: u32) {}
 958
 959 // We have to run a few shell scripts, which choke quite a bit on both `\`
 960 // characters and on `C:\` paths, so normalize both of them away.
 961 pub fn sanitize_sh(path: &Path) -> String {
 962     let path = path.to_str().unwrap().replace("\\", "/");
 963     return change_drive(&path).unwrap_or(path);
 964
 965     fn change_drive(s: &str) -> Option<String> {
 966         let mut ch = s.chars();
 967         let drive = ch.next().unwrap_or('C');
 968         if ch.next() != Some(':') {
 969             return None
 970         }
 971         if ch.next() != Some('/') {
 972             return None
 973         }
 974         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 975     }
 976 }
 977
 978 fn write_file(path: &Path, data: &[u8]) {
 979     let mut vf = t!(fs::File::create(path));
 980     t!(vf.write_all(data));
 981 }
 982
 983 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 984 pub struct Cargo {
 985     pub stage: u32,
 986     pub target: Interned<String>,
 987 }
 988
 989 impl Step for Cargo {
 990     type Output = PathBuf;
 991     const ONLY_BUILD_TARGETS: bool = true;
 992     const ONLY_HOSTS: bool = true;
 993
 994     fn should_run(run: ShouldRun) -> ShouldRun {
 995         run.path("cargo")
 996     }
 997
 998     fn make_run(run: RunConfig) {
 999         run.builder.ensure(Cargo {
1000             stage: run.builder.top_stage,
1001             target: run.target,
1002         });
1003     }
1004
1005     fn run(self, builder: &Builder) -> PathBuf {
1006         let build = builder.build;
1007         let stage = self.stage;
1008         let target = self.target;
1009
1010         println!("Dist cargo stage{} ({})", stage, target);
1011         let src = build.src.join("src/tools/cargo");
1012         let etc = src.join("src/etc");
1013         let release_num = build.release_num("cargo");
1014         let name = pkgname(build, "cargo");
1015         let version = builder.cargo_info.version(build, &release_num);
1016
1017         let tmp = tmpdir(build);
1018         let image = tmp.join("cargo-image");
1019         drop(fs::remove_dir_all(&image));
1020         t!(fs::create_dir_all(&image));
1021
1022         // Prepare the image directory
1023         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1024         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1025         let cargo = builder.ensure(tool::Cargo {
1026             compiler: builder.compiler(stage, build.build),
1027             target
1028         });
1029         install(&cargo, &image.join("bin"), 0o755);
1030         for man in t!(etc.join("man").read_dir()) {
1031             let man = t!(man);
1032             install(&man.path(), &image.join("share/man/man1"), 0o644);
1033         }
1034         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1035         copy(&etc.join("cargo.bashcomp.sh"),
1036              &image.join("etc/bash_completion.d/cargo"));
1037         let doc = image.join("share/doc/cargo");
1038         install(&src.join("README.md"), &doc, 0o644);
1039         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1040         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1041         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1042
1043         // Prepare the overlay
1044         let overlay = tmp.join("cargo-overlay");
1045         drop(fs::remove_dir_all(&overlay));
1046         t!(fs::create_dir_all(&overlay));
1047         install(&src.join("README.md"), &overlay, 0o644);
1048         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1049         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1050         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1051         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1052
1053         // Generate the installer tarball
1054         let mut cmd = rust_installer(builder);
1055         cmd.arg("generate")
1056            .arg("--product-name=Rust")
1057            .arg("--rel-manifest-dir=rustlib")
1058            .arg("--success-message=Rust-is-ready-to-roll.")
1059            .arg("--image-dir").arg(&image)
1060            .arg("--work-dir").arg(&tmpdir(build))
1061            .arg("--output-dir").arg(&distdir(build))
1062            .arg("--non-installed-overlay").arg(&overlay)
1063            .arg(format!("--package-name={}-{}", name, target))
1064            .arg("--component-name=cargo")
1065            .arg("--legacy-manifest-dirs=rustlib,cargo");
1066         build.run(&mut cmd);
1067         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1068     }
1069 }
1070
1071 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1072 pub struct Rls {
1073     pub stage: u32,
1074     pub target: Interned<String>,
1075 }
1076
1077 impl Step for Rls {
1078     type Output = Option<PathBuf>;
1079     const ONLY_BUILD_TARGETS: bool = true;
1080     const ONLY_HOSTS: bool = true;
1081
1082     fn should_run(run: ShouldRun) -> ShouldRun {
1083         run.path("rls")
1084     }
1085
1086     fn make_run(run: RunConfig) {
1087         run.builder.ensure(Rls {
1088             stage: run.builder.top_stage,
1089             target: run.target,
1090         });
1091     }
1092
1093     fn run(self, builder: &Builder) -> Option<PathBuf> {
1094         let build = builder.build;
1095         let stage = self.stage;
1096         let target = self.target;
1097         assert!(build.config.extended);
1098
1099         println!("Dist RLS stage{} ({})", stage, target);
1100         let src = build.src.join("src/tools/rls");
1101         let release_num = build.release_num("rls");
1102         let name = pkgname(build, "rls");
1103         let version = build.rls_info.version(build, &release_num);
1104
1105         let tmp = tmpdir(build);
1106         let image = tmp.join("rls-image");
1107         drop(fs::remove_dir_all(&image));
1108         t!(fs::create_dir_all(&image));
1109
1110         // Prepare the image directory
1111         // We expect RLS to build, because we've exited this step above if tool
1112         // state for RLS isn't testing.
1113         let rls = builder.ensure(tool::Rls {
1114             compiler: builder.compiler(stage, build.build),
1115             target
1116         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1117
1118         install(&rls, &image.join("bin"), 0o755);
1119         let doc = image.join("share/doc/rls");
1120         install(&src.join("README.md"), &doc, 0o644);
1121         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1122         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1123
1124         // Prepare the overlay
1125         let overlay = tmp.join("rls-overlay");
1126         drop(fs::remove_dir_all(&overlay));
1127         t!(fs::create_dir_all(&overlay));
1128         install(&src.join("README.md"), &overlay, 0o644);
1129         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1130         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1131         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1132
1133         // Generate the installer tarball
1134         let mut cmd = rust_installer(builder);
1135         cmd.arg("generate")
1136            .arg("--product-name=Rust")
1137            .arg("--rel-manifest-dir=rustlib")
1138            .arg("--success-message=RLS-ready-to-serve.")
1139            .arg("--image-dir").arg(&image)
1140            .arg("--work-dir").arg(&tmpdir(build))
1141            .arg("--output-dir").arg(&distdir(build))
1142            .arg("--non-installed-overlay").arg(&overlay)
1143            .arg(format!("--package-name={}-{}", name, target))
1144            .arg("--legacy-manifest-dirs=rustlib,cargo")
1145            .arg("--component-name=rls-preview");
1146
1147         build.run(&mut cmd);
1148         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1149     }
1150 }
1151
1152
1153 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1154 pub struct Rustfmt {
1155     pub stage: u32,
1156     pub target: Interned<String>,
1157 }
1158
1159 impl Step for Rustfmt {
1160     type Output = Option<PathBuf>;
1161     const ONLY_BUILD_TARGETS: bool = true;
1162     const ONLY_HOSTS: bool = true;
1163
1164     fn should_run(run: ShouldRun) -> ShouldRun {
1165         run.path("rustfmt")
1166     }
1167
1168     fn make_run(run: RunConfig) {
1169         run.builder.ensure(Rustfmt {
1170             stage: run.builder.top_stage,
1171             target: run.target,
1172         });
1173     }
1174
1175     fn run(self, builder: &Builder) -> Option<PathBuf> {
1176         let build = builder.build;
1177         let stage = self.stage;
1178         let target = self.target;
1179         assert!(build.config.extended);
1180
1181         println!("Dist Rustfmt stage{} ({})", stage, target);
1182         let src = build.src.join("src/tools/rustfmt");
1183         let release_num = build.release_num("rustfmt");
1184         let name = pkgname(build, "rustfmt");
1185         let version = build.rustfmt_info.version(build, &release_num);
1186
1187         let tmp = tmpdir(build);
1188         let image = tmp.join("rustfmt-image");
1189         drop(fs::remove_dir_all(&image));
1190         t!(fs::create_dir_all(&image));
1191
1192         // Prepare the image directory
1193         let rustfmt = builder.ensure(tool::Rustfmt {
1194             compiler: builder.compiler(stage, build.build),
1195             target
1196         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1197         let cargofmt = builder.ensure(tool::Cargofmt {
1198             compiler: builder.compiler(stage, build.build),
1199             target
1200         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1201
1202         install(&rustfmt, &image.join("bin"), 0o755);
1203         install(&cargofmt, &image.join("bin"), 0o755);
1204         let doc = image.join("share/doc/rustfmt");
1205         install(&src.join("README.md"), &doc, 0o644);
1206         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1207         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1208
1209         // Prepare the overlay
1210         let overlay = tmp.join("rustfmt-overlay");
1211         drop(fs::remove_dir_all(&overlay));
1212         t!(fs::create_dir_all(&overlay));
1213         install(&src.join("README.md"), &overlay, 0o644);
1214         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1215         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1216         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1217
1218         // Generate the installer tarball
1219         let mut cmd = rust_installer(builder);
1220         cmd.arg("generate")
1221            .arg("--product-name=Rust")
1222            .arg("--rel-manifest-dir=rustlib")
1223            .arg("--success-message=rustfmt-ready-to-fmt.")
1224            .arg("--image-dir").arg(&image)
1225            .arg("--work-dir").arg(&tmpdir(build))
1226            .arg("--output-dir").arg(&distdir(build))
1227            .arg("--non-installed-overlay").arg(&overlay)
1228            .arg(format!("--package-name={}-{}", name, target))
1229            .arg("--legacy-manifest-dirs=rustlib,cargo")
1230            .arg("--component-name=rustfmt-preview");
1231
1232         build.run(&mut cmd);
1233         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1234     }
1235 }
1236
1237 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1238 pub struct Extended {
1239     stage: u32,
1240     host: Interned<String>,
1241     target: Interned<String>,
1242 }
1243
1244 impl Step for Extended {
1245     type Output = ();
1246     const DEFAULT: bool = true;
1247     const ONLY_BUILD_TARGETS: bool = true;
1248     const ONLY_HOSTS: bool = true;
1249
1250     fn should_run(run: ShouldRun) -> ShouldRun {
1251         let builder = run.builder;
1252         run.path("extended").default_condition(builder.config.extended)
1253     }
1254
1255     fn make_run(run: RunConfig) {
1256         run.builder.ensure(Extended {
1257             stage: run.builder.top_stage,
1258             host: run.host,
1259             target: run.target,
1260         });
1261     }
1262
1263     /// Creates a combined installer for the specified target in the provided stage.
1264     fn run(self, builder: &Builder) {
1265         let build = builder.build;
1266         let stage = self.stage;
1267         let target = self.target;
1268
1269         println!("Dist extended stage{} ({})", stage, target);
1270
1271         let rustc_installer = builder.ensure(Rustc {
1272             compiler: builder.compiler(stage, target),
1273         });
1274         let cargo_installer = builder.ensure(Cargo { stage, target });
1275         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1276         let rls_installer = builder.ensure(Rls { stage, target });
1277         let mingw_installer = builder.ensure(Mingw { host: target });
1278         let analysis_installer = builder.ensure(Analysis {
1279             compiler: builder.compiler(stage, self.host),
1280             target
1281         });
1282
1283         let docs_installer = builder.ensure(Docs { stage, host: target, });
1284         let std_installer = builder.ensure(Std {
1285             compiler: builder.compiler(stage, self.host),
1286             target,
1287         });
1288
1289         let tmp = tmpdir(build);
1290         let overlay = tmp.join("extended-overlay");
1291         let etc = build.src.join("src/etc/installer");
1292         let work = tmp.join("work");
1293
1294         let _ = fs::remove_dir_all(&overlay);
1295         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1296         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1297         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1298         let version = build.rust_version();
1299         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1300         if let Some(sha) = build.rust_sha() {
1301             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1302         }
1303         install(&etc.join("README.md"), &overlay, 0o644);
1304
1305         // When rust-std package split from rustc, we needed to ensure that during
1306         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1307         // the std files during uninstall. To do this ensure that rustc comes
1308         // before rust-std in the list below.
1309         let mut tarballs = Vec::new();
1310         tarballs.push(rustc_installer);
1311         tarballs.push(cargo_installer);
1312         tarballs.extend(rls_installer.clone());
1313         tarballs.extend(rustfmt_installer.clone());
1314         tarballs.push(analysis_installer);
1315         tarballs.push(std_installer);
1316         if build.config.docs {
1317             tarballs.push(docs_installer);
1318         }
1319         if target.contains("pc-windows-gnu") {
1320             tarballs.push(mingw_installer.unwrap());
1321         }
1322         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1323         for tarball in &tarballs[1..] {
1324             input_tarballs.push(",");
1325             input_tarballs.push(tarball);
1326         }
1327
1328         let mut cmd = rust_installer(builder);
1329         cmd.arg("combine")
1330             .arg("--product-name=Rust")
1331             .arg("--rel-manifest-dir=rustlib")
1332             .arg("--success-message=Rust-is-ready-to-roll.")
1333             .arg("--work-dir").arg(&work)
1334             .arg("--output-dir").arg(&distdir(build))
1335             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1336             .arg("--legacy-manifest-dirs=rustlib,cargo")
1337             .arg("--input-tarballs").arg(input_tarballs)
1338             .arg("--non-installed-overlay").arg(&overlay);
1339         build.run(&mut cmd);
1340
1341         let mut license = String::new();
1342         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1343         license.push_str("\n");
1344         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1345         license.push_str("\n");
1346         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1347
1348         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1349         let mut rtf = rtf.to_string();
1350         rtf.push_str("\n");
1351         for line in license.lines() {
1352             rtf.push_str(line);
1353             rtf.push_str("\\line ");
1354         }
1355         rtf.push_str("}");
1356
1357         fn filter(contents: &str, marker: &str) -> String {
1358             let start = format!("tool-{}-start", marker);
1359             let end = format!("tool-{}-end", marker);
1360             let mut lines = Vec::new();
1361             let mut omitted = false;
1362             for line in contents.lines() {
1363                 if line.contains(&start) {
1364                     omitted = true;
1365                 } else if line.contains(&end) {
1366                     omitted = false;
1367                 } else if !omitted {
1368                     lines.push(line);
1369                 }
1370             }
1371
1372             lines.join("\n")
1373         }
1374
1375         let xform = |p: &Path| {
1376             let mut contents = String::new();
1377             t!(t!(File::open(p)).read_to_string(&mut contents));
1378             if rls_installer.is_none() {
1379                 contents = filter(&contents, "rls");
1380             }
1381             if rustfmt_installer.is_none() {
1382                 contents = filter(&contents, "rustfmt");
1383             }
1384             let ret = tmp.join(p.file_name().unwrap());
1385             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1386             return ret
1387         };
1388
1389         if target.contains("apple-darwin") {
1390             let pkg = tmp.join("pkg");
1391             let _ = fs::remove_dir_all(&pkg);
1392
1393             let pkgbuild = |component: &str| {
1394                 let mut cmd = Command::new("pkgbuild");
1395                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1396                     .arg("--scripts").arg(pkg.join(component))
1397                     .arg("--nopayload")
1398                     .arg(pkg.join(component).with_extension("pkg"));
1399                 build.run(&mut cmd);
1400             };
1401
1402             let prepare = |name: &str| {
1403                 t!(fs::create_dir_all(pkg.join(name)));
1404                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1405                         &pkg.join(name));
1406                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1407                 pkgbuild(name);
1408             };
1409             prepare("rustc");
1410             prepare("cargo");
1411             prepare("rust-docs");
1412             prepare("rust-std");
1413             prepare("rust-analysis");
1414
1415             if rls_installer.is_some() {
1416                 prepare("rls");
1417             }
1418
1419             // create an 'uninstall' package
1420             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1421             pkgbuild("uninstall");
1422
1423             t!(fs::create_dir_all(pkg.join("res")));
1424             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1425             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1426             let mut cmd = Command::new("productbuild");
1427             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1428                 .arg("--resources").arg(pkg.join("res"))
1429                 .arg(distdir(build).join(format!("{}-{}.pkg",
1430                                                     pkgname(build, "rust"),
1431                                                     target)))
1432                 .arg("--package-path").arg(&pkg);
1433             build.run(&mut cmd);
1434         }
1435
1436         if target.contains("windows") {
1437             let exe = tmp.join("exe");
1438             let _ = fs::remove_dir_all(&exe);
1439
1440             let prepare = |name: &str| {
1441                 t!(fs::create_dir_all(exe.join(name)));
1442                 let dir = if name == "rust-std" || name == "rust-analysis" {
1443                     format!("{}-{}", name, target)
1444                 } else if name == "rls" {
1445                     "rls-preview".to_string()
1446                 } else {
1447                     name.to_string()
1448                 };
1449                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1450                             .join(dir),
1451                         &exe.join(name));
1452                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1453             };
1454             prepare("rustc");
1455             prepare("cargo");
1456             prepare("rust-analysis");
1457             prepare("rust-docs");
1458             prepare("rust-std");
1459             if rls_installer.is_some() {
1460                 prepare("rls");
1461             }
1462             if target.contains("windows-gnu") {
1463                 prepare("rust-mingw");
1464             }
1465
1466             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1467             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1468             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1469             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1470             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1471
1472             // Generate exe installer
1473             let mut cmd = Command::new("iscc");
1474             cmd.arg("rust.iss")
1475                 .current_dir(&exe);
1476             if target.contains("windows-gnu") {
1477                 cmd.arg("/dMINGW");
1478             }
1479             add_env(build, &mut cmd, target);
1480             build.run(&mut cmd);
1481             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1482                     &distdir(build),
1483                     0o755);
1484
1485             // Generate msi installer
1486             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1487             let heat = wix.join("bin/heat.exe");
1488             let candle = wix.join("bin/candle.exe");
1489             let light = wix.join("bin/light.exe");
1490
1491             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1492             build.run(Command::new(&heat)
1493                             .current_dir(&exe)
1494                             .arg("dir")
1495                             .arg("rustc")
1496                             .args(&heat_flags)
1497                             .arg("-cg").arg("RustcGroup")
1498                             .arg("-dr").arg("Rustc")
1499                             .arg("-var").arg("var.RustcDir")
1500                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1501             build.run(Command::new(&heat)
1502                             .current_dir(&exe)
1503                             .arg("dir")
1504                             .arg("rust-docs")
1505                             .args(&heat_flags)
1506                             .arg("-cg").arg("DocsGroup")
1507                             .arg("-dr").arg("Docs")
1508                             .arg("-var").arg("var.DocsDir")
1509                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1510                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1511             build.run(Command::new(&heat)
1512                             .current_dir(&exe)
1513                             .arg("dir")
1514                             .arg("cargo")
1515                             .args(&heat_flags)
1516                             .arg("-cg").arg("CargoGroup")
1517                             .arg("-dr").arg("Cargo")
1518                             .arg("-var").arg("var.CargoDir")
1519                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1520                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1521             build.run(Command::new(&heat)
1522                             .current_dir(&exe)
1523                             .arg("dir")
1524                             .arg("rust-std")
1525                             .args(&heat_flags)
1526                             .arg("-cg").arg("StdGroup")
1527                             .arg("-dr").arg("Std")
1528                             .arg("-var").arg("var.StdDir")
1529                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1530             if rls_installer.is_some() {
1531                 build.run(Command::new(&heat)
1532                                 .current_dir(&exe)
1533                                 .arg("dir")
1534                                 .arg("rls")
1535                                 .args(&heat_flags)
1536                                 .arg("-cg").arg("RlsGroup")
1537                                 .arg("-dr").arg("Rls")
1538                                 .arg("-var").arg("var.RlsDir")
1539                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1540                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1541             }
1542             build.run(Command::new(&heat)
1543                             .current_dir(&exe)
1544                             .arg("dir")
1545                             .arg("rust-analysis")
1546                             .args(&heat_flags)
1547                             .arg("-cg").arg("AnalysisGroup")
1548                             .arg("-dr").arg("Analysis")
1549                             .arg("-var").arg("var.AnalysisDir")
1550                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1551                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1552             if target.contains("windows-gnu") {
1553                 build.run(Command::new(&heat)
1554                                 .current_dir(&exe)
1555                                 .arg("dir")
1556                                 .arg("rust-mingw")
1557                                 .args(&heat_flags)
1558                                 .arg("-cg").arg("GccGroup")
1559                                 .arg("-dr").arg("Gcc")
1560                                 .arg("-var").arg("var.GccDir")
1561                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1562             }
1563
1564             let candle = |input: &Path| {
1565                 let output = exe.join(input.file_stem().unwrap())
1566                                 .with_extension("wixobj");
1567                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1568                 let mut cmd = Command::new(&candle);
1569                 cmd.current_dir(&exe)
1570                     .arg("-nologo")
1571                     .arg("-dRustcDir=rustc")
1572                     .arg("-dDocsDir=rust-docs")
1573                     .arg("-dCargoDir=cargo")
1574                     .arg("-dStdDir=rust-std")
1575                     .arg("-dAnalysisDir=rust-analysis")
1576                     .arg("-arch").arg(&arch)
1577                     .arg("-out").arg(&output)
1578                     .arg(&input);
1579                 add_env(build, &mut cmd, target);
1580
1581                 if rls_installer.is_some() {
1582                     cmd.arg("-dRlsDir=rls");
1583                 }
1584                 if target.contains("windows-gnu") {
1585                     cmd.arg("-dGccDir=rust-mingw");
1586                 }
1587                 build.run(&mut cmd);
1588             };
1589             candle(&xform(&etc.join("msi/rust.wxs")));
1590             candle(&etc.join("msi/ui.wxs"));
1591             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1592             candle("RustcGroup.wxs".as_ref());
1593             candle("DocsGroup.wxs".as_ref());
1594             candle("CargoGroup.wxs".as_ref());
1595             candle("StdGroup.wxs".as_ref());
1596             if rls_installer.is_some() {
1597                 candle("RlsGroup.wxs".as_ref());
1598             }
1599             candle("AnalysisGroup.wxs".as_ref());
1600
1601             if target.contains("windows-gnu") {
1602                 candle("GccGroup.wxs".as_ref());
1603             }
1604
1605             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1606             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1607             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1608
1609             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1610             let mut cmd = Command::new(&light);
1611             cmd.arg("-nologo")
1612                 .arg("-ext").arg("WixUIExtension")
1613                 .arg("-ext").arg("WixUtilExtension")
1614                 .arg("-out").arg(exe.join(&filename))
1615                 .arg("rust.wixobj")
1616                 .arg("ui.wixobj")
1617                 .arg("rustwelcomedlg.wixobj")
1618                 .arg("RustcGroup.wixobj")
1619                 .arg("DocsGroup.wixobj")
1620                 .arg("CargoGroup.wixobj")
1621                 .arg("StdGroup.wixobj")
1622                 .arg("AnalysisGroup.wixobj")
1623                 .current_dir(&exe);
1624
1625             if rls_installer.is_some() {
1626                 cmd.arg("RlsGroup.wixobj");
1627             }
1628
1629             if target.contains("windows-gnu") {
1630                 cmd.arg("GccGroup.wixobj");
1631             }
1632             // ICE57 wrongly complains about the shortcuts
1633             cmd.arg("-sice:ICE57");
1634
1635             build.run(&mut cmd);
1636
1637             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1638         }
1639     }
1640 }
1641
1642 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1643     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1644     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1645        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1646        .env("CFG_RELEASE", build.rust_release())
1647        .env("CFG_VER_MAJOR", parts.next().unwrap())
1648        .env("CFG_VER_MINOR", parts.next().unwrap())
1649        .env("CFG_VER_PATCH", parts.next().unwrap())
1650        .env("CFG_VER_BUILD", "0") // just needed to build
1651        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1652        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1653        .env("CFG_BUILD", target)
1654        .env("CFG_CHANNEL", &build.config.channel);
1655
1656     if target.contains("windows-gnu") {
1657        cmd.env("CFG_MINGW", "1")
1658           .env("CFG_ABI", "GNU");
1659     } else {
1660        cmd.env("CFG_MINGW", "0")
1661           .env("CFG_ABI", "MSVC");
1662     }
1663
1664     if target.contains("x86_64") {
1665        cmd.env("CFG_PLATFORM", "x64");
1666     } else {
1667        cmd.env("CFG_PLATFORM", "x86");
1668     }
1669 }
1670
1671 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1672 pub struct HashSign;
1673
1674 impl Step for HashSign {
1675     type Output = ();
1676     const ONLY_BUILD_TARGETS: bool = true;
1677     const ONLY_HOSTS: bool = true;
1678     const ONLY_BUILD: bool = true;
1679
1680     fn should_run(run: ShouldRun) -> ShouldRun {
1681         run.path("hash-and-sign")
1682     }
1683
1684     fn make_run(run: RunConfig) {
1685         run.builder.ensure(HashSign);
1686     }
1687
1688     fn run(self, builder: &Builder) {
1689         let build = builder.build;
1690         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1691         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1692             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1693         });
1694         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1695             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1696         });
1697         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1698             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1699         });
1700         let mut pass = String::new();
1701         t!(t!(File::open(&file)).read_to_string(&mut pass));
1702
1703         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1704
1705         cmd.arg(sign);
1706         cmd.arg(distdir(build));
1707         cmd.arg(today.trim());
1708         cmd.arg(build.rust_package_vers());
1709         cmd.arg(build.package_vers(&build.release_num("cargo")));
1710         cmd.arg(build.package_vers(&build.release_num("rls")));
1711         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1712         cmd.arg(addr);
1713
1714         t!(fs::create_dir_all(distdir(build)));
1715
1716         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1717         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1718         let status = t!(child.wait());
1719         assert!(status.success());
1720     }
1721 }