src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Copy over lld if it's there
 447             if builder.config.lld_enabled {
 448                 let exe = exe("lld", &compiler.host);
 449                 let src = builder.sysroot_libdir(compiler, host)
 450                     .parent()
 451                     .unwrap()
 452                     .join("bin")
 453                     .join(&exe);
 454                 let dst = image.join("lib/rustlib")
 455                     .join(&*host)
 456                     .join("bin")
 457                     .join(&exe);
 458                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 459                 copy(&src, &dst);
 460             }
 461
 462             // Man pages
 463             t!(fs::create_dir_all(image.join("share/man/man1")));
 464             let man_src = build.src.join("src/doc/man");
 465             let man_dst = image.join("share/man/man1");
 466             let month_year = t!(time::strftime("%B %Y", &time::now()));
 467             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 468             // to hardlink, and we don't want to edit the source templates
 469             for entry_result in t!(fs::read_dir(man_src)) {
 470                 let file_entry = t!(entry_result);
 471                 let page_src = file_entry.path();
 472                 let page_dst = man_dst.join(file_entry.file_name());
 473                 t!(fs::copy(&page_src, &page_dst));
 474                 // template in month/year and version number
 475                 replace_in_file(&page_dst,
 476                                 &[("<INSERT DATE HERE>", &month_year),
 477                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 478             }
 479
 480             // Debugger scripts
 481             builder.ensure(DebuggerScripts {
 482                 sysroot: INTERNER.intern_path(image.to_owned()),
 483                 host,
 484             });
 485
 486             // Misc license info
 487             let cp = |file: &str| {
 488                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 489             };
 490             cp("COPYRIGHT");
 491             cp("LICENSE-APACHE");
 492             cp("LICENSE-MIT");
 493             cp("README.md");
 494         }
 495     }
 496 }
 497
 498 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 499 pub struct DebuggerScripts {
 500     pub sysroot: Interned<PathBuf>,
 501     pub host: Interned<String>,
 502 }
 503
 504 impl Step for DebuggerScripts {
 505     type Output = ();
 506
 507     fn should_run(run: ShouldRun) -> ShouldRun {
 508         run.path("src/lldb_batchmode.py")
 509     }
 510
 511     fn make_run(run: RunConfig) {
 512         run.builder.ensure(DebuggerScripts {
 513             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 514             host: run.target,
 515         });
 516     }
 517
 518     /// Copies debugger scripts for `target` into the `sysroot` specified.
 519     fn run(self, builder: &Builder) {
 520         let build = builder.build;
 521         let host = self.host;
 522         let sysroot = self.sysroot;
 523         let dst = sysroot.join("lib/rustlib/etc");
 524         t!(fs::create_dir_all(&dst));
 525         let cp_debugger_script = |file: &str| {
 526             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 527         };
 528         if host.contains("windows-msvc") {
 529             // windbg debugger scripts
 530             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 531                 0o755);
 532
 533             cp_debugger_script("natvis/intrinsic.natvis");
 534             cp_debugger_script("natvis/liballoc.natvis");
 535             cp_debugger_script("natvis/libcore.natvis");
 536         } else {
 537             cp_debugger_script("debugger_pretty_printers_common.py");
 538
 539             // gdb debugger scripts
 540             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 541                     0o755);
 542
 543             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 544             cp_debugger_script("gdb_rust_pretty_printing.py");
 545
 546             // lldb debugger scripts
 547             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 548                     0o755);
 549
 550             cp_debugger_script("lldb_rust_formatters.py");
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: Interned<String>,
 559 }
 560
 561 impl Step for Std {
 562     type Output = PathBuf;
 563     const DEFAULT: bool = true;
 564     const ONLY_BUILD_TARGETS: bool = true;
 565
 566     fn should_run(run: ShouldRun) -> ShouldRun {
 567         run.path("src/libstd")
 568     }
 569
 570     fn make_run(run: RunConfig) {
 571         run.builder.ensure(Std {
 572             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 573             target: run.target,
 574         });
 575     }
 576
 577     fn run(self, builder: &Builder) -> PathBuf {
 578         let build = builder.build;
 579         let compiler = self.compiler;
 580         let target = self.target;
 581
 582         let name = pkgname(build, "rust-std");
 583         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 584
 585         // The only true set of target libraries came from the build triple, so
 586         // let's reduce redundant work by only producing archives from that host.
 587         if compiler.host != build.build {
 588             println!("\tskipping, not a build host");
 589             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 590         }
 591
 592         // We want to package up as many target libraries as possible
 593         // for the `rust-std` package, so if this is a host target we
 594         // depend on librustc and otherwise we just depend on libtest.
 595         if build.hosts.iter().any(|t| t == target) {
 596             builder.ensure(compile::Rustc { compiler, target });
 597         } else {
 598             builder.ensure(compile::Test { compiler, target });
 599         }
 600
 601         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 602         let _ = fs::remove_dir_all(&image);
 603
 604         let dst = image.join("lib/rustlib").join(target);
 605         t!(fs::create_dir_all(&dst));
 606         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 607         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 608         cp_filtered(&src, &dst, &|path| {
 609             let name = path.file_name().and_then(|s| s.to_str());
 610             name != Some(build.config.rust_codegen_backends_dir.as_str()) &&
 611                 name != Some("bin")
 612
 613         });
 614
 615         let mut cmd = rust_installer(builder);
 616         cmd.arg("generate")
 617            .arg("--product-name=Rust")
 618            .arg("--rel-manifest-dir=rustlib")
 619            .arg("--success-message=std-is-standing-at-the-ready.")
 620            .arg("--image-dir").arg(&image)
 621            .arg("--work-dir").arg(&tmpdir(build))
 622            .arg("--output-dir").arg(&distdir(build))
 623            .arg(format!("--package-name={}-{}", name, target))
 624            .arg(format!("--component-name=rust-std-{}", target))
 625            .arg("--legacy-manifest-dirs=rustlib,cargo");
 626         build.run(&mut cmd);
 627         t!(fs::remove_dir_all(&image));
 628         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: Interned<String>,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = PathBuf;
 640     const DEFAULT: bool = true;
 641     const ONLY_BUILD_TARGETS: bool = true;
 642
 643     fn should_run(run: ShouldRun) -> ShouldRun {
 644         let builder = run.builder;
 645         run.path("analysis").default_condition(builder.build.config.extended)
 646     }
 647
 648     fn make_run(run: RunConfig) {
 649         run.builder.ensure(Analysis {
 650             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 651             target: run.target,
 652         });
 653     }
 654
 655     /// Creates a tarball of save-analysis metadata, if available.
 656     fn run(self, builder: &Builder) -> PathBuf {
 657         let build = builder.build;
 658         let compiler = self.compiler;
 659         let target = self.target;
 660         assert!(build.config.extended);
 661         println!("Dist analysis");
 662         let name = pkgname(build, "rust-analysis");
 663
 664         if &compiler.host != build.build {
 665             println!("\tskipping, not a build host");
 666             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 667         }
 668
 669         builder.ensure(Std { compiler, target });
 670
 671         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 672         // stage2 artifacts is simply copied from stage1 in that case.
 673         let compiler = if build.force_use_stage1(compiler, target) {
 674             builder.compiler(1, compiler.host)
 675         } else {
 676             compiler.clone()
 677         };
 678
 679         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 680
 681         let src = build.stage_out(compiler, Mode::Libstd)
 682             .join(target).join(build.cargo_dir()).join("deps");
 683
 684         let image_src = src.join("save-analysis");
 685         let dst = image.join("lib/rustlib").join(target).join("analysis");
 686         t!(fs::create_dir_all(&dst));
 687         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 688         cp_r(&image_src, &dst);
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=save-analysis-saved.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(build))
 697            .arg("--output-dir").arg(&distdir(build))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-analysis-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         build.run(&mut cmd);
 702         t!(fs::remove_dir_all(&image));
 703         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 708     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 709         let spath = match path.to_str() {
 710             Some(path) => path,
 711             None => return false,
 712         };
 713         if spath.ends_with("~") || spath.ends_with(".pyc") {
 714             return false
 715         }
 716         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 717             (spath.ends_with(".ll") ||
 718              spath.ends_with(".td") ||
 719              spath.ends_with(".s")) {
 720             return false
 721         }
 722         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 723             return false
 724         }
 725
 726         let full_path = Path::new(dir).join(path);
 727         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 728             return false;
 729         }
 730
 731         let excludes = [
 732             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 733             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 734             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 735             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 736         ];
 737         !path.iter()
 738              .map(|s| s.to_str().unwrap())
 739              .any(|s| excludes.contains(&s))
 740     }
 741
 742     // Copy the directories using our filter
 743     for item in src_dirs {
 744         let dst = &dst_dir.join(item);
 745         t!(fs::create_dir_all(dst));
 746         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 747     }
 748 }
 749
 750 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 751 pub struct Src;
 752
 753 impl Step for Src {
 754     /// The output path of the src installer tarball
 755     type Output = PathBuf;
 756     const DEFAULT: bool = true;
 757     const ONLY_HOSTS: bool = true;
 758     const ONLY_BUILD_TARGETS: bool = true;
 759     const ONLY_BUILD: bool = true;
 760
 761     fn should_run(run: ShouldRun) -> ShouldRun {
 762         run.path("src")
 763     }
 764
 765     fn make_run(run: RunConfig) {
 766         run.builder.ensure(Src);
 767     }
 768
 769     /// Creates the `rust-src` installer component
 770     fn run(self, builder: &Builder) -> PathBuf {
 771         let build = builder.build;
 772         println!("Dist src");
 773
 774         let name = pkgname(build, "rust-src");
 775         let image = tmpdir(build).join(format!("{}-image", name));
 776         let _ = fs::remove_dir_all(&image);
 777
 778         let dst = image.join("lib/rustlib/src");
 779         let dst_src = dst.join("rust");
 780         t!(fs::create_dir_all(&dst_src));
 781
 782         let src_files = [
 783             "src/Cargo.lock",
 784         ];
 785         // This is the reduced set of paths which will become the rust-src component
 786         // (essentially libstd and all of its path dependencies)
 787         let std_src_dirs = [
 788             "src/build_helper",
 789             "src/dlmalloc",
 790             "src/liballoc",
 791             "src/liballoc_jemalloc",
 792             "src/liballoc_system",
 793             "src/libbacktrace",
 794             "src/libcompiler_builtins",
 795             "src/libcore",
 796             "src/liblibc",
 797             "src/libpanic_abort",
 798             "src/libpanic_unwind",
 799             "src/librustc_asan",
 800             "src/librustc_lsan",
 801             "src/librustc_msan",
 802             "src/librustc_tsan",
 803             "src/libstd",
 804             "src/libstd_unicode",
 805             "src/libunwind",
 806             "src/rustc/compiler_builtins_shim",
 807             "src/rustc/libc_shim",
 808             "src/rustc/dlmalloc_shim",
 809             "src/libtest",
 810             "src/libterm",
 811             "src/jemalloc",
 812             "src/libprofiler_builtins",
 813             "src/stdsimd",
 814         ];
 815         let std_src_dirs_exclude = [
 816             "src/libcompiler_builtins/compiler-rt/test",
 817             "src/jemalloc/test/unit",
 818         ];
 819
 820         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 821         for file in src_files.iter() {
 822             copy(&build.src.join(file), &dst_src.join(file));
 823         }
 824
 825         // Create source tarball in rust-installer format
 826         let mut cmd = rust_installer(builder);
 827         cmd.arg("generate")
 828            .arg("--product-name=Rust")
 829            .arg("--rel-manifest-dir=rustlib")
 830            .arg("--success-message=Awesome-Source.")
 831            .arg("--image-dir").arg(&image)
 832            .arg("--work-dir").arg(&tmpdir(build))
 833            .arg("--output-dir").arg(&distdir(build))
 834            .arg(format!("--package-name={}", name))
 835            .arg("--component-name=rust-src")
 836            .arg("--legacy-manifest-dirs=rustlib,cargo");
 837         build.run(&mut cmd);
 838
 839         t!(fs::remove_dir_all(&image));
 840         distdir(build).join(&format!("{}.tar.gz", name))
 841     }
 842 }
 843
 844 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 845
 846 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 847 pub struct PlainSourceTarball;
 848
 849 impl Step for PlainSourceTarball {
 850     /// Produces the location of the tarball generated
 851     type Output = PathBuf;
 852     const DEFAULT: bool = true;
 853     const ONLY_HOSTS: bool = true;
 854     const ONLY_BUILD_TARGETS: bool = true;
 855     const ONLY_BUILD: bool = true;
 856
 857     fn should_run(run: ShouldRun) -> ShouldRun {
 858         let builder = run.builder;
 859         run.path("src").default_condition(builder.config.rust_dist_src)
 860     }
 861
 862     fn make_run(run: RunConfig) {
 863         run.builder.ensure(PlainSourceTarball);
 864     }
 865
 866     /// Creates the plain source tarball
 867     fn run(self, builder: &Builder) -> PathBuf {
 868         let build = builder.build;
 869         println!("Create plain source tarball");
 870
 871         // Make sure that the root folder of tarball has the correct name
 872         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 873         let plain_dst_src = tmpdir(build).join(&plain_name);
 874         let _ = fs::remove_dir_all(&plain_dst_src);
 875         t!(fs::create_dir_all(&plain_dst_src));
 876
 877         // This is the set of root paths which will become part of the source package
 878         let src_files = [
 879             "COPYRIGHT",
 880             "LICENSE-APACHE",
 881             "LICENSE-MIT",
 882             "CONTRIBUTING.md",
 883             "README.md",
 884             "RELEASES.md",
 885             "configure",
 886             "x.py",
 887             "config.toml.example",
 888         ];
 889         let src_dirs = [
 890             "src",
 891         ];
 892
 893         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 894
 895         // Copy the files normally
 896         for item in &src_files {
 897             copy(&build.src.join(item), &plain_dst_src.join(item));
 898         }
 899
 900         // Create the version file
 901         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 902         if let Some(sha) = build.rust_sha() {
 903             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 904         }
 905
 906         // If we're building from git sources, we need to vendor a complete distribution.
 907         if build.rust_info.is_git() {
 908             // Get cargo-vendor installed, if it isn't already.
 909             let mut has_cargo_vendor = false;
 910             let mut cmd = Command::new(&build.initial_cargo);
 911             for line in output(cmd.arg("install").arg("--list")).lines() {
 912                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 913             }
 914             if !has_cargo_vendor {
 915                 let mut cmd = Command::new(&build.initial_cargo);
 916                 cmd.arg("install")
 917                    .arg("--force")
 918                    .arg("--debug")
 919                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 920                    .arg("cargo-vendor")
 921                    .env("RUSTC", &build.initial_rustc);
 922                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 923                     builder.ensure(native::Openssl {
 924                         target: build.config.build,
 925                     });
 926                     cmd.env("OPENSSL_DIR", dir);
 927                 }
 928                 build.run(&mut cmd);
 929             }
 930
 931             // Vendor all Cargo dependencies
 932             let mut cmd = Command::new(&build.initial_cargo);
 933             cmd.arg("vendor")
 934                .current_dir(&plain_dst_src.join("src"));
 935             build.run(&mut cmd);
 936         }
 937
 938         // Create plain source tarball
 939         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 940         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 941         tarball.set_extension(""); // strip .gz
 942         tarball.set_extension(""); // strip .tar
 943         if let Some(dir) = tarball.parent() {
 944             t!(fs::create_dir_all(dir));
 945         }
 946         println!("running installer");
 947         let mut cmd = rust_installer(builder);
 948         cmd.arg("tarball")
 949            .arg("--input").arg(&plain_name)
 950            .arg("--output").arg(&tarball)
 951            .arg("--work-dir=.")
 952            .current_dir(tmpdir(build));
 953         build.run(&mut cmd);
 954         distdir(build).join(&format!("{}.tar.gz", plain_name))
 955     }
 956 }
 957
 958 fn install(src: &Path, dstdir: &Path, perms: u32) {
 959     let dst = dstdir.join(src.file_name().unwrap());
 960     t!(fs::create_dir_all(dstdir));
 961     drop(fs::remove_file(&dst));
 962     {
 963         let mut s = t!(fs::File::open(&src));
 964         let mut d = t!(fs::File::create(&dst));
 965         io::copy(&mut s, &mut d).expect("failed to copy");
 966     }
 967     chmod(&dst, perms);
 968 }
 969
 970 #[cfg(unix)]
 971 fn chmod(path: &Path, perms: u32) {
 972     use std::os::unix::fs::*;
 973     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 974 }
 975 #[cfg(windows)]
 976 fn chmod(_path: &Path, _perms: u32) {}
 977
 978 // We have to run a few shell scripts, which choke quite a bit on both `\`
 979 // characters and on `C:\` paths, so normalize both of them away.
 980 pub fn sanitize_sh(path: &Path) -> String {
 981     let path = path.to_str().unwrap().replace("\\", "/");
 982     return change_drive(&path).unwrap_or(path);
 983
 984     fn change_drive(s: &str) -> Option<String> {
 985         let mut ch = s.chars();
 986         let drive = ch.next().unwrap_or('C');
 987         if ch.next() != Some(':') {
 988             return None
 989         }
 990         if ch.next() != Some('/') {
 991             return None
 992         }
 993         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 994     }
 995 }
 996
 997 fn write_file(path: &Path, data: &[u8]) {
 998     let mut vf = t!(fs::File::create(path));
 999     t!(vf.write_all(data));
1000 }
1001
1002 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1003 pub struct Cargo {
1004     pub stage: u32,
1005     pub target: Interned<String>,
1006 }
1007
1008 impl Step for Cargo {
1009     type Output = PathBuf;
1010     const ONLY_BUILD_TARGETS: bool = true;
1011     const ONLY_HOSTS: bool = true;
1012
1013     fn should_run(run: ShouldRun) -> ShouldRun {
1014         run.path("cargo")
1015     }
1016
1017     fn make_run(run: RunConfig) {
1018         run.builder.ensure(Cargo {
1019             stage: run.builder.top_stage,
1020             target: run.target,
1021         });
1022     }
1023
1024     fn run(self, builder: &Builder) -> PathBuf {
1025         let build = builder.build;
1026         let stage = self.stage;
1027         let target = self.target;
1028
1029         println!("Dist cargo stage{} ({})", stage, target);
1030         let src = build.src.join("src/tools/cargo");
1031         let etc = src.join("src/etc");
1032         let release_num = build.release_num("cargo");
1033         let name = pkgname(build, "cargo");
1034         let version = builder.cargo_info.version(build, &release_num);
1035
1036         let tmp = tmpdir(build);
1037         let image = tmp.join("cargo-image");
1038         drop(fs::remove_dir_all(&image));
1039         t!(fs::create_dir_all(&image));
1040
1041         // Prepare the image directory
1042         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1043         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1044         let cargo = builder.ensure(tool::Cargo {
1045             compiler: builder.compiler(stage, build.build),
1046             target
1047         });
1048         install(&cargo, &image.join("bin"), 0o755);
1049         for man in t!(etc.join("man").read_dir()) {
1050             let man = t!(man);
1051             install(&man.path(), &image.join("share/man/man1"), 0o644);
1052         }
1053         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1054         copy(&etc.join("cargo.bashcomp.sh"),
1055              &image.join("etc/bash_completion.d/cargo"));
1056         let doc = image.join("share/doc/cargo");
1057         install(&src.join("README.md"), &doc, 0o644);
1058         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1059         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1060         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1061
1062         // Prepare the overlay
1063         let overlay = tmp.join("cargo-overlay");
1064         drop(fs::remove_dir_all(&overlay));
1065         t!(fs::create_dir_all(&overlay));
1066         install(&src.join("README.md"), &overlay, 0o644);
1067         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1068         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1069         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1070         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1071
1072         // Generate the installer tarball
1073         let mut cmd = rust_installer(builder);
1074         cmd.arg("generate")
1075            .arg("--product-name=Rust")
1076            .arg("--rel-manifest-dir=rustlib")
1077            .arg("--success-message=Rust-is-ready-to-roll.")
1078            .arg("--image-dir").arg(&image)
1079            .arg("--work-dir").arg(&tmpdir(build))
1080            .arg("--output-dir").arg(&distdir(build))
1081            .arg("--non-installed-overlay").arg(&overlay)
1082            .arg(format!("--package-name={}-{}", name, target))
1083            .arg("--component-name=cargo")
1084            .arg("--legacy-manifest-dirs=rustlib,cargo");
1085         build.run(&mut cmd);
1086         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1087     }
1088 }
1089
1090 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1091 pub struct Rls {
1092     pub stage: u32,
1093     pub target: Interned<String>,
1094 }
1095
1096 impl Step for Rls {
1097     type Output = Option<PathBuf>;
1098     const ONLY_BUILD_TARGETS: bool = true;
1099     const ONLY_HOSTS: bool = true;
1100
1101     fn should_run(run: ShouldRun) -> ShouldRun {
1102         run.path("rls")
1103     }
1104
1105     fn make_run(run: RunConfig) {
1106         run.builder.ensure(Rls {
1107             stage: run.builder.top_stage,
1108             target: run.target,
1109         });
1110     }
1111
1112     fn run(self, builder: &Builder) -> Option<PathBuf> {
1113         let build = builder.build;
1114         let stage = self.stage;
1115         let target = self.target;
1116         assert!(build.config.extended);
1117
1118         println!("Dist RLS stage{} ({})", stage, target);
1119         let src = build.src.join("src/tools/rls");
1120         let release_num = build.release_num("rls");
1121         let name = pkgname(build, "rls");
1122         let version = build.rls_info.version(build, &release_num);
1123
1124         let tmp = tmpdir(build);
1125         let image = tmp.join("rls-image");
1126         drop(fs::remove_dir_all(&image));
1127         t!(fs::create_dir_all(&image));
1128
1129         // Prepare the image directory
1130         // We expect RLS to build, because we've exited this step above if tool
1131         // state for RLS isn't testing.
1132         let rls = builder.ensure(tool::Rls {
1133             compiler: builder.compiler(stage, build.build),
1134             target
1135         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1136
1137         install(&rls, &image.join("bin"), 0o755);
1138         let doc = image.join("share/doc/rls");
1139         install(&src.join("README.md"), &doc, 0o644);
1140         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1141         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1142
1143         // Prepare the overlay
1144         let overlay = tmp.join("rls-overlay");
1145         drop(fs::remove_dir_all(&overlay));
1146         t!(fs::create_dir_all(&overlay));
1147         install(&src.join("README.md"), &overlay, 0o644);
1148         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1149         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1150         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1151
1152         // Generate the installer tarball
1153         let mut cmd = rust_installer(builder);
1154         cmd.arg("generate")
1155            .arg("--product-name=Rust")
1156            .arg("--rel-manifest-dir=rustlib")
1157            .arg("--success-message=RLS-ready-to-serve.")
1158            .arg("--image-dir").arg(&image)
1159            .arg("--work-dir").arg(&tmpdir(build))
1160            .arg("--output-dir").arg(&distdir(build))
1161            .arg("--non-installed-overlay").arg(&overlay)
1162            .arg(format!("--package-name={}-{}", name, target))
1163            .arg("--legacy-manifest-dirs=rustlib,cargo")
1164            .arg("--component-name=rls-preview");
1165
1166         build.run(&mut cmd);
1167         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1168     }
1169 }
1170
1171
1172 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1173 pub struct Rustfmt {
1174     pub stage: u32,
1175     pub target: Interned<String>,
1176 }
1177
1178 impl Step for Rustfmt {
1179     type Output = Option<PathBuf>;
1180     const ONLY_BUILD_TARGETS: bool = true;
1181     const ONLY_HOSTS: bool = true;
1182
1183     fn should_run(run: ShouldRun) -> ShouldRun {
1184         run.path("rustfmt")
1185     }
1186
1187     fn make_run(run: RunConfig) {
1188         run.builder.ensure(Rustfmt {
1189             stage: run.builder.top_stage,
1190             target: run.target,
1191         });
1192     }
1193
1194     fn run(self, builder: &Builder) -> Option<PathBuf> {
1195         let build = builder.build;
1196         let stage = self.stage;
1197         let target = self.target;
1198         assert!(build.config.extended);
1199
1200         println!("Dist Rustfmt stage{} ({})", stage, target);
1201         let src = build.src.join("src/tools/rustfmt");
1202         let release_num = build.release_num("rustfmt");
1203         let name = pkgname(build, "rustfmt");
1204         let version = build.rustfmt_info.version(build, &release_num);
1205
1206         let tmp = tmpdir(build);
1207         let image = tmp.join("rustfmt-image");
1208         drop(fs::remove_dir_all(&image));
1209         t!(fs::create_dir_all(&image));
1210
1211         // Prepare the image directory
1212         let rustfmt = builder.ensure(tool::Rustfmt {
1213             compiler: builder.compiler(stage, build.build),
1214             target
1215         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1216         let cargofmt = builder.ensure(tool::Cargofmt {
1217             compiler: builder.compiler(stage, build.build),
1218             target
1219         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1220
1221         install(&rustfmt, &image.join("bin"), 0o755);
1222         install(&cargofmt, &image.join("bin"), 0o755);
1223         let doc = image.join("share/doc/rustfmt");
1224         install(&src.join("README.md"), &doc, 0o644);
1225         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1226         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1227
1228         // Prepare the overlay
1229         let overlay = tmp.join("rustfmt-overlay");
1230         drop(fs::remove_dir_all(&overlay));
1231         t!(fs::create_dir_all(&overlay));
1232         install(&src.join("README.md"), &overlay, 0o644);
1233         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1234         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1235         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1236
1237         // Generate the installer tarball
1238         let mut cmd = rust_installer(builder);
1239         cmd.arg("generate")
1240            .arg("--product-name=Rust")
1241            .arg("--rel-manifest-dir=rustlib")
1242            .arg("--success-message=rustfmt-ready-to-fmt.")
1243            .arg("--image-dir").arg(&image)
1244            .arg("--work-dir").arg(&tmpdir(build))
1245            .arg("--output-dir").arg(&distdir(build))
1246            .arg("--non-installed-overlay").arg(&overlay)
1247            .arg(format!("--package-name={}-{}", name, target))
1248            .arg("--legacy-manifest-dirs=rustlib,cargo")
1249            .arg("--component-name=rustfmt-preview");
1250
1251         build.run(&mut cmd);
1252         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1253     }
1254 }
1255
1256 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1257 pub struct Extended {
1258     stage: u32,
1259     host: Interned<String>,
1260     target: Interned<String>,
1261 }
1262
1263 impl Step for Extended {
1264     type Output = ();
1265     const DEFAULT: bool = true;
1266     const ONLY_BUILD_TARGETS: bool = true;
1267     const ONLY_HOSTS: bool = true;
1268
1269     fn should_run(run: ShouldRun) -> ShouldRun {
1270         let builder = run.builder;
1271         run.path("extended").default_condition(builder.config.extended)
1272     }
1273
1274     fn make_run(run: RunConfig) {
1275         run.builder.ensure(Extended {
1276             stage: run.builder.top_stage,
1277             host: run.host,
1278             target: run.target,
1279         });
1280     }
1281
1282     /// Creates a combined installer for the specified target in the provided stage.
1283     fn run(self, builder: &Builder) {
1284         let build = builder.build;
1285         let stage = self.stage;
1286         let target = self.target;
1287
1288         println!("Dist extended stage{} ({})", stage, target);
1289
1290         let rustc_installer = builder.ensure(Rustc {
1291             compiler: builder.compiler(stage, target),
1292         });
1293         let cargo_installer = builder.ensure(Cargo { stage, target });
1294         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1295         let rls_installer = builder.ensure(Rls { stage, target });
1296         let mingw_installer = builder.ensure(Mingw { host: target });
1297         let analysis_installer = builder.ensure(Analysis {
1298             compiler: builder.compiler(stage, self.host),
1299             target
1300         });
1301
1302         let docs_installer = builder.ensure(Docs { stage, host: target, });
1303         let std_installer = builder.ensure(Std {
1304             compiler: builder.compiler(stage, self.host),
1305             target,
1306         });
1307
1308         let tmp = tmpdir(build);
1309         let overlay = tmp.join("extended-overlay");
1310         let etc = build.src.join("src/etc/installer");
1311         let work = tmp.join("work");
1312
1313         let _ = fs::remove_dir_all(&overlay);
1314         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1315         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1316         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1317         let version = build.rust_version();
1318         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1319         if let Some(sha) = build.rust_sha() {
1320             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1321         }
1322         install(&etc.join("README.md"), &overlay, 0o644);
1323
1324         // When rust-std package split from rustc, we needed to ensure that during
1325         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1326         // the std files during uninstall. To do this ensure that rustc comes
1327         // before rust-std in the list below.
1328         let mut tarballs = Vec::new();
1329         tarballs.push(rustc_installer);
1330         tarballs.push(cargo_installer);
1331         tarballs.extend(rls_installer.clone());
1332         tarballs.extend(rustfmt_installer.clone());
1333         tarballs.push(analysis_installer);
1334         tarballs.push(std_installer);
1335         if build.config.docs {
1336             tarballs.push(docs_installer);
1337         }
1338         if target.contains("pc-windows-gnu") {
1339             tarballs.push(mingw_installer.unwrap());
1340         }
1341         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1342         for tarball in &tarballs[1..] {
1343             input_tarballs.push(",");
1344             input_tarballs.push(tarball);
1345         }
1346
1347         let mut cmd = rust_installer(builder);
1348         cmd.arg("combine")
1349             .arg("--product-name=Rust")
1350             .arg("--rel-manifest-dir=rustlib")
1351             .arg("--success-message=Rust-is-ready-to-roll.")
1352             .arg("--work-dir").arg(&work)
1353             .arg("--output-dir").arg(&distdir(build))
1354             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1355             .arg("--legacy-manifest-dirs=rustlib,cargo")
1356             .arg("--input-tarballs").arg(input_tarballs)
1357             .arg("--non-installed-overlay").arg(&overlay);
1358         build.run(&mut cmd);
1359
1360         let mut license = String::new();
1361         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1362         license.push_str("\n");
1363         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1364         license.push_str("\n");
1365         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1366
1367         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1368         let mut rtf = rtf.to_string();
1369         rtf.push_str("\n");
1370         for line in license.lines() {
1371             rtf.push_str(line);
1372             rtf.push_str("\\line ");
1373         }
1374         rtf.push_str("}");
1375
1376         fn filter(contents: &str, marker: &str) -> String {
1377             let start = format!("tool-{}-start", marker);
1378             let end = format!("tool-{}-end", marker);
1379             let mut lines = Vec::new();
1380             let mut omitted = false;
1381             for line in contents.lines() {
1382                 if line.contains(&start) {
1383                     omitted = true;
1384                 } else if line.contains(&end) {
1385                     omitted = false;
1386                 } else if !omitted {
1387                     lines.push(line);
1388                 }
1389             }
1390
1391             lines.join("\n")
1392         }
1393
1394         let xform = |p: &Path| {
1395             let mut contents = String::new();
1396             t!(t!(File::open(p)).read_to_string(&mut contents));
1397             if rls_installer.is_none() {
1398                 contents = filter(&contents, "rls");
1399             }
1400             if rustfmt_installer.is_none() {
1401                 contents = filter(&contents, "rustfmt");
1402             }
1403             let ret = tmp.join(p.file_name().unwrap());
1404             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1405             return ret
1406         };
1407
1408         if target.contains("apple-darwin") {
1409             let pkg = tmp.join("pkg");
1410             let _ = fs::remove_dir_all(&pkg);
1411
1412             let pkgbuild = |component: &str| {
1413                 let mut cmd = Command::new("pkgbuild");
1414                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1415                     .arg("--scripts").arg(pkg.join(component))
1416                     .arg("--nopayload")
1417                     .arg(pkg.join(component).with_extension("pkg"));
1418                 build.run(&mut cmd);
1419             };
1420
1421             let prepare = |name: &str| {
1422                 t!(fs::create_dir_all(pkg.join(name)));
1423                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1424                         &pkg.join(name));
1425                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1426                 pkgbuild(name);
1427             };
1428             prepare("rustc");
1429             prepare("cargo");
1430             prepare("rust-docs");
1431             prepare("rust-std");
1432             prepare("rust-analysis");
1433
1434             if rls_installer.is_some() {
1435                 prepare("rls");
1436             }
1437
1438             // create an 'uninstall' package
1439             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1440             pkgbuild("uninstall");
1441
1442             t!(fs::create_dir_all(pkg.join("res")));
1443             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1444             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1445             let mut cmd = Command::new("productbuild");
1446             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1447                 .arg("--resources").arg(pkg.join("res"))
1448                 .arg(distdir(build).join(format!("{}-{}.pkg",
1449                                                     pkgname(build, "rust"),
1450                                                     target)))
1451                 .arg("--package-path").arg(&pkg);
1452             build.run(&mut cmd);
1453         }
1454
1455         if target.contains("windows") {
1456             let exe = tmp.join("exe");
1457             let _ = fs::remove_dir_all(&exe);
1458
1459             let prepare = |name: &str| {
1460                 t!(fs::create_dir_all(exe.join(name)));
1461                 let dir = if name == "rust-std" || name == "rust-analysis" {
1462                     format!("{}-{}", name, target)
1463                 } else if name == "rls" {
1464                     "rls-preview".to_string()
1465                 } else {
1466                     name.to_string()
1467                 };
1468                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1469                             .join(dir),
1470                         &exe.join(name));
1471                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1472             };
1473             prepare("rustc");
1474             prepare("cargo");
1475             prepare("rust-analysis");
1476             prepare("rust-docs");
1477             prepare("rust-std");
1478             if rls_installer.is_some() {
1479                 prepare("rls");
1480             }
1481             if target.contains("windows-gnu") {
1482                 prepare("rust-mingw");
1483             }
1484
1485             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1486             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1487             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1488             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1489             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1490
1491             // Generate exe installer
1492             let mut cmd = Command::new("iscc");
1493             cmd.arg("rust.iss")
1494                 .current_dir(&exe);
1495             if target.contains("windows-gnu") {
1496                 cmd.arg("/dMINGW");
1497             }
1498             add_env(build, &mut cmd, target);
1499             build.run(&mut cmd);
1500             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1501                     &distdir(build),
1502                     0o755);
1503
1504             // Generate msi installer
1505             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1506             let heat = wix.join("bin/heat.exe");
1507             let candle = wix.join("bin/candle.exe");
1508             let light = wix.join("bin/light.exe");
1509
1510             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1511             build.run(Command::new(&heat)
1512                             .current_dir(&exe)
1513                             .arg("dir")
1514                             .arg("rustc")
1515                             .args(&heat_flags)
1516                             .arg("-cg").arg("RustcGroup")
1517                             .arg("-dr").arg("Rustc")
1518                             .arg("-var").arg("var.RustcDir")
1519                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1520             build.run(Command::new(&heat)
1521                             .current_dir(&exe)
1522                             .arg("dir")
1523                             .arg("rust-docs")
1524                             .args(&heat_flags)
1525                             .arg("-cg").arg("DocsGroup")
1526                             .arg("-dr").arg("Docs")
1527                             .arg("-var").arg("var.DocsDir")
1528                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1529                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1530             build.run(Command::new(&heat)
1531                             .current_dir(&exe)
1532                             .arg("dir")
1533                             .arg("cargo")
1534                             .args(&heat_flags)
1535                             .arg("-cg").arg("CargoGroup")
1536                             .arg("-dr").arg("Cargo")
1537                             .arg("-var").arg("var.CargoDir")
1538                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1539                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1540             build.run(Command::new(&heat)
1541                             .current_dir(&exe)
1542                             .arg("dir")
1543                             .arg("rust-std")
1544                             .args(&heat_flags)
1545                             .arg("-cg").arg("StdGroup")
1546                             .arg("-dr").arg("Std")
1547                             .arg("-var").arg("var.StdDir")
1548                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1549             if rls_installer.is_some() {
1550                 build.run(Command::new(&heat)
1551                                 .current_dir(&exe)
1552                                 .arg("dir")
1553                                 .arg("rls")
1554                                 .args(&heat_flags)
1555                                 .arg("-cg").arg("RlsGroup")
1556                                 .arg("-dr").arg("Rls")
1557                                 .arg("-var").arg("var.RlsDir")
1558                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1559                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1560             }
1561             build.run(Command::new(&heat)
1562                             .current_dir(&exe)
1563                             .arg("dir")
1564                             .arg("rust-analysis")
1565                             .args(&heat_flags)
1566                             .arg("-cg").arg("AnalysisGroup")
1567                             .arg("-dr").arg("Analysis")
1568                             .arg("-var").arg("var.AnalysisDir")
1569                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1570                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1571             if target.contains("windows-gnu") {
1572                 build.run(Command::new(&heat)
1573                                 .current_dir(&exe)
1574                                 .arg("dir")
1575                                 .arg("rust-mingw")
1576                                 .args(&heat_flags)
1577                                 .arg("-cg").arg("GccGroup")
1578                                 .arg("-dr").arg("Gcc")
1579                                 .arg("-var").arg("var.GccDir")
1580                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1581             }
1582
1583             let candle = |input: &Path| {
1584                 let output = exe.join(input.file_stem().unwrap())
1585                                 .with_extension("wixobj");
1586                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1587                 let mut cmd = Command::new(&candle);
1588                 cmd.current_dir(&exe)
1589                     .arg("-nologo")
1590                     .arg("-dRustcDir=rustc")
1591                     .arg("-dDocsDir=rust-docs")
1592                     .arg("-dCargoDir=cargo")
1593                     .arg("-dStdDir=rust-std")
1594                     .arg("-dAnalysisDir=rust-analysis")
1595                     .arg("-arch").arg(&arch)
1596                     .arg("-out").arg(&output)
1597                     .arg(&input);
1598                 add_env(build, &mut cmd, target);
1599
1600                 if rls_installer.is_some() {
1601                     cmd.arg("-dRlsDir=rls");
1602                 }
1603                 if target.contains("windows-gnu") {
1604                     cmd.arg("-dGccDir=rust-mingw");
1605                 }
1606                 build.run(&mut cmd);
1607             };
1608             candle(&xform(&etc.join("msi/rust.wxs")));
1609             candle(&etc.join("msi/ui.wxs"));
1610             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1611             candle("RustcGroup.wxs".as_ref());
1612             candle("DocsGroup.wxs".as_ref());
1613             candle("CargoGroup.wxs".as_ref());
1614             candle("StdGroup.wxs".as_ref());
1615             if rls_installer.is_some() {
1616                 candle("RlsGroup.wxs".as_ref());
1617             }
1618             candle("AnalysisGroup.wxs".as_ref());
1619
1620             if target.contains("windows-gnu") {
1621                 candle("GccGroup.wxs".as_ref());
1622             }
1623
1624             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1625             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1626             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1627
1628             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1629             let mut cmd = Command::new(&light);
1630             cmd.arg("-nologo")
1631                 .arg("-ext").arg("WixUIExtension")
1632                 .arg("-ext").arg("WixUtilExtension")
1633                 .arg("-out").arg(exe.join(&filename))
1634                 .arg("rust.wixobj")
1635                 .arg("ui.wixobj")
1636                 .arg("rustwelcomedlg.wixobj")
1637                 .arg("RustcGroup.wixobj")
1638                 .arg("DocsGroup.wixobj")
1639                 .arg("CargoGroup.wixobj")
1640                 .arg("StdGroup.wixobj")
1641                 .arg("AnalysisGroup.wixobj")
1642                 .current_dir(&exe);
1643
1644             if rls_installer.is_some() {
1645                 cmd.arg("RlsGroup.wixobj");
1646             }
1647
1648             if target.contains("windows-gnu") {
1649                 cmd.arg("GccGroup.wixobj");
1650             }
1651             // ICE57 wrongly complains about the shortcuts
1652             cmd.arg("-sice:ICE57");
1653
1654             build.run(&mut cmd);
1655
1656             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1657         }
1658     }
1659 }
1660
1661 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1662     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1663     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1664        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1665        .env("CFG_RELEASE", build.rust_release())
1666        .env("CFG_VER_MAJOR", parts.next().unwrap())
1667        .env("CFG_VER_MINOR", parts.next().unwrap())
1668        .env("CFG_VER_PATCH", parts.next().unwrap())
1669        .env("CFG_VER_BUILD", "0") // just needed to build
1670        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1671        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1672        .env("CFG_BUILD", target)
1673        .env("CFG_CHANNEL", &build.config.channel);
1674
1675     if target.contains("windows-gnu") {
1676        cmd.env("CFG_MINGW", "1")
1677           .env("CFG_ABI", "GNU");
1678     } else {
1679        cmd.env("CFG_MINGW", "0")
1680           .env("CFG_ABI", "MSVC");
1681     }
1682
1683     if target.contains("x86_64") {
1684        cmd.env("CFG_PLATFORM", "x64");
1685     } else {
1686        cmd.env("CFG_PLATFORM", "x86");
1687     }
1688 }
1689
1690 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1691 pub struct HashSign;
1692
1693 impl Step for HashSign {
1694     type Output = ();
1695     const ONLY_BUILD_TARGETS: bool = true;
1696     const ONLY_HOSTS: bool = true;
1697     const ONLY_BUILD: bool = true;
1698
1699     fn should_run(run: ShouldRun) -> ShouldRun {
1700         run.path("hash-and-sign")
1701     }
1702
1703     fn make_run(run: RunConfig) {
1704         run.builder.ensure(HashSign);
1705     }
1706
1707     fn run(self, builder: &Builder) {
1708         let build = builder.build;
1709         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1710         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1711             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1712         });
1713         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1714             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1715         });
1716         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1717             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1718         });
1719         let mut pass = String::new();
1720         t!(t!(File::open(&file)).read_to_string(&mut pass));
1721
1722         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1723
1724         cmd.arg(sign);
1725         cmd.arg(distdir(build));
1726         cmd.arg(today.trim());
1727         cmd.arg(build.rust_package_vers());
1728         cmd.arg(build.package_vers(&build.release_num("cargo")));
1729         cmd.arg(build.package_vers(&build.release_num("rls")));
1730         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1731         cmd.arg(addr);
1732
1733         t!(fs::create_dir_all(distdir(build)));
1734
1735         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1736         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1737         let status = t!(child.wait());
1738         assert!(status.success());
1739     }
1740 }