src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file, exe};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use native;
  35 use tool::{self, Tool};
  36 use cache::{INTERNER, Interned};
  37 use time;
  38
  39 pub fn pkgname(build: &Build, component: &str) -> String {
  40     if component == "cargo" {
  41         format!("{}-{}", component, build.cargo_package_vers())
  42     } else if component == "rls" {
  43         format!("{}-{}", component, build.rls_package_vers())
  44     } else if component == "rustfmt" {
  45         format!("{}-{}", component, build.rustfmt_package_vers())
  46     } else {
  47         assert!(component.starts_with("rust"));
  48         format!("{}-{}", component, build.rust_package_vers())
  49     }
  50 }
  51
  52 fn distdir(build: &Build) -> PathBuf {
  53     build.out.join("dist")
  54 }
  55
  56 pub fn tmpdir(build: &Build) -> PathBuf {
  57     build.out.join("tmp/dist")
  58 }
  59
  60 fn rust_installer(builder: &Builder) -> Command {
  61     builder.tool_cmd(Tool::RustInstaller)
  62 }
  63
  64 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  65 pub struct Docs {
  66     pub stage: u32,
  67     pub host: Interned<String>,
  68 }
  69
  70 impl Step for Docs {
  71     type Output = PathBuf;
  72     const DEFAULT: bool = true;
  73     const ONLY_BUILD_TARGETS: bool = true;
  74
  75     fn should_run(run: ShouldRun) -> ShouldRun {
  76         run.path("src/doc")
  77     }
  78
  79     fn make_run(run: RunConfig) {
  80         run.builder.ensure(Docs {
  81             stage: run.builder.top_stage,
  82             host: run.target,
  83         });
  84     }
  85
  86     /// Builds the `rust-docs` installer component.
  87     fn run(self, builder: &Builder) -> PathBuf {
  88         let build = builder.build;
  89         let host = self.host;
  90
  91         let name = pkgname(build, "rust-docs");
  92
  93         println!("Dist docs ({})", host);
  94         if !build.config.docs {
  95             println!("\tskipping - docs disabled");
  96             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  97         }
  98
  99         builder.default_doc(None);
 100
 101         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 102         let _ = fs::remove_dir_all(&image);
 103
 104         let dst = image.join("share/doc/rust/html");
 105         t!(fs::create_dir_all(&dst));
 106         let src = build.out.join(host).join("doc");
 107         cp_r(&src, &dst);
 108
 109         let mut cmd = rust_installer(builder);
 110         cmd.arg("generate")
 111            .arg("--product-name=Rust-Documentation")
 112            .arg("--rel-manifest-dir=rustlib")
 113            .arg("--success-message=Rust-documentation-is-installed.")
 114            .arg("--image-dir").arg(&image)
 115            .arg("--work-dir").arg(&tmpdir(build))
 116            .arg("--output-dir").arg(&distdir(build))
 117            .arg(format!("--package-name={}-{}", name, host))
 118            .arg("--component-name=rust-docs")
 119            .arg("--legacy-manifest-dirs=rustlib,cargo")
 120            .arg("--bulk-dirs=share/doc/rust/html");
 121         build.run(&mut cmd);
 122         t!(fs::remove_dir_all(&image));
 123
 124         // As part of this step, *also* copy the docs directory to a directory which
 125         // buildbot typically uploads.
 126         if host == build.build {
 127             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 128             t!(fs::create_dir_all(&dst));
 129             cp_r(&src, &dst);
 130         }
 131
 132         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 133     }
 134 }
 135
 136 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 137     let mut found = Vec::with_capacity(files.len());
 138
 139     for file in files {
 140         let file_path =
 141             path.iter()
 142                 .map(|dir| dir.join(file))
 143                 .find(|p| p.exists());
 144
 145         if let Some(file_path) = file_path {
 146             found.push(file_path);
 147         } else {
 148             panic!("Could not find '{}' in {:?}", file, path);
 149         }
 150     }
 151
 152     found
 153 }
 154
 155 fn make_win_dist(
 156     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 157 ) {
 158     //Ask gcc where it keeps its stuff
 159     let mut cmd = Command::new(build.cc(target_triple));
 160     cmd.arg("-print-search-dirs");
 161     let gcc_out = output(&mut cmd);
 162
 163     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 164     let mut lib_path = Vec::new();
 165
 166     for line in gcc_out.lines() {
 167         let idx = line.find(':').unwrap();
 168         let key = &line[..idx];
 169         let trim_chars: &[_] = &[' ', '='];
 170         let value =
 171             line[(idx + 1)..]
 172                 .trim_left_matches(trim_chars)
 173                 .split(';')
 174                 .map(PathBuf::from);
 175
 176         if key == "programs" {
 177             bin_path.extend(value);
 178         } else if key == "libraries" {
 179             lib_path.extend(value);
 180         }
 181     }
 182
 183     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 184     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 185     if target_triple.starts_with("i686-") {
 186         rustc_dlls.push("libgcc_s_dw2-1.dll");
 187     } else {
 188         rustc_dlls.push("libgcc_s_seh-1.dll");
 189     }
 190
 191     let target_libs = [ //MinGW libs
 192         "libgcc.a",
 193         "libgcc_eh.a",
 194         "libgcc_s.a",
 195         "libm.a",
 196         "libmingw32.a",
 197         "libmingwex.a",
 198         "libstdc++.a",
 199         "libiconv.a",
 200         "libmoldname.a",
 201         "libpthread.a",
 202         //Windows import libs
 203         "libadvapi32.a",
 204         "libbcrypt.a",
 205         "libcomctl32.a",
 206         "libcomdlg32.a",
 207         "libcrypt32.a",
 208         "libgdi32.a",
 209         "libimagehlp.a",
 210         "libiphlpapi.a",
 211         "libkernel32.a",
 212         "libmsvcrt.a",
 213         "libodbc32.a",
 214         "libole32.a",
 215         "liboleaut32.a",
 216         "libopengl32.a",
 217         "libpsapi.a",
 218         "librpcrt4.a",
 219         "libsetupapi.a",
 220         "libshell32.a",
 221         "libuser32.a",
 222         "libuserenv.a",
 223         "libuuid.a",
 224         "libwinhttp.a",
 225         "libwinmm.a",
 226         "libwinspool.a",
 227         "libws2_32.a",
 228         "libwsock32.a",
 229         "libdbghelp.a",
 230         "libmsimg32.a",
 231     ];
 232
 233     //Find mingw artifacts we want to bundle
 234     let target_tools = find_files(&target_tools, &bin_path);
 235     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 236     let target_libs = find_files(&target_libs, &lib_path);
 237
 238     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 239         let file_name = src.file_name().unwrap();
 240         let dest = dest_folder.join(file_name);
 241         copy(src, &dest);
 242     }
 243
 244     //Copy runtime dlls next to rustc.exe
 245     let dist_bin_dir = rust_root.join("bin/");
 246     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 247     for src in rustc_dlls {
 248         copy_to_folder(&src, &dist_bin_dir);
 249     }
 250
 251     //Copy platform tools to platform-specific bin directory
 252     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 253     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 254     for src in target_tools {
 255         copy_to_folder(&src, &target_bin_dir);
 256     }
 257
 258     //Copy platform libs to platform-specific lib directory
 259     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 260     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 261     for src in target_libs {
 262         copy_to_folder(&src, &target_lib_dir);
 263     }
 264 }
 265
 266 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 267 pub struct Mingw {
 268     host: Interned<String>,
 269 }
 270
 271 impl Step for Mingw {
 272     type Output = Option<PathBuf>;
 273     const DEFAULT: bool = true;
 274     const ONLY_BUILD_TARGETS: bool = true;
 275
 276     fn should_run(run: ShouldRun) -> ShouldRun {
 277         run.never()
 278     }
 279
 280     fn make_run(run: RunConfig) {
 281         run.builder.ensure(Mingw { host: run.target });
 282     }
 283
 284     /// Build the `rust-mingw` installer component.
 285     ///
 286     /// This contains all the bits and pieces to run the MinGW Windows targets
 287     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 288     fn run(self, builder: &Builder) -> Option<PathBuf> {
 289         let build = builder.build;
 290         let host = self.host;
 291
 292         if !host.contains("pc-windows-gnu") {
 293             return None;
 294         }
 295
 296         println!("Dist mingw ({})", host);
 297         let name = pkgname(build, "rust-mingw");
 298         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 299         let _ = fs::remove_dir_all(&image);
 300         t!(fs::create_dir_all(&image));
 301
 302         // The first argument is a "temporary directory" which is just
 303         // thrown away (this contains the runtime DLLs included in the rustc package
 304         // above) and the second argument is where to place all the MinGW components
 305         // (which is what we want).
 306         make_win_dist(&tmpdir(build), &image, host, &build);
 307
 308         let mut cmd = rust_installer(builder);
 309         cmd.arg("generate")
 310            .arg("--product-name=Rust-MinGW")
 311            .arg("--rel-manifest-dir=rustlib")
 312            .arg("--success-message=Rust-MinGW-is-installed.")
 313            .arg("--image-dir").arg(&image)
 314            .arg("--work-dir").arg(&tmpdir(build))
 315            .arg("--output-dir").arg(&distdir(build))
 316            .arg(format!("--package-name={}-{}", name, host))
 317            .arg("--component-name=rust-mingw")
 318            .arg("--legacy-manifest-dirs=rustlib,cargo");
 319         build.run(&mut cmd);
 320         t!(fs::remove_dir_all(&image));
 321         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 322     }
 323 }
 324
 325 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 326 pub struct Rustc {
 327     pub compiler: Compiler,
 328 }
 329
 330 impl Step for Rustc {
 331     type Output = PathBuf;
 332     const DEFAULT: bool = true;
 333     const ONLY_HOSTS: bool = true;
 334     const ONLY_BUILD_TARGETS: bool = true;
 335
 336     fn should_run(run: ShouldRun) -> ShouldRun {
 337         run.path("src/librustc")
 338     }
 339
 340     fn make_run(run: RunConfig) {
 341         run.builder.ensure(Rustc {
 342             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 343         });
 344     }
 345
 346     /// Creates the `rustc` installer component.
 347     fn run(self, builder: &Builder) -> PathBuf {
 348         let build = builder.build;
 349         let compiler = self.compiler;
 350         let host = self.compiler.host;
 351
 352         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 353         let name = pkgname(build, "rustc");
 354         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 357         let _ = fs::remove_dir_all(&overlay);
 358
 359         // Prepare the rustc "image", what will actually end up getting installed
 360         prepare_image(builder, compiler, &image);
 361
 362         // Prepare the overlay which is part of the tarball but won't actually be
 363         // installed
 364         let cp = |file: &str| {
 365             install(&build.src.join(file), &overlay, 0o644);
 366         };
 367         cp("COPYRIGHT");
 368         cp("LICENSE-APACHE");
 369         cp("LICENSE-MIT");
 370         cp("README.md");
 371         // tiny morsel of metadata is used by rust-packaging
 372         let version = build.rust_version();
 373         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 374         if let Some(sha) = build.rust_sha() {
 375             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 376         }
 377
 378         // On MinGW we've got a few runtime DLL dependencies that we need to
 379         // include. The first argument to this script is where to put these DLLs
 380         // (the image we're creating), and the second argument is a junk directory
 381         // to ignore all other MinGW stuff the script creates.
 382         //
 383         // On 32-bit MinGW we're always including a DLL which needs some extra
 384         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 385         // anything requiring us to distribute a license, but it's likely the
 386         // install will *also* include the rust-mingw package, which also needs
 387         // licenses, so to be safe we just include it here in all MinGW packages.
 388         if host.contains("pc-windows-gnu") {
 389             make_win_dist(&image, &tmpdir(build), host, build);
 390
 391             let dst = image.join("share/doc");
 392             t!(fs::create_dir_all(&dst));
 393             cp_r(&build.src.join("src/etc/third-party"), &dst);
 394         }
 395
 396         // Finally, wrap everything up in a nice tarball!
 397         let mut cmd = rust_installer(builder);
 398         cmd.arg("generate")
 399            .arg("--product-name=Rust")
 400            .arg("--rel-manifest-dir=rustlib")
 401            .arg("--success-message=Rust-is-ready-to-roll.")
 402            .arg("--image-dir").arg(&image)
 403            .arg("--work-dir").arg(&tmpdir(build))
 404            .arg("--output-dir").arg(&distdir(build))
 405            .arg("--non-installed-overlay").arg(&overlay)
 406            .arg(format!("--package-name={}-{}", name, host))
 407            .arg("--component-name=rustc")
 408            .arg("--legacy-manifest-dirs=rustlib,cargo");
 409         build.run(&mut cmd);
 410         t!(fs::remove_dir_all(&image));
 411         t!(fs::remove_dir_all(&overlay));
 412
 413         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 414
 415         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 416             let host = compiler.host;
 417             let build = builder.build;
 418             let src = builder.sysroot(compiler);
 419             let libdir = libdir(&host);
 420
 421             // Copy rustc/rustdoc binaries
 422             t!(fs::create_dir_all(image.join("bin")));
 423             cp_r(&src.join("bin"), &image.join("bin"));
 424
 425             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 426
 427             // Copy runtime DLLs needed by the compiler
 428             if libdir != "bin" {
 429                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 430                     let name = entry.file_name();
 431                     if let Some(s) = name.to_str() {
 432                         if is_dylib(s) {
 433                             install(&entry.path(), &image.join(libdir), 0o644);
 434                         }
 435                     }
 436                 }
 437             }
 438
 439             // Copy over the codegen backends
 440             let backends_src = builder.sysroot_codegen_backends(compiler);
 441             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 442             let backends_dst = image.join(&backends_rel);
 443             t!(fs::create_dir_all(&backends_dst));
 444             cp_r(&backends_src, &backends_dst);
 445
 446             // Copy over lld if it's there
 447             if builder.config.lld_enabled {
 448                 let exe = exe("lld", &compiler.host);
 449                 let src = builder.sysroot_libdir(compiler, host)
 450                     .parent()
 451                     .unwrap()
 452                     .join("bin")
 453                     .join(&exe);
 454                 let dst = image.join("lib/rustlib")
 455                     .join(&*host)
 456                     .join("bin")
 457                     .join(&exe);
 458                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 459                 copy(&src, &dst);
 460             }
 461
 462             // Man pages
 463             t!(fs::create_dir_all(image.join("share/man/man1")));
 464             let man_src = build.src.join("src/doc/man");
 465             let man_dst = image.join("share/man/man1");
 466             let month_year = t!(time::strftime("%B %Y", &time::now()));
 467             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 468             // to hardlink, and we don't want to edit the source templates
 469             for entry_result in t!(fs::read_dir(man_src)) {
 470                 let file_entry = t!(entry_result);
 471                 let page_src = file_entry.path();
 472                 let page_dst = man_dst.join(file_entry.file_name());
 473                 t!(fs::copy(&page_src, &page_dst));
 474                 // template in month/year and version number
 475                 replace_in_file(&page_dst,
 476                                 &[("<INSERT DATE HERE>", &month_year),
 477                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 478             }
 479
 480             // Debugger scripts
 481             builder.ensure(DebuggerScripts {
 482                 sysroot: INTERNER.intern_path(image.to_owned()),
 483                 host,
 484             });
 485
 486             // Misc license info
 487             let cp = |file: &str| {
 488                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 489             };
 490             cp("COPYRIGHT");
 491             cp("LICENSE-APACHE");
 492             cp("LICENSE-MIT");
 493             cp("README.md");
 494         }
 495     }
 496 }
 497
 498 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 499 pub struct DebuggerScripts {
 500     pub sysroot: Interned<PathBuf>,
 501     pub host: Interned<String>,
 502 }
 503
 504 impl Step for DebuggerScripts {
 505     type Output = ();
 506
 507     fn should_run(run: ShouldRun) -> ShouldRun {
 508         run.path("src/lldb_batchmode.py")
 509     }
 510
 511     fn make_run(run: RunConfig) {
 512         run.builder.ensure(DebuggerScripts {
 513             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 514             host: run.target,
 515         });
 516     }
 517
 518     /// Copies debugger scripts for `target` into the `sysroot` specified.
 519     fn run(self, builder: &Builder) {
 520         let build = builder.build;
 521         let host = self.host;
 522         let sysroot = self.sysroot;
 523         let dst = sysroot.join("lib/rustlib/etc");
 524         t!(fs::create_dir_all(&dst));
 525         let cp_debugger_script = |file: &str| {
 526             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 527         };
 528         if host.contains("windows-msvc") {
 529             // windbg debugger scripts
 530             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 531                 0o755);
 532
 533             cp_debugger_script("natvis/intrinsic.natvis");
 534             cp_debugger_script("natvis/liballoc.natvis");
 535             cp_debugger_script("natvis/libcore.natvis");
 536         } else {
 537             cp_debugger_script("debugger_pretty_printers_common.py");
 538
 539             // gdb debugger scripts
 540             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 541                     0o755);
 542
 543             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 544             cp_debugger_script("gdb_rust_pretty_printing.py");
 545
 546             // lldb debugger scripts
 547             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 548                     0o755);
 549
 550             cp_debugger_script("lldb_rust_formatters.py");
 551         }
 552     }
 553 }
 554
 555 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 556 pub struct Std {
 557     pub compiler: Compiler,
 558     pub target: Interned<String>,
 559 }
 560
 561 impl Step for Std {
 562     type Output = PathBuf;
 563     const DEFAULT: bool = true;
 564     const ONLY_BUILD_TARGETS: bool = true;
 565
 566     fn should_run(run: ShouldRun) -> ShouldRun {
 567         run.path("src/libstd")
 568     }
 569
 570     fn make_run(run: RunConfig) {
 571         run.builder.ensure(Std {
 572             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 573             target: run.target,
 574         });
 575     }
 576
 577     fn run(self, builder: &Builder) -> PathBuf {
 578         let build = builder.build;
 579         let compiler = self.compiler;
 580         let target = self.target;
 581
 582         let name = pkgname(build, "rust-std");
 583         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 584
 585         // The only true set of target libraries came from the build triple, so
 586         // let's reduce redundant work by only producing archives from that host.
 587         if compiler.host != build.build {
 588             println!("\tskipping, not a build host");
 589             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 590         }
 591
 592         // We want to package up as many target libraries as possible
 593         // for the `rust-std` package, so if this is a host target we
 594         // depend on librustc and otherwise we just depend on libtest.
 595         if build.hosts.iter().any(|t| t == target) {
 596             builder.ensure(compile::Rustc { compiler, target });
 597         } else {
 598             builder.ensure(compile::Test { compiler, target });
 599         }
 600
 601         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 602         let _ = fs::remove_dir_all(&image);
 603
 604         let dst = image.join("lib/rustlib").join(target);
 605         t!(fs::create_dir_all(&dst));
 606         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 607         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 608         cp_filtered(&src, &dst, &|path| {
 609             let name = path.file_name().and_then(|s| s.to_str());
 610             name != Some(build.config.rust_codegen_backends_dir.as_str()) &&
 611                 name != Some("bin")
 612
 613         });
 614
 615         let mut cmd = rust_installer(builder);
 616         cmd.arg("generate")
 617            .arg("--product-name=Rust")
 618            .arg("--rel-manifest-dir=rustlib")
 619            .arg("--success-message=std-is-standing-at-the-ready.")
 620            .arg("--image-dir").arg(&image)
 621            .arg("--work-dir").arg(&tmpdir(build))
 622            .arg("--output-dir").arg(&distdir(build))
 623            .arg(format!("--package-name={}-{}", name, target))
 624            .arg(format!("--component-name=rust-std-{}", target))
 625            .arg("--legacy-manifest-dirs=rustlib,cargo");
 626         build.run(&mut cmd);
 627         t!(fs::remove_dir_all(&image));
 628         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 629     }
 630 }
 631
 632 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Analysis {
 634     pub compiler: Compiler,
 635     pub target: Interned<String>,
 636 }
 637
 638 impl Step for Analysis {
 639     type Output = PathBuf;
 640     const DEFAULT: bool = true;
 641     const ONLY_BUILD_TARGETS: bool = true;
 642
 643     fn should_run(run: ShouldRun) -> ShouldRun {
 644         let builder = run.builder;
 645         run.path("analysis").default_condition(builder.build.config.extended)
 646     }
 647
 648     fn make_run(run: RunConfig) {
 649         run.builder.ensure(Analysis {
 650             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 651             target: run.target,
 652         });
 653     }
 654
 655     /// Creates a tarball of save-analysis metadata, if available.
 656     fn run(self, builder: &Builder) -> PathBuf {
 657         let build = builder.build;
 658         let compiler = self.compiler;
 659         let target = self.target;
 660         assert!(build.config.extended);
 661         println!("Dist analysis");
 662         let name = pkgname(build, "rust-analysis");
 663
 664         if &compiler.host != build.build {
 665             println!("\tskipping, not a build host");
 666             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 667         }
 668
 669         builder.ensure(Std { compiler, target });
 670
 671         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 672         // stage2 artifacts is simply copied from stage1 in that case.
 673         let compiler = if build.force_use_stage1(compiler, target) {
 674             builder.compiler(1, compiler.host)
 675         } else {
 676             compiler.clone()
 677         };
 678
 679         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 680
 681         let src = build.stage_out(compiler, Mode::Libstd)
 682             .join(target).join(build.cargo_dir()).join("deps");
 683
 684         let image_src = src.join("save-analysis");
 685         let dst = image.join("lib/rustlib").join(target).join("analysis");
 686         t!(fs::create_dir_all(&dst));
 687         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 688         cp_r(&image_src, &dst);
 689
 690         let mut cmd = rust_installer(builder);
 691         cmd.arg("generate")
 692            .arg("--product-name=Rust")
 693            .arg("--rel-manifest-dir=rustlib")
 694            .arg("--success-message=save-analysis-saved.")
 695            .arg("--image-dir").arg(&image)
 696            .arg("--work-dir").arg(&tmpdir(build))
 697            .arg("--output-dir").arg(&distdir(build))
 698            .arg(format!("--package-name={}-{}", name, target))
 699            .arg(format!("--component-name=rust-analysis-{}", target))
 700            .arg("--legacy-manifest-dirs=rustlib,cargo");
 701         build.run(&mut cmd);
 702         t!(fs::remove_dir_all(&image));
 703         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 704     }
 705 }
 706
 707 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 708     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 709         let spath = match path.to_str() {
 710             Some(path) => path,
 711             None => return false,
 712         };
 713         if spath.ends_with("~") || spath.ends_with(".pyc") {
 714             return false
 715         }
 716         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 717             (spath.ends_with(".ll") ||
 718              spath.ends_with(".td") ||
 719              spath.ends_with(".s")) {
 720             return false
 721         }
 722         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 723             return false
 724         }
 725
 726         let full_path = Path::new(dir).join(path);
 727         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 728             return false;
 729         }
 730
 731         let excludes = [
 732             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 733             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 734             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 735             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 736         ];
 737         !path.iter()
 738              .map(|s| s.to_str().unwrap())
 739              .any(|s| excludes.contains(&s))
 740     }
 741
 742     // Copy the directories using our filter
 743     for item in src_dirs {
 744         let dst = &dst_dir.join(item);
 745         t!(fs::create_dir_all(dst));
 746         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 747     }
 748 }
 749
 750 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 751 pub struct Src;
 752
 753 impl Step for Src {
 754     /// The output path of the src installer tarball
 755     type Output = PathBuf;
 756     const DEFAULT: bool = true;
 757     const ONLY_HOSTS: bool = true;
 758     const ONLY_BUILD_TARGETS: bool = true;
 759     const ONLY_BUILD: bool = true;
 760
 761     fn should_run(run: ShouldRun) -> ShouldRun {
 762         run.path("src")
 763     }
 764
 765     fn make_run(run: RunConfig) {
 766         run.builder.ensure(Src);
 767     }
 768
 769     /// Creates the `rust-src` installer component
 770     fn run(self, builder: &Builder) -> PathBuf {
 771         let build = builder.build;
 772         println!("Dist src");
 773
 774         let name = pkgname(build, "rust-src");
 775         let image = tmpdir(build).join(format!("{}-image", name));
 776         let _ = fs::remove_dir_all(&image);
 777
 778         let dst = image.join("lib/rustlib/src");
 779         let dst_src = dst.join("rust");
 780         t!(fs::create_dir_all(&dst_src));
 781
 782         let src_files = [
 783             "src/Cargo.lock",
 784         ];
 785         // This is the reduced set of paths which will become the rust-src component
 786         // (essentially libstd and all of its path dependencies)
 787         let std_src_dirs = [
 788             "src/build_helper",
 789             "src/dlmalloc",
 790             "src/liballoc",
 791             "src/liballoc_jemalloc",
 792             "src/liballoc_system",
 793             "src/libbacktrace",
 794             "src/libcompiler_builtins",
 795             "src/libcore",
 796             "src/liblibc",
 797             "src/libpanic_abort",
 798             "src/libpanic_unwind",
 799             "src/librustc_asan",
 800             "src/librustc_lsan",
 801             "src/librustc_msan",
 802             "src/librustc_tsan",
 803             "src/libstd",
 804             "src/libstd_unicode",
 805             "src/libunwind",
 806             "src/rustc/compiler_builtins_shim",
 807             "src/rustc/libc_shim",
 808             "src/rustc/dlmalloc_shim",
 809             "src/libtest",
 810             "src/libterm",
 811             "src/jemalloc",
 812             "src/libprofiler_builtins",
 813         ];
 814         let std_src_dirs_exclude = [
 815             "src/libcompiler_builtins/compiler-rt/test",
 816             "src/jemalloc/test/unit",
 817         ];
 818
 819         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 820         for file in src_files.iter() {
 821             copy(&build.src.join(file), &dst_src.join(file));
 822         }
 823
 824         // Create source tarball in rust-installer format
 825         let mut cmd = rust_installer(builder);
 826         cmd.arg("generate")
 827            .arg("--product-name=Rust")
 828            .arg("--rel-manifest-dir=rustlib")
 829            .arg("--success-message=Awesome-Source.")
 830            .arg("--image-dir").arg(&image)
 831            .arg("--work-dir").arg(&tmpdir(build))
 832            .arg("--output-dir").arg(&distdir(build))
 833            .arg(format!("--package-name={}", name))
 834            .arg("--component-name=rust-src")
 835            .arg("--legacy-manifest-dirs=rustlib,cargo");
 836         build.run(&mut cmd);
 837
 838         t!(fs::remove_dir_all(&image));
 839         distdir(build).join(&format!("{}.tar.gz", name))
 840     }
 841 }
 842
 843 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 844
 845 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 846 pub struct PlainSourceTarball;
 847
 848 impl Step for PlainSourceTarball {
 849     /// Produces the location of the tarball generated
 850     type Output = PathBuf;
 851     const DEFAULT: bool = true;
 852     const ONLY_HOSTS: bool = true;
 853     const ONLY_BUILD_TARGETS: bool = true;
 854     const ONLY_BUILD: bool = true;
 855
 856     fn should_run(run: ShouldRun) -> ShouldRun {
 857         let builder = run.builder;
 858         run.path("src").default_condition(builder.config.rust_dist_src)
 859     }
 860
 861     fn make_run(run: RunConfig) {
 862         run.builder.ensure(PlainSourceTarball);
 863     }
 864
 865     /// Creates the plain source tarball
 866     fn run(self, builder: &Builder) -> PathBuf {
 867         let build = builder.build;
 868         println!("Create plain source tarball");
 869
 870         // Make sure that the root folder of tarball has the correct name
 871         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 872         let plain_dst_src = tmpdir(build).join(&plain_name);
 873         let _ = fs::remove_dir_all(&plain_dst_src);
 874         t!(fs::create_dir_all(&plain_dst_src));
 875
 876         // This is the set of root paths which will become part of the source package
 877         let src_files = [
 878             "COPYRIGHT",
 879             "LICENSE-APACHE",
 880             "LICENSE-MIT",
 881             "CONTRIBUTING.md",
 882             "README.md",
 883             "RELEASES.md",
 884             "configure",
 885             "x.py",
 886             "config.toml.example",
 887         ];
 888         let src_dirs = [
 889             "src",
 890         ];
 891
 892         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 893
 894         // Copy the files normally
 895         for item in &src_files {
 896             copy(&build.src.join(item), &plain_dst_src.join(item));
 897         }
 898
 899         // Create the version file
 900         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 901         if let Some(sha) = build.rust_sha() {
 902             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 903         }
 904
 905         // If we're building from git sources, we need to vendor a complete distribution.
 906         if build.rust_info.is_git() {
 907             // Get cargo-vendor installed, if it isn't already.
 908             let mut has_cargo_vendor = false;
 909             let mut cmd = Command::new(&build.initial_cargo);
 910             for line in output(cmd.arg("install").arg("--list")).lines() {
 911                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 912             }
 913             if !has_cargo_vendor {
 914                 let mut cmd = Command::new(&build.initial_cargo);
 915                 cmd.arg("install")
 916                    .arg("--force")
 917                    .arg("--debug")
 918                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 919                    .arg("cargo-vendor")
 920                    .env("RUSTC", &build.initial_rustc);
 921                 if let Some(dir) = build.openssl_install_dir(build.config.build) {
 922                     builder.ensure(native::Openssl {
 923                         target: build.config.build,
 924                     });
 925                     cmd.env("OPENSSL_DIR", dir);
 926                 }
 927                 build.run(&mut cmd);
 928             }
 929
 930             // Vendor all Cargo dependencies
 931             let mut cmd = Command::new(&build.initial_cargo);
 932             cmd.arg("vendor")
 933                .current_dir(&plain_dst_src.join("src"));
 934             build.run(&mut cmd);
 935         }
 936
 937         // Create plain source tarball
 938         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 939         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 940         tarball.set_extension(""); // strip .gz
 941         tarball.set_extension(""); // strip .tar
 942         if let Some(dir) = tarball.parent() {
 943             t!(fs::create_dir_all(dir));
 944         }
 945         println!("running installer");
 946         let mut cmd = rust_installer(builder);
 947         cmd.arg("tarball")
 948            .arg("--input").arg(&plain_name)
 949            .arg("--output").arg(&tarball)
 950            .arg("--work-dir=.")
 951            .current_dir(tmpdir(build));
 952         build.run(&mut cmd);
 953         distdir(build).join(&format!("{}.tar.gz", plain_name))
 954     }
 955 }
 956
 957 fn install(src: &Path, dstdir: &Path, perms: u32) {
 958     let dst = dstdir.join(src.file_name().unwrap());
 959     t!(fs::create_dir_all(dstdir));
 960     drop(fs::remove_file(&dst));
 961     {
 962         let mut s = t!(fs::File::open(&src));
 963         let mut d = t!(fs::File::create(&dst));
 964         io::copy(&mut s, &mut d).expect("failed to copy");
 965     }
 966     chmod(&dst, perms);
 967 }
 968
 969 #[cfg(unix)]
 970 fn chmod(path: &Path, perms: u32) {
 971     use std::os::unix::fs::*;
 972     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 973 }
 974 #[cfg(windows)]
 975 fn chmod(_path: &Path, _perms: u32) {}
 976
 977 // We have to run a few shell scripts, which choke quite a bit on both `\`
 978 // characters and on `C:\` paths, so normalize both of them away.
 979 pub fn sanitize_sh(path: &Path) -> String {
 980     let path = path.to_str().unwrap().replace("\\", "/");
 981     return change_drive(&path).unwrap_or(path);
 982
 983     fn change_drive(s: &str) -> Option<String> {
 984         let mut ch = s.chars();
 985         let drive = ch.next().unwrap_or('C');
 986         if ch.next() != Some(':') {
 987             return None
 988         }
 989         if ch.next() != Some('/') {
 990             return None
 991         }
 992         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 993     }
 994 }
 995
 996 fn write_file(path: &Path, data: &[u8]) {
 997     let mut vf = t!(fs::File::create(path));
 998     t!(vf.write_all(data));
 999 }
1000
1001 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1002 pub struct Cargo {
1003     pub stage: u32,
1004     pub target: Interned<String>,
1005 }
1006
1007 impl Step for Cargo {
1008     type Output = PathBuf;
1009     const ONLY_BUILD_TARGETS: bool = true;
1010     const ONLY_HOSTS: bool = true;
1011
1012     fn should_run(run: ShouldRun) -> ShouldRun {
1013         run.path("cargo")
1014     }
1015
1016     fn make_run(run: RunConfig) {
1017         run.builder.ensure(Cargo {
1018             stage: run.builder.top_stage,
1019             target: run.target,
1020         });
1021     }
1022
1023     fn run(self, builder: &Builder) -> PathBuf {
1024         let build = builder.build;
1025         let stage = self.stage;
1026         let target = self.target;
1027
1028         println!("Dist cargo stage{} ({})", stage, target);
1029         let src = build.src.join("src/tools/cargo");
1030         let etc = src.join("src/etc");
1031         let release_num = build.release_num("cargo");
1032         let name = pkgname(build, "cargo");
1033         let version = builder.cargo_info.version(build, &release_num);
1034
1035         let tmp = tmpdir(build);
1036         let image = tmp.join("cargo-image");
1037         drop(fs::remove_dir_all(&image));
1038         t!(fs::create_dir_all(&image));
1039
1040         // Prepare the image directory
1041         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1042         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1043         let cargo = builder.ensure(tool::Cargo {
1044             compiler: builder.compiler(stage, build.build),
1045             target
1046         });
1047         install(&cargo, &image.join("bin"), 0o755);
1048         for man in t!(etc.join("man").read_dir()) {
1049             let man = t!(man);
1050             install(&man.path(), &image.join("share/man/man1"), 0o644);
1051         }
1052         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1053         copy(&etc.join("cargo.bashcomp.sh"),
1054              &image.join("etc/bash_completion.d/cargo"));
1055         let doc = image.join("share/doc/cargo");
1056         install(&src.join("README.md"), &doc, 0o644);
1057         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1058         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1059         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1060
1061         // Prepare the overlay
1062         let overlay = tmp.join("cargo-overlay");
1063         drop(fs::remove_dir_all(&overlay));
1064         t!(fs::create_dir_all(&overlay));
1065         install(&src.join("README.md"), &overlay, 0o644);
1066         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1067         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1068         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1069         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1070
1071         // Generate the installer tarball
1072         let mut cmd = rust_installer(builder);
1073         cmd.arg("generate")
1074            .arg("--product-name=Rust")
1075            .arg("--rel-manifest-dir=rustlib")
1076            .arg("--success-message=Rust-is-ready-to-roll.")
1077            .arg("--image-dir").arg(&image)
1078            .arg("--work-dir").arg(&tmpdir(build))
1079            .arg("--output-dir").arg(&distdir(build))
1080            .arg("--non-installed-overlay").arg(&overlay)
1081            .arg(format!("--package-name={}-{}", name, target))
1082            .arg("--component-name=cargo")
1083            .arg("--legacy-manifest-dirs=rustlib,cargo");
1084         build.run(&mut cmd);
1085         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1086     }
1087 }
1088
1089 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1090 pub struct Rls {
1091     pub stage: u32,
1092     pub target: Interned<String>,
1093 }
1094
1095 impl Step for Rls {
1096     type Output = Option<PathBuf>;
1097     const ONLY_BUILD_TARGETS: bool = true;
1098     const ONLY_HOSTS: bool = true;
1099
1100     fn should_run(run: ShouldRun) -> ShouldRun {
1101         run.path("rls")
1102     }
1103
1104     fn make_run(run: RunConfig) {
1105         run.builder.ensure(Rls {
1106             stage: run.builder.top_stage,
1107             target: run.target,
1108         });
1109     }
1110
1111     fn run(self, builder: &Builder) -> Option<PathBuf> {
1112         let build = builder.build;
1113         let stage = self.stage;
1114         let target = self.target;
1115         assert!(build.config.extended);
1116
1117         println!("Dist RLS stage{} ({})", stage, target);
1118         let src = build.src.join("src/tools/rls");
1119         let release_num = build.release_num("rls");
1120         let name = pkgname(build, "rls");
1121         let version = build.rls_info.version(build, &release_num);
1122
1123         let tmp = tmpdir(build);
1124         let image = tmp.join("rls-image");
1125         drop(fs::remove_dir_all(&image));
1126         t!(fs::create_dir_all(&image));
1127
1128         // Prepare the image directory
1129         // We expect RLS to build, because we've exited this step above if tool
1130         // state for RLS isn't testing.
1131         let rls = builder.ensure(tool::Rls {
1132             compiler: builder.compiler(stage, build.build),
1133             target
1134         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1135
1136         install(&rls, &image.join("bin"), 0o755);
1137         let doc = image.join("share/doc/rls");
1138         install(&src.join("README.md"), &doc, 0o644);
1139         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1140         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1141
1142         // Prepare the overlay
1143         let overlay = tmp.join("rls-overlay");
1144         drop(fs::remove_dir_all(&overlay));
1145         t!(fs::create_dir_all(&overlay));
1146         install(&src.join("README.md"), &overlay, 0o644);
1147         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1148         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1149         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1150
1151         // Generate the installer tarball
1152         let mut cmd = rust_installer(builder);
1153         cmd.arg("generate")
1154            .arg("--product-name=Rust")
1155            .arg("--rel-manifest-dir=rustlib")
1156            .arg("--success-message=RLS-ready-to-serve.")
1157            .arg("--image-dir").arg(&image)
1158            .arg("--work-dir").arg(&tmpdir(build))
1159            .arg("--output-dir").arg(&distdir(build))
1160            .arg("--non-installed-overlay").arg(&overlay)
1161            .arg(format!("--package-name={}-{}", name, target))
1162            .arg("--legacy-manifest-dirs=rustlib,cargo")
1163            .arg("--component-name=rls-preview");
1164
1165         build.run(&mut cmd);
1166         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1167     }
1168 }
1169
1170
1171 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1172 pub struct Rustfmt {
1173     pub stage: u32,
1174     pub target: Interned<String>,
1175 }
1176
1177 impl Step for Rustfmt {
1178     type Output = Option<PathBuf>;
1179     const ONLY_BUILD_TARGETS: bool = true;
1180     const ONLY_HOSTS: bool = true;
1181
1182     fn should_run(run: ShouldRun) -> ShouldRun {
1183         run.path("rustfmt")
1184     }
1185
1186     fn make_run(run: RunConfig) {
1187         run.builder.ensure(Rustfmt {
1188             stage: run.builder.top_stage,
1189             target: run.target,
1190         });
1191     }
1192
1193     fn run(self, builder: &Builder) -> Option<PathBuf> {
1194         let build = builder.build;
1195         let stage = self.stage;
1196         let target = self.target;
1197         assert!(build.config.extended);
1198
1199         println!("Dist Rustfmt stage{} ({})", stage, target);
1200         let src = build.src.join("src/tools/rustfmt");
1201         let release_num = build.release_num("rustfmt");
1202         let name = pkgname(build, "rustfmt");
1203         let version = build.rustfmt_info.version(build, &release_num);
1204
1205         let tmp = tmpdir(build);
1206         let image = tmp.join("rustfmt-image");
1207         drop(fs::remove_dir_all(&image));
1208         t!(fs::create_dir_all(&image));
1209
1210         // Prepare the image directory
1211         let rustfmt = builder.ensure(tool::Rustfmt {
1212             compiler: builder.compiler(stage, build.build),
1213             target
1214         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1215         let cargofmt = builder.ensure(tool::Cargofmt {
1216             compiler: builder.compiler(stage, build.build),
1217             target
1218         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1219
1220         install(&rustfmt, &image.join("bin"), 0o755);
1221         install(&cargofmt, &image.join("bin"), 0o755);
1222         let doc = image.join("share/doc/rustfmt");
1223         install(&src.join("README.md"), &doc, 0o644);
1224         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1225         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1226
1227         // Prepare the overlay
1228         let overlay = tmp.join("rustfmt-overlay");
1229         drop(fs::remove_dir_all(&overlay));
1230         t!(fs::create_dir_all(&overlay));
1231         install(&src.join("README.md"), &overlay, 0o644);
1232         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1233         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1234         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1235
1236         // Generate the installer tarball
1237         let mut cmd = rust_installer(builder);
1238         cmd.arg("generate")
1239            .arg("--product-name=Rust")
1240            .arg("--rel-manifest-dir=rustlib")
1241            .arg("--success-message=rustfmt-ready-to-fmt.")
1242            .arg("--image-dir").arg(&image)
1243            .arg("--work-dir").arg(&tmpdir(build))
1244            .arg("--output-dir").arg(&distdir(build))
1245            .arg("--non-installed-overlay").arg(&overlay)
1246            .arg(format!("--package-name={}-{}", name, target))
1247            .arg("--legacy-manifest-dirs=rustlib,cargo")
1248            .arg("--component-name=rustfmt-preview");
1249
1250         build.run(&mut cmd);
1251         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1252     }
1253 }
1254
1255 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1256 pub struct Extended {
1257     stage: u32,
1258     host: Interned<String>,
1259     target: Interned<String>,
1260 }
1261
1262 impl Step for Extended {
1263     type Output = ();
1264     const DEFAULT: bool = true;
1265     const ONLY_BUILD_TARGETS: bool = true;
1266     const ONLY_HOSTS: bool = true;
1267
1268     fn should_run(run: ShouldRun) -> ShouldRun {
1269         let builder = run.builder;
1270         run.path("extended").default_condition(builder.config.extended)
1271     }
1272
1273     fn make_run(run: RunConfig) {
1274         run.builder.ensure(Extended {
1275             stage: run.builder.top_stage,
1276             host: run.host,
1277             target: run.target,
1278         });
1279     }
1280
1281     /// Creates a combined installer for the specified target in the provided stage.
1282     fn run(self, builder: &Builder) {
1283         let build = builder.build;
1284         let stage = self.stage;
1285         let target = self.target;
1286
1287         println!("Dist extended stage{} ({})", stage, target);
1288
1289         let rustc_installer = builder.ensure(Rustc {
1290             compiler: builder.compiler(stage, target),
1291         });
1292         let cargo_installer = builder.ensure(Cargo { stage, target });
1293         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1294         let rls_installer = builder.ensure(Rls { stage, target });
1295         let mingw_installer = builder.ensure(Mingw { host: target });
1296         let analysis_installer = builder.ensure(Analysis {
1297             compiler: builder.compiler(stage, self.host),
1298             target
1299         });
1300
1301         let docs_installer = builder.ensure(Docs { stage, host: target, });
1302         let std_installer = builder.ensure(Std {
1303             compiler: builder.compiler(stage, self.host),
1304             target,
1305         });
1306
1307         let tmp = tmpdir(build);
1308         let overlay = tmp.join("extended-overlay");
1309         let etc = build.src.join("src/etc/installer");
1310         let work = tmp.join("work");
1311
1312         let _ = fs::remove_dir_all(&overlay);
1313         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1314         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1315         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1316         let version = build.rust_version();
1317         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1318         if let Some(sha) = build.rust_sha() {
1319             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1320         }
1321         install(&etc.join("README.md"), &overlay, 0o644);
1322
1323         // When rust-std package split from rustc, we needed to ensure that during
1324         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1325         // the std files during uninstall. To do this ensure that rustc comes
1326         // before rust-std in the list below.
1327         let mut tarballs = Vec::new();
1328         tarballs.push(rustc_installer);
1329         tarballs.push(cargo_installer);
1330         tarballs.extend(rls_installer.clone());
1331         tarballs.extend(rustfmt_installer.clone());
1332         tarballs.push(analysis_installer);
1333         tarballs.push(std_installer);
1334         if build.config.docs {
1335             tarballs.push(docs_installer);
1336         }
1337         if target.contains("pc-windows-gnu") {
1338             tarballs.push(mingw_installer.unwrap());
1339         }
1340         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1341         for tarball in &tarballs[1..] {
1342             input_tarballs.push(",");
1343             input_tarballs.push(tarball);
1344         }
1345
1346         let mut cmd = rust_installer(builder);
1347         cmd.arg("combine")
1348             .arg("--product-name=Rust")
1349             .arg("--rel-manifest-dir=rustlib")
1350             .arg("--success-message=Rust-is-ready-to-roll.")
1351             .arg("--work-dir").arg(&work)
1352             .arg("--output-dir").arg(&distdir(build))
1353             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1354             .arg("--legacy-manifest-dirs=rustlib,cargo")
1355             .arg("--input-tarballs").arg(input_tarballs)
1356             .arg("--non-installed-overlay").arg(&overlay);
1357         build.run(&mut cmd);
1358
1359         let mut license = String::new();
1360         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1361         license.push_str("\n");
1362         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1363         license.push_str("\n");
1364         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1365
1366         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1367         let mut rtf = rtf.to_string();
1368         rtf.push_str("\n");
1369         for line in license.lines() {
1370             rtf.push_str(line);
1371             rtf.push_str("\\line ");
1372         }
1373         rtf.push_str("}");
1374
1375         fn filter(contents: &str, marker: &str) -> String {
1376             let start = format!("tool-{}-start", marker);
1377             let end = format!("tool-{}-end", marker);
1378             let mut lines = Vec::new();
1379             let mut omitted = false;
1380             for line in contents.lines() {
1381                 if line.contains(&start) {
1382                     omitted = true;
1383                 } else if line.contains(&end) {
1384                     omitted = false;
1385                 } else if !omitted {
1386                     lines.push(line);
1387                 }
1388             }
1389
1390             lines.join("\n")
1391         }
1392
1393         let xform = |p: &Path| {
1394             let mut contents = String::new();
1395             t!(t!(File::open(p)).read_to_string(&mut contents));
1396             if rls_installer.is_none() {
1397                 contents = filter(&contents, "rls");
1398             }
1399             if rustfmt_installer.is_none() {
1400                 contents = filter(&contents, "rustfmt");
1401             }
1402             let ret = tmp.join(p.file_name().unwrap());
1403             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1404             return ret
1405         };
1406
1407         if target.contains("apple-darwin") {
1408             let pkg = tmp.join("pkg");
1409             let _ = fs::remove_dir_all(&pkg);
1410
1411             let pkgbuild = |component: &str| {
1412                 let mut cmd = Command::new("pkgbuild");
1413                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1414                     .arg("--scripts").arg(pkg.join(component))
1415                     .arg("--nopayload")
1416                     .arg(pkg.join(component).with_extension("pkg"));
1417                 build.run(&mut cmd);
1418             };
1419
1420             let prepare = |name: &str| {
1421                 t!(fs::create_dir_all(pkg.join(name)));
1422                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1423                         &pkg.join(name));
1424                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1425                 pkgbuild(name);
1426             };
1427             prepare("rustc");
1428             prepare("cargo");
1429             prepare("rust-docs");
1430             prepare("rust-std");
1431             prepare("rust-analysis");
1432
1433             if rls_installer.is_some() {
1434                 prepare("rls");
1435             }
1436
1437             // create an 'uninstall' package
1438             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1439             pkgbuild("uninstall");
1440
1441             t!(fs::create_dir_all(pkg.join("res")));
1442             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1443             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1444             let mut cmd = Command::new("productbuild");
1445             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1446                 .arg("--resources").arg(pkg.join("res"))
1447                 .arg(distdir(build).join(format!("{}-{}.pkg",
1448                                                     pkgname(build, "rust"),
1449                                                     target)))
1450                 .arg("--package-path").arg(&pkg);
1451             build.run(&mut cmd);
1452         }
1453
1454         if target.contains("windows") {
1455             let exe = tmp.join("exe");
1456             let _ = fs::remove_dir_all(&exe);
1457
1458             let prepare = |name: &str| {
1459                 t!(fs::create_dir_all(exe.join(name)));
1460                 let dir = if name == "rust-std" || name == "rust-analysis" {
1461                     format!("{}-{}", name, target)
1462                 } else if name == "rls" {
1463                     "rls-preview".to_string()
1464                 } else {
1465                     name.to_string()
1466                 };
1467                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1468                             .join(dir),
1469                         &exe.join(name));
1470                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1471             };
1472             prepare("rustc");
1473             prepare("cargo");
1474             prepare("rust-analysis");
1475             prepare("rust-docs");
1476             prepare("rust-std");
1477             if rls_installer.is_some() {
1478                 prepare("rls");
1479             }
1480             if target.contains("windows-gnu") {
1481                 prepare("rust-mingw");
1482             }
1483
1484             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1485             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1486             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1487             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1488             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1489
1490             // Generate exe installer
1491             let mut cmd = Command::new("iscc");
1492             cmd.arg("rust.iss")
1493                 .current_dir(&exe);
1494             if target.contains("windows-gnu") {
1495                 cmd.arg("/dMINGW");
1496             }
1497             add_env(build, &mut cmd, target);
1498             build.run(&mut cmd);
1499             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1500                     &distdir(build),
1501                     0o755);
1502
1503             // Generate msi installer
1504             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1505             let heat = wix.join("bin/heat.exe");
1506             let candle = wix.join("bin/candle.exe");
1507             let light = wix.join("bin/light.exe");
1508
1509             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1510             build.run(Command::new(&heat)
1511                             .current_dir(&exe)
1512                             .arg("dir")
1513                             .arg("rustc")
1514                             .args(&heat_flags)
1515                             .arg("-cg").arg("RustcGroup")
1516                             .arg("-dr").arg("Rustc")
1517                             .arg("-var").arg("var.RustcDir")
1518                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1519             build.run(Command::new(&heat)
1520                             .current_dir(&exe)
1521                             .arg("dir")
1522                             .arg("rust-docs")
1523                             .args(&heat_flags)
1524                             .arg("-cg").arg("DocsGroup")
1525                             .arg("-dr").arg("Docs")
1526                             .arg("-var").arg("var.DocsDir")
1527                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1528                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1529             build.run(Command::new(&heat)
1530                             .current_dir(&exe)
1531                             .arg("dir")
1532                             .arg("cargo")
1533                             .args(&heat_flags)
1534                             .arg("-cg").arg("CargoGroup")
1535                             .arg("-dr").arg("Cargo")
1536                             .arg("-var").arg("var.CargoDir")
1537                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1538                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1539             build.run(Command::new(&heat)
1540                             .current_dir(&exe)
1541                             .arg("dir")
1542                             .arg("rust-std")
1543                             .args(&heat_flags)
1544                             .arg("-cg").arg("StdGroup")
1545                             .arg("-dr").arg("Std")
1546                             .arg("-var").arg("var.StdDir")
1547                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1548             if rls_installer.is_some() {
1549                 build.run(Command::new(&heat)
1550                                 .current_dir(&exe)
1551                                 .arg("dir")
1552                                 .arg("rls")
1553                                 .args(&heat_flags)
1554                                 .arg("-cg").arg("RlsGroup")
1555                                 .arg("-dr").arg("Rls")
1556                                 .arg("-var").arg("var.RlsDir")
1557                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1558                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1559             }
1560             build.run(Command::new(&heat)
1561                             .current_dir(&exe)
1562                             .arg("dir")
1563                             .arg("rust-analysis")
1564                             .args(&heat_flags)
1565                             .arg("-cg").arg("AnalysisGroup")
1566                             .arg("-dr").arg("Analysis")
1567                             .arg("-var").arg("var.AnalysisDir")
1568                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1569                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1570             if target.contains("windows-gnu") {
1571                 build.run(Command::new(&heat)
1572                                 .current_dir(&exe)
1573                                 .arg("dir")
1574                                 .arg("rust-mingw")
1575                                 .args(&heat_flags)
1576                                 .arg("-cg").arg("GccGroup")
1577                                 .arg("-dr").arg("Gcc")
1578                                 .arg("-var").arg("var.GccDir")
1579                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1580             }
1581
1582             let candle = |input: &Path| {
1583                 let output = exe.join(input.file_stem().unwrap())
1584                                 .with_extension("wixobj");
1585                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1586                 let mut cmd = Command::new(&candle);
1587                 cmd.current_dir(&exe)
1588                     .arg("-nologo")
1589                     .arg("-dRustcDir=rustc")
1590                     .arg("-dDocsDir=rust-docs")
1591                     .arg("-dCargoDir=cargo")
1592                     .arg("-dStdDir=rust-std")
1593                     .arg("-dAnalysisDir=rust-analysis")
1594                     .arg("-arch").arg(&arch)
1595                     .arg("-out").arg(&output)
1596                     .arg(&input);
1597                 add_env(build, &mut cmd, target);
1598
1599                 if rls_installer.is_some() {
1600                     cmd.arg("-dRlsDir=rls");
1601                 }
1602                 if target.contains("windows-gnu") {
1603                     cmd.arg("-dGccDir=rust-mingw");
1604                 }
1605                 build.run(&mut cmd);
1606             };
1607             candle(&xform(&etc.join("msi/rust.wxs")));
1608             candle(&etc.join("msi/ui.wxs"));
1609             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1610             candle("RustcGroup.wxs".as_ref());
1611             candle("DocsGroup.wxs".as_ref());
1612             candle("CargoGroup.wxs".as_ref());
1613             candle("StdGroup.wxs".as_ref());
1614             if rls_installer.is_some() {
1615                 candle("RlsGroup.wxs".as_ref());
1616             }
1617             candle("AnalysisGroup.wxs".as_ref());
1618
1619             if target.contains("windows-gnu") {
1620                 candle("GccGroup.wxs".as_ref());
1621             }
1622
1623             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1624             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1625             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1626
1627             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1628             let mut cmd = Command::new(&light);
1629             cmd.arg("-nologo")
1630                 .arg("-ext").arg("WixUIExtension")
1631                 .arg("-ext").arg("WixUtilExtension")
1632                 .arg("-out").arg(exe.join(&filename))
1633                 .arg("rust.wixobj")
1634                 .arg("ui.wixobj")
1635                 .arg("rustwelcomedlg.wixobj")
1636                 .arg("RustcGroup.wixobj")
1637                 .arg("DocsGroup.wixobj")
1638                 .arg("CargoGroup.wixobj")
1639                 .arg("StdGroup.wixobj")
1640                 .arg("AnalysisGroup.wixobj")
1641                 .current_dir(&exe);
1642
1643             if rls_installer.is_some() {
1644                 cmd.arg("RlsGroup.wixobj");
1645             }
1646
1647             if target.contains("windows-gnu") {
1648                 cmd.arg("GccGroup.wixobj");
1649             }
1650             // ICE57 wrongly complains about the shortcuts
1651             cmd.arg("-sice:ICE57");
1652
1653             build.run(&mut cmd);
1654
1655             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1656         }
1657     }
1658 }
1659
1660 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1661     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1662     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1663        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1664        .env("CFG_RELEASE", build.rust_release())
1665        .env("CFG_VER_MAJOR", parts.next().unwrap())
1666        .env("CFG_VER_MINOR", parts.next().unwrap())
1667        .env("CFG_VER_PATCH", parts.next().unwrap())
1668        .env("CFG_VER_BUILD", "0") // just needed to build
1669        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1670        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1671        .env("CFG_BUILD", target)
1672        .env("CFG_CHANNEL", &build.config.channel);
1673
1674     if target.contains("windows-gnu") {
1675        cmd.env("CFG_MINGW", "1")
1676           .env("CFG_ABI", "GNU");
1677     } else {
1678        cmd.env("CFG_MINGW", "0")
1679           .env("CFG_ABI", "MSVC");
1680     }
1681
1682     if target.contains("x86_64") {
1683        cmd.env("CFG_PLATFORM", "x64");
1684     } else {
1685        cmd.env("CFG_PLATFORM", "x86");
1686     }
1687 }
1688
1689 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1690 pub struct HashSign;
1691
1692 impl Step for HashSign {
1693     type Output = ();
1694     const ONLY_BUILD_TARGETS: bool = true;
1695     const ONLY_HOSTS: bool = true;
1696     const ONLY_BUILD: bool = true;
1697
1698     fn should_run(run: ShouldRun) -> ShouldRun {
1699         run.path("hash-and-sign")
1700     }
1701
1702     fn make_run(run: RunConfig) {
1703         run.builder.ensure(HashSign);
1704     }
1705
1706     fn run(self, builder: &Builder) {
1707         let build = builder.build;
1708         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1709         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1710             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1711         });
1712         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1713             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1714         });
1715         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1716             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1717         });
1718         let mut pass = String::new();
1719         t!(t!(File::open(&file)).read_to_string(&mut pass));
1720
1721         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1722
1723         cmd.arg(sign);
1724         cmd.arg(distdir(build));
1725         cmd.arg(today.trim());
1726         cmd.arg(build.rust_package_vers());
1727         cmd.arg(build.package_vers(&build.release_num("cargo")));
1728         cmd.arg(build.package_vers(&build.release_num("rls")));
1729         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1730         cmd.arg(addr);
1731
1732         t!(fs::create_dir_all(distdir(build)));
1733
1734         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1735         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1736         let status = t!(child.wait());
1737         assert!(status.success());
1738     }
1739 }