src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36
  37 pub fn pkgname(build: &Build, component: &str) -> String {
  38     if component == "cargo" {
  39         format!("{}-{}", component, build.cargo_package_vers())
  40     } else if component == "rls" {
  41         format!("{}-{}", component, build.rls_package_vers())
  42     } else if component == "rustfmt" {
  43         format!("{}-{}", component, build.rustfmt_package_vers())
  44     } else {
  45         assert!(component.starts_with("rust"));
  46         format!("{}-{}", component, build.rust_package_vers())
  47     }
  48 }
  49
  50 fn distdir(build: &Build) -> PathBuf {
  51     build.out.join("dist")
  52 }
  53
  54 pub fn tmpdir(build: &Build) -> PathBuf {
  55     build.out.join("tmp/dist")
  56 }
  57
  58 fn rust_installer(builder: &Builder) -> Command {
  59     builder.tool_cmd(Tool::RustInstaller)
  60 }
  61
  62 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  63 pub struct Docs {
  64     pub stage: u32,
  65     pub host: Interned<String>,
  66 }
  67
  68 impl Step for Docs {
  69     type Output = PathBuf;
  70     const DEFAULT: bool = true;
  71     const ONLY_BUILD_TARGETS: bool = true;
  72
  73     fn should_run(run: ShouldRun) -> ShouldRun {
  74         run.path("src/doc")
  75     }
  76
  77     fn make_run(run: RunConfig) {
  78         run.builder.ensure(Docs {
  79             stage: run.builder.top_stage,
  80             host: run.target,
  81         });
  82     }
  83
  84     /// Builds the `rust-docs` installer component.
  85     fn run(self, builder: &Builder) -> PathBuf {
  86         let build = builder.build;
  87         let host = self.host;
  88
  89         let name = pkgname(build, "rust-docs");
  90
  91         println!("Dist docs ({})", host);
  92         if !build.config.docs {
  93             println!("\tskipping - docs disabled");
  94             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  95         }
  96
  97         builder.default_doc(None);
  98
  99         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 100         let _ = fs::remove_dir_all(&image);
 101
 102         let dst = image.join("share/doc/rust/html");
 103         t!(fs::create_dir_all(&dst));
 104         let src = build.out.join(host).join("doc");
 105         cp_r(&src, &dst);
 106
 107         let mut cmd = rust_installer(builder);
 108         cmd.arg("generate")
 109            .arg("--product-name=Rust-Documentation")
 110            .arg("--rel-manifest-dir=rustlib")
 111            .arg("--success-message=Rust-documentation-is-installed.")
 112            .arg("--image-dir").arg(&image)
 113            .arg("--work-dir").arg(&tmpdir(build))
 114            .arg("--output-dir").arg(&distdir(build))
 115            .arg(format!("--package-name={}-{}", name, host))
 116            .arg("--component-name=rust-docs")
 117            .arg("--legacy-manifest-dirs=rustlib,cargo")
 118            .arg("--bulk-dirs=share/doc/rust/html");
 119         build.run(&mut cmd);
 120         t!(fs::remove_dir_all(&image));
 121
 122         // As part of this step, *also* copy the docs directory to a directory which
 123         // buildbot typically uploads.
 124         if host == build.build {
 125             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 126             t!(fs::create_dir_all(&dst));
 127             cp_r(&src, &dst);
 128         }
 129
 130         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 131     }
 132 }
 133
 134 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 135     let mut found = Vec::with_capacity(files.len());
 136
 137     for file in files {
 138         let file_path =
 139             path.iter()
 140                 .map(|dir| dir.join(file))
 141                 .find(|p| p.exists());
 142
 143         if let Some(file_path) = file_path {
 144             found.push(file_path);
 145         } else {
 146             panic!("Could not find '{}' in {:?}", file, path);
 147         }
 148     }
 149
 150     found
 151 }
 152
 153 fn make_win_dist(
 154     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 155 ) {
 156     //Ask gcc where it keeps its stuff
 157     let mut cmd = Command::new(build.cc(target_triple));
 158     cmd.arg("-print-search-dirs");
 159     let gcc_out = output(&mut cmd);
 160
 161     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 162     let mut lib_path = Vec::new();
 163
 164     for line in gcc_out.lines() {
 165         let idx = line.find(':').unwrap();
 166         let key = &line[..idx];
 167         let trim_chars: &[_] = &[' ', '='];
 168         let value =
 169             line[(idx + 1)..]
 170                 .trim_left_matches(trim_chars)
 171                 .split(';')
 172                 .map(PathBuf::from);
 173
 174         if key == "programs" {
 175             bin_path.extend(value);
 176         } else if key == "libraries" {
 177             lib_path.extend(value);
 178         }
 179     }
 180
 181     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 182     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 183     if target_triple.starts_with("i686-") {
 184         rustc_dlls.push("libgcc_s_dw2-1.dll");
 185     } else {
 186         rustc_dlls.push("libgcc_s_seh-1.dll");
 187     }
 188
 189     let target_libs = [ //MinGW libs
 190         "libgcc.a",
 191         "libgcc_eh.a",
 192         "libgcc_s.a",
 193         "libm.a",
 194         "libmingw32.a",
 195         "libmingwex.a",
 196         "libstdc++.a",
 197         "libiconv.a",
 198         "libmoldname.a",
 199         "libpthread.a",
 200         //Windows import libs
 201         "libadvapi32.a",
 202         "libbcrypt.a",
 203         "libcomctl32.a",
 204         "libcomdlg32.a",
 205         "libcrypt32.a",
 206         "libgdi32.a",
 207         "libimagehlp.a",
 208         "libiphlpapi.a",
 209         "libkernel32.a",
 210         "libmsvcrt.a",
 211         "libodbc32.a",
 212         "libole32.a",
 213         "liboleaut32.a",
 214         "libopengl32.a",
 215         "libpsapi.a",
 216         "librpcrt4.a",
 217         "libsetupapi.a",
 218         "libshell32.a",
 219         "libuser32.a",
 220         "libuserenv.a",
 221         "libuuid.a",
 222         "libwinhttp.a",
 223         "libwinmm.a",
 224         "libwinspool.a",
 225         "libws2_32.a",
 226         "libwsock32.a",
 227         "libdbghelp.a",
 228         "libmsimg32.a",
 229     ];
 230
 231     //Find mingw artifacts we want to bundle
 232     let target_tools = find_files(&target_tools, &bin_path);
 233     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 234     let target_libs = find_files(&target_libs, &lib_path);
 235
 236     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 237         let file_name = src.file_name().unwrap();
 238         let dest = dest_folder.join(file_name);
 239         copy(src, &dest);
 240     }
 241
 242     //Copy runtime dlls next to rustc.exe
 243     let dist_bin_dir = rust_root.join("bin/");
 244     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 245     for src in rustc_dlls {
 246         copy_to_folder(&src, &dist_bin_dir);
 247     }
 248
 249     //Copy platform tools to platform-specific bin directory
 250     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 251     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 252     for src in target_tools {
 253         copy_to_folder(&src, &target_bin_dir);
 254     }
 255
 256     //Copy platform libs to platform-specific lib directory
 257     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 258     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 259     for src in target_libs {
 260         copy_to_folder(&src, &target_lib_dir);
 261     }
 262 }
 263
 264 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 265 pub struct Mingw {
 266     host: Interned<String>,
 267 }
 268
 269 impl Step for Mingw {
 270     type Output = Option<PathBuf>;
 271     const DEFAULT: bool = true;
 272     const ONLY_BUILD_TARGETS: bool = true;
 273
 274     fn should_run(run: ShouldRun) -> ShouldRun {
 275         run.never()
 276     }
 277
 278     fn make_run(run: RunConfig) {
 279         run.builder.ensure(Mingw { host: run.target });
 280     }
 281
 282     /// Build the `rust-mingw` installer component.
 283     ///
 284     /// This contains all the bits and pieces to run the MinGW Windows targets
 285     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 286     fn run(self, builder: &Builder) -> Option<PathBuf> {
 287         let build = builder.build;
 288         let host = self.host;
 289
 290         if !host.contains("pc-windows-gnu") {
 291             return None;
 292         }
 293
 294         println!("Dist mingw ({})", host);
 295         let name = pkgname(build, "rust-mingw");
 296         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 297         let _ = fs::remove_dir_all(&image);
 298         t!(fs::create_dir_all(&image));
 299
 300         // The first argument is a "temporary directory" which is just
 301         // thrown away (this contains the runtime DLLs included in the rustc package
 302         // above) and the second argument is where to place all the MinGW components
 303         // (which is what we want).
 304         make_win_dist(&tmpdir(build), &image, host, &build);
 305
 306         let mut cmd = rust_installer(builder);
 307         cmd.arg("generate")
 308            .arg("--product-name=Rust-MinGW")
 309            .arg("--rel-manifest-dir=rustlib")
 310            .arg("--success-message=Rust-MinGW-is-installed.")
 311            .arg("--image-dir").arg(&image)
 312            .arg("--work-dir").arg(&tmpdir(build))
 313            .arg("--output-dir").arg(&distdir(build))
 314            .arg(format!("--package-name={}-{}", name, host))
 315            .arg("--component-name=rust-mingw")
 316            .arg("--legacy-manifest-dirs=rustlib,cargo");
 317         build.run(&mut cmd);
 318         t!(fs::remove_dir_all(&image));
 319         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 320     }
 321 }
 322
 323 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 324 pub struct Rustc {
 325     pub compiler: Compiler,
 326 }
 327
 328 impl Step for Rustc {
 329     type Output = PathBuf;
 330     const DEFAULT: bool = true;
 331     const ONLY_HOSTS: bool = true;
 332     const ONLY_BUILD_TARGETS: bool = true;
 333
 334     fn should_run(run: ShouldRun) -> ShouldRun {
 335         run.path("src/librustc")
 336     }
 337
 338     fn make_run(run: RunConfig) {
 339         run.builder.ensure(Rustc {
 340             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 341         });
 342     }
 343
 344     /// Creates the `rustc` installer component.
 345     fn run(self, builder: &Builder) -> PathBuf {
 346         let build = builder.build;
 347         let compiler = self.compiler;
 348         let host = self.compiler.host;
 349
 350         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 351         let name = pkgname(build, "rustc");
 352         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 353         let _ = fs::remove_dir_all(&image);
 354         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 355         let _ = fs::remove_dir_all(&overlay);
 356
 357         // Prepare the rustc "image", what will actually end up getting installed
 358         prepare_image(builder, compiler, &image);
 359
 360         // Prepare the overlay which is part of the tarball but won't actually be
 361         // installed
 362         let cp = |file: &str| {
 363             install(&build.src.join(file), &overlay, 0o644);
 364         };
 365         cp("COPYRIGHT");
 366         cp("LICENSE-APACHE");
 367         cp("LICENSE-MIT");
 368         cp("README.md");
 369         // tiny morsel of metadata is used by rust-packaging
 370         let version = build.rust_version();
 371         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 372         if let Some(sha) = build.rust_sha() {
 373             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 374         }
 375
 376         // On MinGW we've got a few runtime DLL dependencies that we need to
 377         // include. The first argument to this script is where to put these DLLs
 378         // (the image we're creating), and the second argument is a junk directory
 379         // to ignore all other MinGW stuff the script creates.
 380         //
 381         // On 32-bit MinGW we're always including a DLL which needs some extra
 382         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 383         // anything requiring us to distribute a license, but it's likely the
 384         // install will *also* include the rust-mingw package, which also needs
 385         // licenses, so to be safe we just include it here in all MinGW packages.
 386         if host.contains("pc-windows-gnu") {
 387             make_win_dist(&image, &tmpdir(build), host, build);
 388
 389             let dst = image.join("share/doc");
 390             t!(fs::create_dir_all(&dst));
 391             cp_r(&build.src.join("src/etc/third-party"), &dst);
 392         }
 393
 394         // Finally, wrap everything up in a nice tarball!
 395         let mut cmd = rust_installer(builder);
 396         cmd.arg("generate")
 397            .arg("--product-name=Rust")
 398            .arg("--rel-manifest-dir=rustlib")
 399            .arg("--success-message=Rust-is-ready-to-roll.")
 400            .arg("--image-dir").arg(&image)
 401            .arg("--work-dir").arg(&tmpdir(build))
 402            .arg("--output-dir").arg(&distdir(build))
 403            .arg("--non-installed-overlay").arg(&overlay)
 404            .arg(format!("--package-name={}-{}", name, host))
 405            .arg("--component-name=rustc")
 406            .arg("--legacy-manifest-dirs=rustlib,cargo");
 407         build.run(&mut cmd);
 408         t!(fs::remove_dir_all(&image));
 409         t!(fs::remove_dir_all(&overlay));
 410
 411         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 412
 413         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 414             let host = compiler.host;
 415             let build = builder.build;
 416             let src = builder.sysroot(compiler);
 417             let libdir = libdir(&host);
 418
 419             // Copy rustc/rustdoc binaries
 420             t!(fs::create_dir_all(image.join("bin")));
 421             cp_r(&src.join("bin"), &image.join("bin"));
 422
 423             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 424
 425             // Copy runtime DLLs needed by the compiler
 426             if libdir != "bin" {
 427                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 428                     let name = entry.file_name();
 429                     if let Some(s) = name.to_str() {
 430                         if is_dylib(s) {
 431                             install(&entry.path(), &image.join(libdir), 0o644);
 432                         }
 433                     }
 434                 }
 435             }
 436
 437             // Man pages
 438             t!(fs::create_dir_all(image.join("share/man/man1")));
 439             let man_src = build.src.join("src/doc/man");
 440             let man_dst = image.join("share/man/man1");
 441             let date_output = output(Command::new("date").arg("+%B %Y"));
 442             let month_year = date_output.trim();
 443             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 444             // to hardlink, and we don't want to edit the source templates
 445             for entry_result in t!(fs::read_dir(man_src)) {
 446                 let file_entry = t!(entry_result);
 447                 let page_src = file_entry.path();
 448                 let page_dst = man_dst.join(file_entry.file_name());
 449                 t!(fs::copy(&page_src, &page_dst));
 450                 // template in month/year and version number
 451                 replace_in_file(&page_dst,
 452                                 &[("<INSERT DATE HERE>", month_year),
 453                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 454             }
 455
 456             // Debugger scripts
 457             builder.ensure(DebuggerScripts {
 458                 sysroot: INTERNER.intern_path(image.to_owned()),
 459                 host,
 460             });
 461
 462             // Misc license info
 463             let cp = |file: &str| {
 464                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 465             };
 466             cp("COPYRIGHT");
 467             cp("LICENSE-APACHE");
 468             cp("LICENSE-MIT");
 469             cp("README.md");
 470         }
 471     }
 472 }
 473
 474 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 475 pub struct DebuggerScripts {
 476     pub sysroot: Interned<PathBuf>,
 477     pub host: Interned<String>,
 478 }
 479
 480 impl Step for DebuggerScripts {
 481     type Output = ();
 482
 483     fn should_run(run: ShouldRun) -> ShouldRun {
 484         run.path("src/lldb_batchmode.py")
 485     }
 486
 487     fn make_run(run: RunConfig) {
 488         run.builder.ensure(DebuggerScripts {
 489             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 490             host: run.target,
 491         });
 492     }
 493
 494     /// Copies debugger scripts for `target` into the `sysroot` specified.
 495     fn run(self, builder: &Builder) {
 496         let build = builder.build;
 497         let host = self.host;
 498         let sysroot = self.sysroot;
 499         let dst = sysroot.join("lib/rustlib/etc");
 500         t!(fs::create_dir_all(&dst));
 501         let cp_debugger_script = |file: &str| {
 502             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 503         };
 504         if host.contains("windows-msvc") {
 505             // windbg debugger scripts
 506             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 507                 0o755);
 508
 509             cp_debugger_script("natvis/intrinsic.natvis");
 510             cp_debugger_script("natvis/liballoc.natvis");
 511             cp_debugger_script("natvis/libcore.natvis");
 512         } else {
 513             cp_debugger_script("debugger_pretty_printers_common.py");
 514
 515             // gdb debugger scripts
 516             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 517                     0o755);
 518
 519             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 520             cp_debugger_script("gdb_rust_pretty_printing.py");
 521
 522             // lldb debugger scripts
 523             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 524                     0o755);
 525
 526             cp_debugger_script("lldb_rust_formatters.py");
 527         }
 528     }
 529 }
 530
 531 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 532 pub struct Std {
 533     pub compiler: Compiler,
 534     pub target: Interned<String>,
 535 }
 536
 537 impl Step for Std {
 538     type Output = PathBuf;
 539     const DEFAULT: bool = true;
 540     const ONLY_BUILD_TARGETS: bool = true;
 541
 542     fn should_run(run: ShouldRun) -> ShouldRun {
 543         run.path("src/libstd")
 544     }
 545
 546     fn make_run(run: RunConfig) {
 547         run.builder.ensure(Std {
 548             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 549             target: run.target,
 550         });
 551     }
 552
 553     fn run(self, builder: &Builder) -> PathBuf {
 554         let build = builder.build;
 555         let compiler = self.compiler;
 556         let target = self.target;
 557
 558         let name = pkgname(build, "rust-std");
 559         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 560
 561         // The only true set of target libraries came from the build triple, so
 562         // let's reduce redundant work by only producing archives from that host.
 563         if compiler.host != build.build {
 564             println!("\tskipping, not a build host");
 565             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 566         }
 567
 568         // We want to package up as many target libraries as possible
 569         // for the `rust-std` package, so if this is a host target we
 570         // depend on librustc and otherwise we just depend on libtest.
 571         if build.hosts.iter().any(|t| t == target) {
 572             builder.ensure(compile::Rustc { compiler, target });
 573         } else {
 574             builder.ensure(compile::Test { compiler, target });
 575         }
 576
 577         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 578         let _ = fs::remove_dir_all(&image);
 579
 580         let dst = image.join("lib/rustlib").join(target);
 581         t!(fs::create_dir_all(&dst));
 582         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 583         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 584         cp_r(&src, &dst);
 585
 586         let mut cmd = rust_installer(builder);
 587         cmd.arg("generate")
 588            .arg("--product-name=Rust")
 589            .arg("--rel-manifest-dir=rustlib")
 590            .arg("--success-message=std-is-standing-at-the-ready.")
 591            .arg("--image-dir").arg(&image)
 592            .arg("--work-dir").arg(&tmpdir(build))
 593            .arg("--output-dir").arg(&distdir(build))
 594            .arg(format!("--package-name={}-{}", name, target))
 595            .arg(format!("--component-name=rust-std-{}", target))
 596            .arg("--legacy-manifest-dirs=rustlib,cargo");
 597         build.run(&mut cmd);
 598         t!(fs::remove_dir_all(&image));
 599         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 600     }
 601 }
 602
 603 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 604 pub struct Analysis {
 605     pub compiler: Compiler,
 606     pub target: Interned<String>,
 607 }
 608
 609 impl Step for Analysis {
 610     type Output = PathBuf;
 611     const DEFAULT: bool = true;
 612     const ONLY_BUILD_TARGETS: bool = true;
 613
 614     fn should_run(run: ShouldRun) -> ShouldRun {
 615         let builder = run.builder;
 616         run.path("analysis").default_condition(builder.build.config.extended)
 617     }
 618
 619     fn make_run(run: RunConfig) {
 620         run.builder.ensure(Analysis {
 621             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 622             target: run.target,
 623         });
 624     }
 625
 626     /// Creates a tarball of save-analysis metadata, if available.
 627     fn run(self, builder: &Builder) -> PathBuf {
 628         let build = builder.build;
 629         let compiler = self.compiler;
 630         let target = self.target;
 631         assert!(build.config.extended);
 632         println!("Dist analysis");
 633         let name = pkgname(build, "rust-analysis");
 634
 635         if &compiler.host != build.build {
 636             println!("\tskipping, not a build host");
 637             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 638         }
 639
 640         builder.ensure(Std { compiler, target });
 641
 642         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 643         // stage2 artifacts is simply copied from stage1 in that case.
 644         let compiler = if build.force_use_stage1(compiler, target) {
 645             builder.compiler(1, compiler.host)
 646         } else {
 647             compiler.clone()
 648         };
 649
 650         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 651
 652         let src = build.stage_out(compiler, Mode::Libstd)
 653             .join(target).join(build.cargo_dir()).join("deps");
 654
 655         let image_src = src.join("save-analysis");
 656         let dst = image.join("lib/rustlib").join(target).join("analysis");
 657         t!(fs::create_dir_all(&dst));
 658         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 659         cp_r(&image_src, &dst);
 660
 661         let mut cmd = rust_installer(builder);
 662         cmd.arg("generate")
 663            .arg("--product-name=Rust")
 664            .arg("--rel-manifest-dir=rustlib")
 665            .arg("--success-message=save-analysis-saved.")
 666            .arg("--image-dir").arg(&image)
 667            .arg("--work-dir").arg(&tmpdir(build))
 668            .arg("--output-dir").arg(&distdir(build))
 669            .arg(format!("--package-name={}-{}", name, target))
 670            .arg(format!("--component-name=rust-analysis-{}", target))
 671            .arg("--legacy-manifest-dirs=rustlib,cargo");
 672         build.run(&mut cmd);
 673         t!(fs::remove_dir_all(&image));
 674         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 675     }
 676 }
 677
 678 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 679     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 680         let spath = match path.to_str() {
 681             Some(path) => path,
 682             None => return false,
 683         };
 684         if spath.ends_with("~") || spath.ends_with(".pyc") {
 685             return false
 686         }
 687         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 688             (spath.ends_with(".ll") ||
 689              spath.ends_with(".td") ||
 690              spath.ends_with(".s")) {
 691             return false
 692         }
 693         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 694             return false
 695         }
 696
 697         let full_path = Path::new(dir).join(path);
 698         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 699             return false;
 700         }
 701
 702         let excludes = [
 703             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 704             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 705             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 706             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 707         ];
 708         !path.iter()
 709              .map(|s| s.to_str().unwrap())
 710              .any(|s| excludes.contains(&s))
 711     }
 712
 713     // Copy the directories using our filter
 714     for item in src_dirs {
 715         let dst = &dst_dir.join(item);
 716         t!(fs::create_dir_all(dst));
 717         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 718     }
 719 }
 720
 721 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 722 pub struct Src;
 723
 724 impl Step for Src {
 725     /// The output path of the src installer tarball
 726     type Output = PathBuf;
 727     const DEFAULT: bool = true;
 728     const ONLY_HOSTS: bool = true;
 729     const ONLY_BUILD_TARGETS: bool = true;
 730     const ONLY_BUILD: bool = true;
 731
 732     fn should_run(run: ShouldRun) -> ShouldRun {
 733         run.path("src")
 734     }
 735
 736     fn make_run(run: RunConfig) {
 737         run.builder.ensure(Src);
 738     }
 739
 740     /// Creates the `rust-src` installer component
 741     fn run(self, builder: &Builder) -> PathBuf {
 742         let build = builder.build;
 743         println!("Dist src");
 744
 745         let name = pkgname(build, "rust-src");
 746         let image = tmpdir(build).join(format!("{}-image", name));
 747         let _ = fs::remove_dir_all(&image);
 748
 749         let dst = image.join("lib/rustlib/src");
 750         let dst_src = dst.join("rust");
 751         t!(fs::create_dir_all(&dst_src));
 752
 753         let src_files = [
 754             "src/Cargo.lock",
 755         ];
 756         // This is the reduced set of paths which will become the rust-src component
 757         // (essentially libstd and all of its path dependencies)
 758         let std_src_dirs = [
 759             "src/build_helper",
 760             "src/dlmalloc",
 761             "src/liballoc",
 762             "src/liballoc_jemalloc",
 763             "src/liballoc_system",
 764             "src/libbacktrace",
 765             "src/libcompiler_builtins",
 766             "src/libcore",
 767             "src/liblibc",
 768             "src/libpanic_abort",
 769             "src/libpanic_unwind",
 770             "src/librustc_asan",
 771             "src/librustc_lsan",
 772             "src/librustc_msan",
 773             "src/librustc_tsan",
 774             "src/libstd",
 775             "src/libstd_unicode",
 776             "src/libunwind",
 777             "src/rustc/compiler_builtins_shim",
 778             "src/rustc/libc_shim",
 779             "src/rustc/dlmalloc_shim",
 780             "src/libtest",
 781             "src/libterm",
 782             "src/jemalloc",
 783             "src/libprofiler_builtins",
 784         ];
 785         let std_src_dirs_exclude = [
 786             "src/libcompiler_builtins/compiler-rt/test",
 787             "src/jemalloc/test/unit",
 788         ];
 789
 790         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 791         for file in src_files.iter() {
 792             copy(&build.src.join(file), &dst_src.join(file));
 793         }
 794
 795         // Create source tarball in rust-installer format
 796         let mut cmd = rust_installer(builder);
 797         cmd.arg("generate")
 798            .arg("--product-name=Rust")
 799            .arg("--rel-manifest-dir=rustlib")
 800            .arg("--success-message=Awesome-Source.")
 801            .arg("--image-dir").arg(&image)
 802            .arg("--work-dir").arg(&tmpdir(build))
 803            .arg("--output-dir").arg(&distdir(build))
 804            .arg(format!("--package-name={}", name))
 805            .arg("--component-name=rust-src")
 806            .arg("--legacy-manifest-dirs=rustlib,cargo");
 807         build.run(&mut cmd);
 808
 809         t!(fs::remove_dir_all(&image));
 810         distdir(build).join(&format!("{}.tar.gz", name))
 811     }
 812 }
 813
 814 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 815
 816 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 817 pub struct PlainSourceTarball;
 818
 819 impl Step for PlainSourceTarball {
 820     /// Produces the location of the tarball generated
 821     type Output = PathBuf;
 822     const DEFAULT: bool = true;
 823     const ONLY_HOSTS: bool = true;
 824     const ONLY_BUILD_TARGETS: bool = true;
 825     const ONLY_BUILD: bool = true;
 826
 827     fn should_run(run: ShouldRun) -> ShouldRun {
 828         let builder = run.builder;
 829         run.path("src").default_condition(builder.config.rust_dist_src)
 830     }
 831
 832     fn make_run(run: RunConfig) {
 833         run.builder.ensure(PlainSourceTarball);
 834     }
 835
 836     /// Creates the plain source tarball
 837     fn run(self, builder: &Builder) -> PathBuf {
 838         let build = builder.build;
 839         println!("Create plain source tarball");
 840
 841         // Make sure that the root folder of tarball has the correct name
 842         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 843         let plain_dst_src = tmpdir(build).join(&plain_name);
 844         let _ = fs::remove_dir_all(&plain_dst_src);
 845         t!(fs::create_dir_all(&plain_dst_src));
 846
 847         // This is the set of root paths which will become part of the source package
 848         let src_files = [
 849             "COPYRIGHT",
 850             "LICENSE-APACHE",
 851             "LICENSE-MIT",
 852             "CONTRIBUTING.md",
 853             "README.md",
 854             "RELEASES.md",
 855             "configure",
 856             "x.py",
 857             "config.toml.example",
 858         ];
 859         let src_dirs = [
 860             "src",
 861         ];
 862
 863         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 864
 865         // Copy the files normally
 866         for item in &src_files {
 867             copy(&build.src.join(item), &plain_dst_src.join(item));
 868         }
 869
 870         // Create the version file
 871         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 872         if let Some(sha) = build.rust_sha() {
 873             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 874         }
 875
 876         // If we're building from git sources, we need to vendor a complete distribution.
 877         if build.rust_info.is_git() {
 878             // Get cargo-vendor installed, if it isn't already.
 879             let mut has_cargo_vendor = false;
 880             let mut cmd = Command::new(&build.initial_cargo);
 881             for line in output(cmd.arg("install").arg("--list")).lines() {
 882                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 883             }
 884             if !has_cargo_vendor {
 885                 let mut cmd = Command::new(&build.initial_cargo);
 886                 cmd.arg("install")
 887                    .arg("--force")
 888                    .arg("--debug")
 889                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 890                    .arg("cargo-vendor")
 891                    .env("RUSTC", &build.initial_rustc);
 892                 build.run(&mut cmd);
 893             }
 894
 895             // Vendor all Cargo dependencies
 896             let mut cmd = Command::new(&build.initial_cargo);
 897             cmd.arg("vendor")
 898                .current_dir(&plain_dst_src.join("src"));
 899             build.run(&mut cmd);
 900         }
 901
 902         // Create plain source tarball
 903         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 904         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 905         tarball.set_extension(""); // strip .gz
 906         tarball.set_extension(""); // strip .tar
 907         if let Some(dir) = tarball.parent() {
 908             t!(fs::create_dir_all(dir));
 909         }
 910         println!("running installer");
 911         let mut cmd = rust_installer(builder);
 912         cmd.arg("tarball")
 913            .arg("--input").arg(&plain_name)
 914            .arg("--output").arg(&tarball)
 915            .arg("--work-dir=.")
 916            .current_dir(tmpdir(build));
 917         build.run(&mut cmd);
 918         distdir(build).join(&format!("{}.tar.gz", plain_name))
 919     }
 920 }
 921
 922 fn install(src: &Path, dstdir: &Path, perms: u32) {
 923     let dst = dstdir.join(src.file_name().unwrap());
 924     t!(fs::create_dir_all(dstdir));
 925     drop(fs::remove_file(&dst));
 926     {
 927         let mut s = t!(fs::File::open(&src));
 928         let mut d = t!(fs::File::create(&dst));
 929         io::copy(&mut s, &mut d).expect("failed to copy");
 930     }
 931     chmod(&dst, perms);
 932 }
 933
 934 #[cfg(unix)]
 935 fn chmod(path: &Path, perms: u32) {
 936     use std::os::unix::fs::*;
 937     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 938 }
 939 #[cfg(windows)]
 940 fn chmod(_path: &Path, _perms: u32) {}
 941
 942 // We have to run a few shell scripts, which choke quite a bit on both `\`
 943 // characters and on `C:\` paths, so normalize both of them away.
 944 pub fn sanitize_sh(path: &Path) -> String {
 945     let path = path.to_str().unwrap().replace("\\", "/");
 946     return change_drive(&path).unwrap_or(path);
 947
 948     fn change_drive(s: &str) -> Option<String> {
 949         let mut ch = s.chars();
 950         let drive = ch.next().unwrap_or('C');
 951         if ch.next() != Some(':') {
 952             return None
 953         }
 954         if ch.next() != Some('/') {
 955             return None
 956         }
 957         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 958     }
 959 }
 960
 961 fn write_file(path: &Path, data: &[u8]) {
 962     let mut vf = t!(fs::File::create(path));
 963     t!(vf.write_all(data));
 964 }
 965
 966 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 967 pub struct Cargo {
 968     pub stage: u32,
 969     pub target: Interned<String>,
 970 }
 971
 972 impl Step for Cargo {
 973     type Output = PathBuf;
 974     const ONLY_BUILD_TARGETS: bool = true;
 975     const ONLY_HOSTS: bool = true;
 976
 977     fn should_run(run: ShouldRun) -> ShouldRun {
 978         run.path("cargo")
 979     }
 980
 981     fn make_run(run: RunConfig) {
 982         run.builder.ensure(Cargo {
 983             stage: run.builder.top_stage,
 984             target: run.target,
 985         });
 986     }
 987
 988     fn run(self, builder: &Builder) -> PathBuf {
 989         let build = builder.build;
 990         let stage = self.stage;
 991         let target = self.target;
 992
 993         println!("Dist cargo stage{} ({})", stage, target);
 994         let src = build.src.join("src/tools/cargo");
 995         let etc = src.join("src/etc");
 996         let release_num = build.release_num("cargo");
 997         let name = pkgname(build, "cargo");
 998         let version = builder.cargo_info.version(build, &release_num);
 999
1000         let tmp = tmpdir(build);
1001         let image = tmp.join("cargo-image");
1002         drop(fs::remove_dir_all(&image));
1003         t!(fs::create_dir_all(&image));
1004
1005         // Prepare the image directory
1006         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1007         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1008         let cargo = builder.ensure(tool::Cargo {
1009             compiler: builder.compiler(stage, build.build),
1010             target
1011         });
1012         install(&cargo, &image.join("bin"), 0o755);
1013         for man in t!(etc.join("man").read_dir()) {
1014             let man = t!(man);
1015             install(&man.path(), &image.join("share/man/man1"), 0o644);
1016         }
1017         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1018         copy(&etc.join("cargo.bashcomp.sh"),
1019              &image.join("etc/bash_completion.d/cargo"));
1020         let doc = image.join("share/doc/cargo");
1021         install(&src.join("README.md"), &doc, 0o644);
1022         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1023         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1024         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1025
1026         // Prepare the overlay
1027         let overlay = tmp.join("cargo-overlay");
1028         drop(fs::remove_dir_all(&overlay));
1029         t!(fs::create_dir_all(&overlay));
1030         install(&src.join("README.md"), &overlay, 0o644);
1031         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1032         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1033         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1034         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1035
1036         // Generate the installer tarball
1037         let mut cmd = rust_installer(builder);
1038         cmd.arg("generate")
1039            .arg("--product-name=Rust")
1040            .arg("--rel-manifest-dir=rustlib")
1041            .arg("--success-message=Rust-is-ready-to-roll.")
1042            .arg("--image-dir").arg(&image)
1043            .arg("--work-dir").arg(&tmpdir(build))
1044            .arg("--output-dir").arg(&distdir(build))
1045            .arg("--non-installed-overlay").arg(&overlay)
1046            .arg(format!("--package-name={}-{}", name, target))
1047            .arg("--component-name=cargo")
1048            .arg("--legacy-manifest-dirs=rustlib,cargo");
1049         build.run(&mut cmd);
1050         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1051     }
1052 }
1053
1054 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1055 pub struct Rls {
1056     pub stage: u32,
1057     pub target: Interned<String>,
1058 }
1059
1060 impl Step for Rls {
1061     type Output = Option<PathBuf>;
1062     const ONLY_BUILD_TARGETS: bool = true;
1063     const ONLY_HOSTS: bool = true;
1064
1065     fn should_run(run: ShouldRun) -> ShouldRun {
1066         run.path("rls")
1067     }
1068
1069     fn make_run(run: RunConfig) {
1070         run.builder.ensure(Rls {
1071             stage: run.builder.top_stage,
1072             target: run.target,
1073         });
1074     }
1075
1076     fn run(self, builder: &Builder) -> Option<PathBuf> {
1077         let build = builder.build;
1078         let stage = self.stage;
1079         let target = self.target;
1080         assert!(build.config.extended);
1081
1082         println!("Dist RLS stage{} ({})", stage, target);
1083         let src = build.src.join("src/tools/rls");
1084         let release_num = build.release_num("rls");
1085         let name = pkgname(build, "rls");
1086         let version = build.rls_info.version(build, &release_num);
1087
1088         let tmp = tmpdir(build);
1089         let image = tmp.join("rls-image");
1090         drop(fs::remove_dir_all(&image));
1091         t!(fs::create_dir_all(&image));
1092
1093         // Prepare the image directory
1094         // We expect RLS to build, because we've exited this step above if tool
1095         // state for RLS isn't testing.
1096         let rls = builder.ensure(tool::Rls {
1097             compiler: builder.compiler(stage, build.build),
1098             target
1099         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1100
1101         install(&rls, &image.join("bin"), 0o755);
1102         let doc = image.join("share/doc/rls");
1103         install(&src.join("README.md"), &doc, 0o644);
1104         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1105         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1106
1107         // Prepare the overlay
1108         let overlay = tmp.join("rls-overlay");
1109         drop(fs::remove_dir_all(&overlay));
1110         t!(fs::create_dir_all(&overlay));
1111         install(&src.join("README.md"), &overlay, 0o644);
1112         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1113         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1114         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1115
1116         // Generate the installer tarball
1117         let mut cmd = rust_installer(builder);
1118         cmd.arg("generate")
1119            .arg("--product-name=Rust")
1120            .arg("--rel-manifest-dir=rustlib")
1121            .arg("--success-message=RLS-ready-to-serve.")
1122            .arg("--image-dir").arg(&image)
1123            .arg("--work-dir").arg(&tmpdir(build))
1124            .arg("--output-dir").arg(&distdir(build))
1125            .arg("--non-installed-overlay").arg(&overlay)
1126            .arg(format!("--package-name={}-{}", name, target))
1127            .arg("--legacy-manifest-dirs=rustlib,cargo")
1128            .arg("--component-name=rls-preview");
1129
1130         build.run(&mut cmd);
1131         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1132     }
1133 }
1134
1135
1136 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1137 pub struct Rustfmt {
1138     pub stage: u32,
1139     pub target: Interned<String>,
1140 }
1141
1142 impl Step for Rustfmt {
1143     type Output = Option<PathBuf>;
1144     const ONLY_BUILD_TARGETS: bool = true;
1145     const ONLY_HOSTS: bool = true;
1146
1147     fn should_run(run: ShouldRun) -> ShouldRun {
1148         run.path("rustfmt")
1149     }
1150
1151     fn make_run(run: RunConfig) {
1152         run.builder.ensure(Rustfmt {
1153             stage: run.builder.top_stage,
1154             target: run.target,
1155         });
1156     }
1157
1158     fn run(self, builder: &Builder) -> Option<PathBuf> {
1159         let build = builder.build;
1160         let stage = self.stage;
1161         let target = self.target;
1162         assert!(build.config.extended);
1163
1164         println!("Dist Rustfmt stage{} ({})", stage, target);
1165         let src = build.src.join("src/tools/rustfmt");
1166         let release_num = build.release_num("rustfmt");
1167         let name = pkgname(build, "rustfmt");
1168         let version = build.rustfmt_info.version(build, &release_num);
1169
1170         let tmp = tmpdir(build);
1171         let image = tmp.join("rustfmt-image");
1172         drop(fs::remove_dir_all(&image));
1173         t!(fs::create_dir_all(&image));
1174
1175         // Prepare the image directory
1176         let rustfmt = builder.ensure(tool::Rustfmt {
1177             compiler: builder.compiler(stage, build.build),
1178             target
1179         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1180         let cargofmt = builder.ensure(tool::Cargofmt {
1181             compiler: builder.compiler(stage, build.build),
1182             target
1183         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1184
1185         install(&rustfmt, &image.join("bin"), 0o755);
1186         install(&cargofmt, &image.join("bin"), 0o755);
1187         let doc = image.join("share/doc/rustfmt");
1188         install(&src.join("README.md"), &doc, 0o644);
1189         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1190         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1191
1192         // Prepare the overlay
1193         let overlay = tmp.join("rustfmt-overlay");
1194         drop(fs::remove_dir_all(&overlay));
1195         t!(fs::create_dir_all(&overlay));
1196         install(&src.join("README.md"), &overlay, 0o644);
1197         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1198         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1199         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1200
1201         // Generate the installer tarball
1202         let mut cmd = rust_installer(builder);
1203         cmd.arg("generate")
1204            .arg("--product-name=Rust")
1205            .arg("--rel-manifest-dir=rustlib")
1206            .arg("--success-message=rustfmt-ready-to-fmt.")
1207            .arg("--image-dir").arg(&image)
1208            .arg("--work-dir").arg(&tmpdir(build))
1209            .arg("--output-dir").arg(&distdir(build))
1210            .arg("--non-installed-overlay").arg(&overlay)
1211            .arg(format!("--package-name={}-{}", name, target))
1212            .arg("--legacy-manifest-dirs=rustlib,cargo")
1213            .arg("--component-name=rustfmt-preview");
1214
1215         build.run(&mut cmd);
1216         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1217     }
1218 }
1219
1220
1221 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1222 pub struct DontDistWithMiriEnabled;
1223
1224 impl Step for DontDistWithMiriEnabled {
1225     type Output = PathBuf;
1226     const DEFAULT: bool = true;
1227
1228     fn should_run(run: ShouldRun) -> ShouldRun {
1229         let build_miri = run.builder.build.config.test_miri;
1230         run.default_condition(build_miri)
1231     }
1232
1233     fn make_run(run: RunConfig) {
1234         run.builder.ensure(DontDistWithMiriEnabled);
1235     }
1236
1237     fn run(self, _: &Builder) -> PathBuf {
1238         panic!("Do not distribute with miri enabled.\n\
1239                 The distributed libraries would include all MIR (increasing binary size).
1240                 The distributed MIR would include validation statements.");
1241     }
1242 }
1243
1244
1245 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1246 pub struct Extended {
1247     stage: u32,
1248     host: Interned<String>,
1249     target: Interned<String>,
1250 }
1251
1252 impl Step for Extended {
1253     type Output = ();
1254     const DEFAULT: bool = true;
1255     const ONLY_BUILD_TARGETS: bool = true;
1256     const ONLY_HOSTS: bool = true;
1257
1258     fn should_run(run: ShouldRun) -> ShouldRun {
1259         let builder = run.builder;
1260         run.path("extended").default_condition(builder.config.extended)
1261     }
1262
1263     fn make_run(run: RunConfig) {
1264         run.builder.ensure(Extended {
1265             stage: run.builder.top_stage,
1266             host: run.host,
1267             target: run.target,
1268         });
1269     }
1270
1271     /// Creates a combined installer for the specified target in the provided stage.
1272     fn run(self, builder: &Builder) {
1273         let build = builder.build;
1274         let stage = self.stage;
1275         let target = self.target;
1276
1277         println!("Dist extended stage{} ({})", stage, target);
1278
1279         let rustc_installer = builder.ensure(Rustc {
1280             compiler: builder.compiler(stage, target),
1281         });
1282         let cargo_installer = builder.ensure(Cargo { stage, target });
1283         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1284         let rls_installer = builder.ensure(Rls { stage, target });
1285         let mingw_installer = builder.ensure(Mingw { host: target });
1286         let analysis_installer = builder.ensure(Analysis {
1287             compiler: builder.compiler(stage, self.host),
1288             target
1289         });
1290
1291         let docs_installer = builder.ensure(Docs { stage, host: target, });
1292         let std_installer = builder.ensure(Std {
1293             compiler: builder.compiler(stage, self.host),
1294             target,
1295         });
1296
1297         let tmp = tmpdir(build);
1298         let overlay = tmp.join("extended-overlay");
1299         let etc = build.src.join("src/etc/installer");
1300         let work = tmp.join("work");
1301
1302         let _ = fs::remove_dir_all(&overlay);
1303         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1304         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1305         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1306         let version = build.rust_version();
1307         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1308         if let Some(sha) = build.rust_sha() {
1309             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1310         }
1311         install(&etc.join("README.md"), &overlay, 0o644);
1312
1313         // When rust-std package split from rustc, we needed to ensure that during
1314         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1315         // the std files during uninstall. To do this ensure that rustc comes
1316         // before rust-std in the list below.
1317         let mut tarballs = Vec::new();
1318         tarballs.push(rustc_installer);
1319         tarballs.push(cargo_installer);
1320         tarballs.extend(rls_installer.clone());
1321         tarballs.extend(rustfmt_installer.clone());
1322         tarballs.push(analysis_installer);
1323         tarballs.push(std_installer);
1324         if build.config.docs {
1325             tarballs.push(docs_installer);
1326         }
1327         if target.contains("pc-windows-gnu") {
1328             tarballs.push(mingw_installer.unwrap());
1329         }
1330         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1331         for tarball in &tarballs[1..] {
1332             input_tarballs.push(",");
1333             input_tarballs.push(tarball);
1334         }
1335
1336         let mut cmd = rust_installer(builder);
1337         cmd.arg("combine")
1338             .arg("--product-name=Rust")
1339             .arg("--rel-manifest-dir=rustlib")
1340             .arg("--success-message=Rust-is-ready-to-roll.")
1341             .arg("--work-dir").arg(&work)
1342             .arg("--output-dir").arg(&distdir(build))
1343             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1344             .arg("--legacy-manifest-dirs=rustlib,cargo")
1345             .arg("--input-tarballs").arg(input_tarballs)
1346             .arg("--non-installed-overlay").arg(&overlay);
1347         build.run(&mut cmd);
1348
1349         let mut license = String::new();
1350         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1351         license.push_str("\n");
1352         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1353         license.push_str("\n");
1354         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1355
1356         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1357         let mut rtf = rtf.to_string();
1358         rtf.push_str("\n");
1359         for line in license.lines() {
1360             rtf.push_str(line);
1361             rtf.push_str("\\line ");
1362         }
1363         rtf.push_str("}");
1364
1365         fn filter(contents: &str, marker: &str) -> String {
1366             let start = format!("tool-{}-start", marker);
1367             let end = format!("tool-{}-end", marker);
1368             let mut lines = Vec::new();
1369             let mut omitted = false;
1370             for line in contents.lines() {
1371                 if line.contains(&start) {
1372                     omitted = true;
1373                 } else if line.contains(&end) {
1374                     omitted = false;
1375                 } else if !omitted {
1376                     lines.push(line);
1377                 }
1378             }
1379
1380             lines.join("\n")
1381         }
1382
1383         let xform = |p: &Path| {
1384             let mut contents = String::new();
1385             t!(t!(File::open(p)).read_to_string(&mut contents));
1386             if rls_installer.is_none() {
1387                 contents = filter(&contents, "rls");
1388             }
1389             if rustfmt_installer.is_none() {
1390                 contents = filter(&contents, "rustfmt");
1391             }
1392             let ret = tmp.join(p.file_name().unwrap());
1393             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1394             return ret
1395         };
1396
1397         if target.contains("apple-darwin") {
1398             let pkg = tmp.join("pkg");
1399             let _ = fs::remove_dir_all(&pkg);
1400
1401             let pkgbuild = |component: &str| {
1402                 let mut cmd = Command::new("pkgbuild");
1403                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1404                     .arg("--scripts").arg(pkg.join(component))
1405                     .arg("--nopayload")
1406                     .arg(pkg.join(component).with_extension("pkg"));
1407                 build.run(&mut cmd);
1408             };
1409
1410             let prepare = |name: &str| {
1411                 t!(fs::create_dir_all(pkg.join(name)));
1412                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1413                         &pkg.join(name));
1414                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1415                 pkgbuild(name);
1416             };
1417             prepare("rustc");
1418             prepare("cargo");
1419             prepare("rust-docs");
1420             prepare("rust-std");
1421             prepare("rust-analysis");
1422
1423             if rls_installer.is_some() {
1424                 prepare("rls");
1425             }
1426
1427             // create an 'uninstall' package
1428             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1429             pkgbuild("uninstall");
1430
1431             t!(fs::create_dir_all(pkg.join("res")));
1432             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1433             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1434             let mut cmd = Command::new("productbuild");
1435             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1436                 .arg("--resources").arg(pkg.join("res"))
1437                 .arg(distdir(build).join(format!("{}-{}.pkg",
1438                                                     pkgname(build, "rust"),
1439                                                     target)))
1440                 .arg("--package-path").arg(&pkg);
1441             build.run(&mut cmd);
1442         }
1443
1444         if target.contains("windows") {
1445             let exe = tmp.join("exe");
1446             let _ = fs::remove_dir_all(&exe);
1447
1448             let prepare = |name: &str| {
1449                 t!(fs::create_dir_all(exe.join(name)));
1450                 let dir = if name == "rust-std" || name == "rust-analysis" {
1451                     format!("{}-{}", name, target)
1452                 } else if name == "rls" {
1453                     "rls-preview".to_string()
1454                 } else {
1455                     name.to_string()
1456                 };
1457                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1458                             .join(dir),
1459                         &exe.join(name));
1460                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1461             };
1462             prepare("rustc");
1463             prepare("cargo");
1464             prepare("rust-analysis");
1465             prepare("rust-docs");
1466             prepare("rust-std");
1467             if rls_installer.is_some() {
1468                 prepare("rls");
1469             }
1470             if target.contains("windows-gnu") {
1471                 prepare("rust-mingw");
1472             }
1473
1474             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1475             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1476             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1477             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1478             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1479
1480             // Generate exe installer
1481             let mut cmd = Command::new("iscc");
1482             cmd.arg("rust.iss")
1483                 .current_dir(&exe);
1484             if target.contains("windows-gnu") {
1485                 cmd.arg("/dMINGW");
1486             }
1487             add_env(build, &mut cmd, target);
1488             build.run(&mut cmd);
1489             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1490                     &distdir(build),
1491                     0o755);
1492
1493             // Generate msi installer
1494             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1495             let heat = wix.join("bin/heat.exe");
1496             let candle = wix.join("bin/candle.exe");
1497             let light = wix.join("bin/light.exe");
1498
1499             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1500             build.run(Command::new(&heat)
1501                             .current_dir(&exe)
1502                             .arg("dir")
1503                             .arg("rustc")
1504                             .args(&heat_flags)
1505                             .arg("-cg").arg("RustcGroup")
1506                             .arg("-dr").arg("Rustc")
1507                             .arg("-var").arg("var.RustcDir")
1508                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1509             build.run(Command::new(&heat)
1510                             .current_dir(&exe)
1511                             .arg("dir")
1512                             .arg("rust-docs")
1513                             .args(&heat_flags)
1514                             .arg("-cg").arg("DocsGroup")
1515                             .arg("-dr").arg("Docs")
1516                             .arg("-var").arg("var.DocsDir")
1517                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1518                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1519             build.run(Command::new(&heat)
1520                             .current_dir(&exe)
1521                             .arg("dir")
1522                             .arg("cargo")
1523                             .args(&heat_flags)
1524                             .arg("-cg").arg("CargoGroup")
1525                             .arg("-dr").arg("Cargo")
1526                             .arg("-var").arg("var.CargoDir")
1527                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1528                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1529             build.run(Command::new(&heat)
1530                             .current_dir(&exe)
1531                             .arg("dir")
1532                             .arg("rust-std")
1533                             .args(&heat_flags)
1534                             .arg("-cg").arg("StdGroup")
1535                             .arg("-dr").arg("Std")
1536                             .arg("-var").arg("var.StdDir")
1537                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1538             if rls_installer.is_some() {
1539                 build.run(Command::new(&heat)
1540                                 .current_dir(&exe)
1541                                 .arg("dir")
1542                                 .arg("rls")
1543                                 .args(&heat_flags)
1544                                 .arg("-cg").arg("RlsGroup")
1545                                 .arg("-dr").arg("Rls")
1546                                 .arg("-var").arg("var.RlsDir")
1547                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1548                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1549             }
1550             build.run(Command::new(&heat)
1551                             .current_dir(&exe)
1552                             .arg("dir")
1553                             .arg("rust-analysis")
1554                             .args(&heat_flags)
1555                             .arg("-cg").arg("AnalysisGroup")
1556                             .arg("-dr").arg("Analysis")
1557                             .arg("-var").arg("var.AnalysisDir")
1558                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1559                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1560             if target.contains("windows-gnu") {
1561                 build.run(Command::new(&heat)
1562                                 .current_dir(&exe)
1563                                 .arg("dir")
1564                                 .arg("rust-mingw")
1565                                 .args(&heat_flags)
1566                                 .arg("-cg").arg("GccGroup")
1567                                 .arg("-dr").arg("Gcc")
1568                                 .arg("-var").arg("var.GccDir")
1569                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1570             }
1571
1572             let candle = |input: &Path| {
1573                 let output = exe.join(input.file_stem().unwrap())
1574                                 .with_extension("wixobj");
1575                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1576                 let mut cmd = Command::new(&candle);
1577                 cmd.current_dir(&exe)
1578                     .arg("-nologo")
1579                     .arg("-dRustcDir=rustc")
1580                     .arg("-dDocsDir=rust-docs")
1581                     .arg("-dCargoDir=cargo")
1582                     .arg("-dStdDir=rust-std")
1583                     .arg("-dAnalysisDir=rust-analysis")
1584                     .arg("-arch").arg(&arch)
1585                     .arg("-out").arg(&output)
1586                     .arg(&input);
1587                 add_env(build, &mut cmd, target);
1588
1589                 if rls_installer.is_some() {
1590                     cmd.arg("-dRlsDir=rls");
1591                 }
1592                 if target.contains("windows-gnu") {
1593                     cmd.arg("-dGccDir=rust-mingw");
1594                 }
1595                 build.run(&mut cmd);
1596             };
1597             candle(&xform(&etc.join("msi/rust.wxs")));
1598             candle(&etc.join("msi/ui.wxs"));
1599             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1600             candle("RustcGroup.wxs".as_ref());
1601             candle("DocsGroup.wxs".as_ref());
1602             candle("CargoGroup.wxs".as_ref());
1603             candle("StdGroup.wxs".as_ref());
1604             if rls_installer.is_some() {
1605                 candle("RlsGroup.wxs".as_ref());
1606             }
1607             candle("AnalysisGroup.wxs".as_ref());
1608
1609             if target.contains("windows-gnu") {
1610                 candle("GccGroup.wxs".as_ref());
1611             }
1612
1613             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1614             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1615             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1616
1617             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1618             let mut cmd = Command::new(&light);
1619             cmd.arg("-nologo")
1620                 .arg("-ext").arg("WixUIExtension")
1621                 .arg("-ext").arg("WixUtilExtension")
1622                 .arg("-out").arg(exe.join(&filename))
1623                 .arg("rust.wixobj")
1624                 .arg("ui.wixobj")
1625                 .arg("rustwelcomedlg.wixobj")
1626                 .arg("RustcGroup.wixobj")
1627                 .arg("DocsGroup.wixobj")
1628                 .arg("CargoGroup.wixobj")
1629                 .arg("StdGroup.wixobj")
1630                 .arg("AnalysisGroup.wixobj")
1631                 .current_dir(&exe);
1632
1633             if rls_installer.is_some() {
1634                 cmd.arg("RlsGroup.wixobj");
1635             }
1636
1637             if target.contains("windows-gnu") {
1638                 cmd.arg("GccGroup.wixobj");
1639             }
1640             // ICE57 wrongly complains about the shortcuts
1641             cmd.arg("-sice:ICE57");
1642
1643             build.run(&mut cmd);
1644
1645             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1646         }
1647     }
1648 }
1649
1650 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1651     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1652     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1653        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1654        .env("CFG_RELEASE", build.rust_release())
1655        .env("CFG_PRERELEASE_VERSION", channel::CFG_PRERELEASE_VERSION)
1656        .env("CFG_VER_MAJOR", parts.next().unwrap())
1657        .env("CFG_VER_MINOR", parts.next().unwrap())
1658        .env("CFG_VER_PATCH", parts.next().unwrap())
1659        .env("CFG_VER_BUILD", "0") // just needed to build
1660        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1661        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1662        .env("CFG_BUILD", target)
1663        .env("CFG_CHANNEL", &build.config.channel);
1664
1665     if target.contains("windows-gnu") {
1666        cmd.env("CFG_MINGW", "1")
1667           .env("CFG_ABI", "GNU");
1668     } else {
1669        cmd.env("CFG_MINGW", "0")
1670           .env("CFG_ABI", "MSVC");
1671     }
1672
1673     if target.contains("x86_64") {
1674        cmd.env("CFG_PLATFORM", "x64");
1675     } else {
1676        cmd.env("CFG_PLATFORM", "x86");
1677     }
1678 }
1679
1680 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1681 pub struct HashSign;
1682
1683 impl Step for HashSign {
1684     type Output = ();
1685     const ONLY_BUILD_TARGETS: bool = true;
1686     const ONLY_HOSTS: bool = true;
1687     const ONLY_BUILD: bool = true;
1688
1689     fn should_run(run: ShouldRun) -> ShouldRun {
1690         run.path("hash-and-sign")
1691     }
1692
1693     fn make_run(run: RunConfig) {
1694         run.builder.ensure(HashSign);
1695     }
1696
1697     fn run(self, builder: &Builder) {
1698         let build = builder.build;
1699         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1700         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1701             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1702         });
1703         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1704             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1705         });
1706         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1707             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1708         });
1709         let mut pass = String::new();
1710         t!(t!(File::open(&file)).read_to_string(&mut pass));
1711
1712         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1713
1714         cmd.arg(sign);
1715         cmd.arg(distdir(build));
1716         cmd.arg(today.trim());
1717         cmd.arg(build.rust_package_vers());
1718         cmd.arg(build.package_vers(&build.release_num("cargo")));
1719         cmd.arg(build.package_vers(&build.release_num("rls")));
1720         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1721         cmd.arg(addr);
1722
1723         t!(fs::create_dir_all(distdir(build)));
1724
1725         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1726         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1727         let status = t!(child.wait());
1728         assert!(status.success());
1729     }
1730 }