src/bootstrap/dist.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Implementation of the various distribution aspects of the compiler.
  12 //!
  13 //! This module is responsible for creating tarballs of the standard library,
  14 //! compiler, and documentation. This ends up being what we distribute to
  15 //! everyone as well.
  16 //!
  17 //! No tarball is actually created literally in this file, but rather we shell
  18 //! out to `rust-installer` still. This may one day be replaced with bits and
  19 //! pieces of `rustup.rs`!
  20
  21 use std::env;
  22 use std::fs::{self, File};
  23 use std::io::{self, Read, Write};
  24 use std::path::{PathBuf, Path};
  25 use std::process::{Command, Stdio};
  26
  27 use build_helper::output;
  28
  29 use {Build, Compiler, Mode};
  30 use channel;
  31 use util::{cp_r, libdir, is_dylib, cp_filtered, copy, replace_in_file};
  32 use builder::{Builder, RunConfig, ShouldRun, Step};
  33 use compile;
  34 use tool::{self, Tool};
  35 use cache::{INTERNER, Interned};
  36 use time;
  37
  38 pub fn pkgname(build: &Build, component: &str) -> String {
  39     if component == "cargo" {
  40         format!("{}-{}", component, build.cargo_package_vers())
  41     } else if component == "rls" {
  42         format!("{}-{}", component, build.rls_package_vers())
  43     } else if component == "rustfmt" {
  44         format!("{}-{}", component, build.rustfmt_package_vers())
  45     } else {
  46         assert!(component.starts_with("rust"));
  47         format!("{}-{}", component, build.rust_package_vers())
  48     }
  49 }
  50
  51 fn distdir(build: &Build) -> PathBuf {
  52     build.out.join("dist")
  53 }
  54
  55 pub fn tmpdir(build: &Build) -> PathBuf {
  56     build.out.join("tmp/dist")
  57 }
  58
  59 fn rust_installer(builder: &Builder) -> Command {
  60     builder.tool_cmd(Tool::RustInstaller)
  61 }
  62
  63 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
  64 pub struct Docs {
  65     pub stage: u32,
  66     pub host: Interned<String>,
  67 }
  68
  69 impl Step for Docs {
  70     type Output = PathBuf;
  71     const DEFAULT: bool = true;
  72     const ONLY_BUILD_TARGETS: bool = true;
  73
  74     fn should_run(run: ShouldRun) -> ShouldRun {
  75         run.path("src/doc")
  76     }
  77
  78     fn make_run(run: RunConfig) {
  79         run.builder.ensure(Docs {
  80             stage: run.builder.top_stage,
  81             host: run.target,
  82         });
  83     }
  84
  85     /// Builds the `rust-docs` installer component.
  86     fn run(self, builder: &Builder) -> PathBuf {
  87         let build = builder.build;
  88         let host = self.host;
  89
  90         let name = pkgname(build, "rust-docs");
  91
  92         println!("Dist docs ({})", host);
  93         if !build.config.docs {
  94             println!("\tskipping - docs disabled");
  95             return distdir(build).join(format!("{}-{}.tar.gz", name, host));
  96         }
  97
  98         builder.default_doc(None);
  99
 100         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 101         let _ = fs::remove_dir_all(&image);
 102
 103         let dst = image.join("share/doc/rust/html");
 104         t!(fs::create_dir_all(&dst));
 105         let src = build.out.join(host).join("doc");
 106         cp_r(&src, &dst);
 107
 108         let mut cmd = rust_installer(builder);
 109         cmd.arg("generate")
 110            .arg("--product-name=Rust-Documentation")
 111            .arg("--rel-manifest-dir=rustlib")
 112            .arg("--success-message=Rust-documentation-is-installed.")
 113            .arg("--image-dir").arg(&image)
 114            .arg("--work-dir").arg(&tmpdir(build))
 115            .arg("--output-dir").arg(&distdir(build))
 116            .arg(format!("--package-name={}-{}", name, host))
 117            .arg("--component-name=rust-docs")
 118            .arg("--legacy-manifest-dirs=rustlib,cargo")
 119            .arg("--bulk-dirs=share/doc/rust/html");
 120         build.run(&mut cmd);
 121         t!(fs::remove_dir_all(&image));
 122
 123         // As part of this step, *also* copy the docs directory to a directory which
 124         // buildbot typically uploads.
 125         if host == build.build {
 126             let dst = distdir(build).join("doc").join(build.rust_package_vers());
 127             t!(fs::create_dir_all(&dst));
 128             cp_r(&src, &dst);
 129         }
 130
 131         distdir(build).join(format!("{}-{}.tar.gz", name, host))
 132     }
 133 }
 134
 135 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 136     let mut found = Vec::with_capacity(files.len());
 137
 138     for file in files {
 139         let file_path =
 140             path.iter()
 141                 .map(|dir| dir.join(file))
 142                 .find(|p| p.exists());
 143
 144         if let Some(file_path) = file_path {
 145             found.push(file_path);
 146         } else {
 147             panic!("Could not find '{}' in {:?}", file, path);
 148         }
 149     }
 150
 151     found
 152 }
 153
 154 fn make_win_dist(
 155     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, build: &Build
 156 ) {
 157     //Ask gcc where it keeps its stuff
 158     let mut cmd = Command::new(build.cc(target_triple));
 159     cmd.arg("-print-search-dirs");
 160     let gcc_out = output(&mut cmd);
 161
 162     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 163     let mut lib_path = Vec::new();
 164
 165     for line in gcc_out.lines() {
 166         let idx = line.find(':').unwrap();
 167         let key = &line[..idx];
 168         let trim_chars: &[_] = &[' ', '='];
 169         let value =
 170             line[(idx + 1)..]
 171                 .trim_left_matches(trim_chars)
 172                 .split(';')
 173                 .map(PathBuf::from);
 174
 175         if key == "programs" {
 176             bin_path.extend(value);
 177         } else if key == "libraries" {
 178             lib_path.extend(value);
 179         }
 180     }
 181
 182     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 183     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 184     if target_triple.starts_with("i686-") {
 185         rustc_dlls.push("libgcc_s_dw2-1.dll");
 186     } else {
 187         rustc_dlls.push("libgcc_s_seh-1.dll");
 188     }
 189
 190     let target_libs = [ //MinGW libs
 191         "libgcc.a",
 192         "libgcc_eh.a",
 193         "libgcc_s.a",
 194         "libm.a",
 195         "libmingw32.a",
 196         "libmingwex.a",
 197         "libstdc++.a",
 198         "libiconv.a",
 199         "libmoldname.a",
 200         "libpthread.a",
 201         //Windows import libs
 202         "libadvapi32.a",
 203         "libbcrypt.a",
 204         "libcomctl32.a",
 205         "libcomdlg32.a",
 206         "libcrypt32.a",
 207         "libgdi32.a",
 208         "libimagehlp.a",
 209         "libiphlpapi.a",
 210         "libkernel32.a",
 211         "libmsvcrt.a",
 212         "libodbc32.a",
 213         "libole32.a",
 214         "liboleaut32.a",
 215         "libopengl32.a",
 216         "libpsapi.a",
 217         "librpcrt4.a",
 218         "libsetupapi.a",
 219         "libshell32.a",
 220         "libuser32.a",
 221         "libuserenv.a",
 222         "libuuid.a",
 223         "libwinhttp.a",
 224         "libwinmm.a",
 225         "libwinspool.a",
 226         "libws2_32.a",
 227         "libwsock32.a",
 228         "libdbghelp.a",
 229         "libmsimg32.a",
 230     ];
 231
 232     //Find mingw artifacts we want to bundle
 233     let target_tools = find_files(&target_tools, &bin_path);
 234     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 235     let target_libs = find_files(&target_libs, &lib_path);
 236
 237     fn copy_to_folder(src: &Path, dest_folder: &Path) {
 238         let file_name = src.file_name().unwrap();
 239         let dest = dest_folder.join(file_name);
 240         copy(src, &dest);
 241     }
 242
 243     //Copy runtime dlls next to rustc.exe
 244     let dist_bin_dir = rust_root.join("bin/");
 245     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 246     for src in rustc_dlls {
 247         copy_to_folder(&src, &dist_bin_dir);
 248     }
 249
 250     //Copy platform tools to platform-specific bin directory
 251     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 252     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 253     for src in target_tools {
 254         copy_to_folder(&src, &target_bin_dir);
 255     }
 256
 257     //Copy platform libs to platform-specific lib directory
 258     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 259     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 260     for src in target_libs {
 261         copy_to_folder(&src, &target_lib_dir);
 262     }
 263 }
 264
 265 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 266 pub struct Mingw {
 267     host: Interned<String>,
 268 }
 269
 270 impl Step for Mingw {
 271     type Output = Option<PathBuf>;
 272     const DEFAULT: bool = true;
 273     const ONLY_BUILD_TARGETS: bool = true;
 274
 275     fn should_run(run: ShouldRun) -> ShouldRun {
 276         run.never()
 277     }
 278
 279     fn make_run(run: RunConfig) {
 280         run.builder.ensure(Mingw { host: run.target });
 281     }
 282
 283     /// Build the `rust-mingw` installer component.
 284     ///
 285     /// This contains all the bits and pieces to run the MinGW Windows targets
 286     /// without any extra installed software (e.g. we bundle gcc, libraries, etc).
 287     fn run(self, builder: &Builder) -> Option<PathBuf> {
 288         let build = builder.build;
 289         let host = self.host;
 290
 291         if !host.contains("pc-windows-gnu") {
 292             return None;
 293         }
 294
 295         println!("Dist mingw ({})", host);
 296         let name = pkgname(build, "rust-mingw");
 297         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 298         let _ = fs::remove_dir_all(&image);
 299         t!(fs::create_dir_all(&image));
 300
 301         // The first argument is a "temporary directory" which is just
 302         // thrown away (this contains the runtime DLLs included in the rustc package
 303         // above) and the second argument is where to place all the MinGW components
 304         // (which is what we want).
 305         make_win_dist(&tmpdir(build), &image, host, &build);
 306
 307         let mut cmd = rust_installer(builder);
 308         cmd.arg("generate")
 309            .arg("--product-name=Rust-MinGW")
 310            .arg("--rel-manifest-dir=rustlib")
 311            .arg("--success-message=Rust-MinGW-is-installed.")
 312            .arg("--image-dir").arg(&image)
 313            .arg("--work-dir").arg(&tmpdir(build))
 314            .arg("--output-dir").arg(&distdir(build))
 315            .arg(format!("--package-name={}-{}", name, host))
 316            .arg("--component-name=rust-mingw")
 317            .arg("--legacy-manifest-dirs=rustlib,cargo");
 318         build.run(&mut cmd);
 319         t!(fs::remove_dir_all(&image));
 320         Some(distdir(build).join(format!("{}-{}.tar.gz", name, host)))
 321     }
 322 }
 323
 324 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Rustc {
 326     pub compiler: Compiler,
 327 }
 328
 329 impl Step for Rustc {
 330     type Output = PathBuf;
 331     const DEFAULT: bool = true;
 332     const ONLY_HOSTS: bool = true;
 333     const ONLY_BUILD_TARGETS: bool = true;
 334
 335     fn should_run(run: ShouldRun) -> ShouldRun {
 336         run.path("src/librustc")
 337     }
 338
 339     fn make_run(run: RunConfig) {
 340         run.builder.ensure(Rustc {
 341             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 342         });
 343     }
 344
 345     /// Creates the `rustc` installer component.
 346     fn run(self, builder: &Builder) -> PathBuf {
 347         let build = builder.build;
 348         let compiler = self.compiler;
 349         let host = self.compiler.host;
 350
 351         println!("Dist rustc stage{} ({})", compiler.stage, compiler.host);
 352         let name = pkgname(build, "rustc");
 353         let image = tmpdir(build).join(format!("{}-{}-image", name, host));
 354         let _ = fs::remove_dir_all(&image);
 355         let overlay = tmpdir(build).join(format!("{}-{}-overlay", name, host));
 356         let _ = fs::remove_dir_all(&overlay);
 357
 358         // Prepare the rustc "image", what will actually end up getting installed
 359         prepare_image(builder, compiler, &image);
 360
 361         // Prepare the overlay which is part of the tarball but won't actually be
 362         // installed
 363         let cp = |file: &str| {
 364             install(&build.src.join(file), &overlay, 0o644);
 365         };
 366         cp("COPYRIGHT");
 367         cp("LICENSE-APACHE");
 368         cp("LICENSE-MIT");
 369         cp("README.md");
 370         // tiny morsel of metadata is used by rust-packaging
 371         let version = build.rust_version();
 372         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
 373         if let Some(sha) = build.rust_sha() {
 374             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
 375         }
 376
 377         // On MinGW we've got a few runtime DLL dependencies that we need to
 378         // include. The first argument to this script is where to put these DLLs
 379         // (the image we're creating), and the second argument is a junk directory
 380         // to ignore all other MinGW stuff the script creates.
 381         //
 382         // On 32-bit MinGW we're always including a DLL which needs some extra
 383         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 384         // anything requiring us to distribute a license, but it's likely the
 385         // install will *also* include the rust-mingw package, which also needs
 386         // licenses, so to be safe we just include it here in all MinGW packages.
 387         if host.contains("pc-windows-gnu") {
 388             make_win_dist(&image, &tmpdir(build), host, build);
 389
 390             let dst = image.join("share/doc");
 391             t!(fs::create_dir_all(&dst));
 392             cp_r(&build.src.join("src/etc/third-party"), &dst);
 393         }
 394
 395         // Finally, wrap everything up in a nice tarball!
 396         let mut cmd = rust_installer(builder);
 397         cmd.arg("generate")
 398            .arg("--product-name=Rust")
 399            .arg("--rel-manifest-dir=rustlib")
 400            .arg("--success-message=Rust-is-ready-to-roll.")
 401            .arg("--image-dir").arg(&image)
 402            .arg("--work-dir").arg(&tmpdir(build))
 403            .arg("--output-dir").arg(&distdir(build))
 404            .arg("--non-installed-overlay").arg(&overlay)
 405            .arg(format!("--package-name={}-{}", name, host))
 406            .arg("--component-name=rustc")
 407            .arg("--legacy-manifest-dirs=rustlib,cargo");
 408         build.run(&mut cmd);
 409         t!(fs::remove_dir_all(&image));
 410         t!(fs::remove_dir_all(&overlay));
 411
 412         return distdir(build).join(format!("{}-{}.tar.gz", name, host));
 413
 414         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 415             let host = compiler.host;
 416             let build = builder.build;
 417             let src = builder.sysroot(compiler);
 418             let libdir = libdir(&host);
 419
 420             // Copy rustc/rustdoc binaries
 421             t!(fs::create_dir_all(image.join("bin")));
 422             cp_r(&src.join("bin"), &image.join("bin"));
 423
 424             install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 425
 426             // Copy runtime DLLs needed by the compiler
 427             if libdir != "bin" {
 428                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
 429                     let name = entry.file_name();
 430                     if let Some(s) = name.to_str() {
 431                         if is_dylib(s) {
 432                             install(&entry.path(), &image.join(libdir), 0o644);
 433                         }
 434                     }
 435                 }
 436             }
 437
 438             // Copy over the codegen backends
 439             let backends_src = builder.sysroot_codegen_backends(compiler);
 440             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 441             let backends_dst = image.join(&backends_rel);
 442             t!(fs::create_dir_all(&backends_dst));
 443             cp_r(&backends_src, &backends_dst);
 444
 445             // Man pages
 446             t!(fs::create_dir_all(image.join("share/man/man1")));
 447             let man_src = build.src.join("src/doc/man");
 448             let man_dst = image.join("share/man/man1");
 449             let month_year = t!(time::strftime("%B %Y", &time::now()));
 450             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 451             // to hardlink, and we don't want to edit the source templates
 452             for entry_result in t!(fs::read_dir(man_src)) {
 453                 let file_entry = t!(entry_result);
 454                 let page_src = file_entry.path();
 455                 let page_dst = man_dst.join(file_entry.file_name());
 456                 t!(fs::copy(&page_src, &page_dst));
 457                 // template in month/year and version number
 458                 replace_in_file(&page_dst,
 459                                 &[("<INSERT DATE HERE>", &month_year),
 460                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 461             }
 462
 463             // Debugger scripts
 464             builder.ensure(DebuggerScripts {
 465                 sysroot: INTERNER.intern_path(image.to_owned()),
 466                 host,
 467             });
 468
 469             // Misc license info
 470             let cp = |file: &str| {
 471                 install(&build.src.join(file), &image.join("share/doc/rust"), 0o644);
 472             };
 473             cp("COPYRIGHT");
 474             cp("LICENSE-APACHE");
 475             cp("LICENSE-MIT");
 476             cp("README.md");
 477         }
 478     }
 479 }
 480
 481 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 482 pub struct DebuggerScripts {
 483     pub sysroot: Interned<PathBuf>,
 484     pub host: Interned<String>,
 485 }
 486
 487 impl Step for DebuggerScripts {
 488     type Output = ();
 489
 490     fn should_run(run: ShouldRun) -> ShouldRun {
 491         run.path("src/lldb_batchmode.py")
 492     }
 493
 494     fn make_run(run: RunConfig) {
 495         run.builder.ensure(DebuggerScripts {
 496             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 497             host: run.target,
 498         });
 499     }
 500
 501     /// Copies debugger scripts for `target` into the `sysroot` specified.
 502     fn run(self, builder: &Builder) {
 503         let build = builder.build;
 504         let host = self.host;
 505         let sysroot = self.sysroot;
 506         let dst = sysroot.join("lib/rustlib/etc");
 507         t!(fs::create_dir_all(&dst));
 508         let cp_debugger_script = |file: &str| {
 509             install(&build.src.join("src/etc/").join(file), &dst, 0o644);
 510         };
 511         if host.contains("windows-msvc") {
 512             // windbg debugger scripts
 513             install(&build.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 514                 0o755);
 515
 516             cp_debugger_script("natvis/intrinsic.natvis");
 517             cp_debugger_script("natvis/liballoc.natvis");
 518             cp_debugger_script("natvis/libcore.natvis");
 519         } else {
 520             cp_debugger_script("debugger_pretty_printers_common.py");
 521
 522             // gdb debugger scripts
 523             install(&build.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 524                     0o755);
 525
 526             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 527             cp_debugger_script("gdb_rust_pretty_printing.py");
 528
 529             // lldb debugger scripts
 530             install(&build.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 531                     0o755);
 532
 533             cp_debugger_script("lldb_rust_formatters.py");
 534         }
 535     }
 536 }
 537
 538 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 539 pub struct Std {
 540     pub compiler: Compiler,
 541     pub target: Interned<String>,
 542 }
 543
 544 impl Step for Std {
 545     type Output = PathBuf;
 546     const DEFAULT: bool = true;
 547     const ONLY_BUILD_TARGETS: bool = true;
 548
 549     fn should_run(run: ShouldRun) -> ShouldRun {
 550         run.path("src/libstd")
 551     }
 552
 553     fn make_run(run: RunConfig) {
 554         run.builder.ensure(Std {
 555             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 556             target: run.target,
 557         });
 558     }
 559
 560     fn run(self, builder: &Builder) -> PathBuf {
 561         let build = builder.build;
 562         let compiler = self.compiler;
 563         let target = self.target;
 564
 565         let name = pkgname(build, "rust-std");
 566         println!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target);
 567
 568         // The only true set of target libraries came from the build triple, so
 569         // let's reduce redundant work by only producing archives from that host.
 570         if compiler.host != build.build {
 571             println!("\tskipping, not a build host");
 572             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 573         }
 574
 575         // We want to package up as many target libraries as possible
 576         // for the `rust-std` package, so if this is a host target we
 577         // depend on librustc and otherwise we just depend on libtest.
 578         if build.hosts.iter().any(|t| t == target) {
 579             builder.ensure(compile::Rustc { compiler, target });
 580         } else {
 581             builder.ensure(compile::Test { compiler, target });
 582         }
 583
 584         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 585         let _ = fs::remove_dir_all(&image);
 586
 587         let dst = image.join("lib/rustlib").join(target);
 588         t!(fs::create_dir_all(&dst));
 589         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 590         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 591         cp_filtered(&src, &dst, &|path| {
 592             path.file_name().and_then(|s| s.to_str()) != Some("codegen-backends")
 593         });
 594
 595         let mut cmd = rust_installer(builder);
 596         cmd.arg("generate")
 597            .arg("--product-name=Rust")
 598            .arg("--rel-manifest-dir=rustlib")
 599            .arg("--success-message=std-is-standing-at-the-ready.")
 600            .arg("--image-dir").arg(&image)
 601            .arg("--work-dir").arg(&tmpdir(build))
 602            .arg("--output-dir").arg(&distdir(build))
 603            .arg(format!("--package-name={}-{}", name, target))
 604            .arg(format!("--component-name=rust-std-{}", target))
 605            .arg("--legacy-manifest-dirs=rustlib,cargo");
 606         build.run(&mut cmd);
 607         t!(fs::remove_dir_all(&image));
 608         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 609     }
 610 }
 611
 612 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 613 pub struct Analysis {
 614     pub compiler: Compiler,
 615     pub target: Interned<String>,
 616 }
 617
 618 impl Step for Analysis {
 619     type Output = PathBuf;
 620     const DEFAULT: bool = true;
 621     const ONLY_BUILD_TARGETS: bool = true;
 622
 623     fn should_run(run: ShouldRun) -> ShouldRun {
 624         let builder = run.builder;
 625         run.path("analysis").default_condition(builder.build.config.extended)
 626     }
 627
 628     fn make_run(run: RunConfig) {
 629         run.builder.ensure(Analysis {
 630             compiler: run.builder.compiler(run.builder.top_stage, run.host),
 631             target: run.target,
 632         });
 633     }
 634
 635     /// Creates a tarball of save-analysis metadata, if available.
 636     fn run(self, builder: &Builder) -> PathBuf {
 637         let build = builder.build;
 638         let compiler = self.compiler;
 639         let target = self.target;
 640         assert!(build.config.extended);
 641         println!("Dist analysis");
 642         let name = pkgname(build, "rust-analysis");
 643
 644         if &compiler.host != build.build {
 645             println!("\tskipping, not a build host");
 646             return distdir(build).join(format!("{}-{}.tar.gz", name, target));
 647         }
 648
 649         builder.ensure(Std { compiler, target });
 650
 651         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 652         // stage2 artifacts is simply copied from stage1 in that case.
 653         let compiler = if build.force_use_stage1(compiler, target) {
 654             builder.compiler(1, compiler.host)
 655         } else {
 656             compiler.clone()
 657         };
 658
 659         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 660
 661         let src = build.stage_out(compiler, Mode::Libstd)
 662             .join(target).join(build.cargo_dir()).join("deps");
 663
 664         let image_src = src.join("save-analysis");
 665         let dst = image.join("lib/rustlib").join(target).join("analysis");
 666         t!(fs::create_dir_all(&dst));
 667         println!("image_src: {:?}, dst: {:?}", image_src, dst);
 668         cp_r(&image_src, &dst);
 669
 670         let mut cmd = rust_installer(builder);
 671         cmd.arg("generate")
 672            .arg("--product-name=Rust")
 673            .arg("--rel-manifest-dir=rustlib")
 674            .arg("--success-message=save-analysis-saved.")
 675            .arg("--image-dir").arg(&image)
 676            .arg("--work-dir").arg(&tmpdir(build))
 677            .arg("--output-dir").arg(&distdir(build))
 678            .arg(format!("--package-name={}-{}", name, target))
 679            .arg(format!("--component-name=rust-analysis-{}", target))
 680            .arg("--legacy-manifest-dirs=rustlib,cargo");
 681         build.run(&mut cmd);
 682         t!(fs::remove_dir_all(&image));
 683         distdir(build).join(format!("{}-{}.tar.gz", name, target))
 684     }
 685 }
 686
 687 fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 688     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 689         let spath = match path.to_str() {
 690             Some(path) => path,
 691             None => return false,
 692         };
 693         if spath.ends_with("~") || spath.ends_with(".pyc") {
 694             return false
 695         }
 696         if (spath.contains("llvm/test") || spath.contains("llvm\\test")) &&
 697             (spath.ends_with(".ll") ||
 698              spath.ends_with(".td") ||
 699              spath.ends_with(".s")) {
 700             return false
 701         }
 702         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 703             return false
 704         }
 705
 706         let full_path = Path::new(dir).join(path);
 707         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 708             return false;
 709         }
 710
 711         let excludes = [
 712             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 713             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 714             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 715             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 716         ];
 717         !path.iter()
 718              .map(|s| s.to_str().unwrap())
 719              .any(|s| excludes.contains(&s))
 720     }
 721
 722     // Copy the directories using our filter
 723     for item in src_dirs {
 724         let dst = &dst_dir.join(item);
 725         t!(fs::create_dir_all(dst));
 726         cp_filtered(&build.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 727     }
 728 }
 729
 730 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 731 pub struct Src;
 732
 733 impl Step for Src {
 734     /// The output path of the src installer tarball
 735     type Output = PathBuf;
 736     const DEFAULT: bool = true;
 737     const ONLY_HOSTS: bool = true;
 738     const ONLY_BUILD_TARGETS: bool = true;
 739     const ONLY_BUILD: bool = true;
 740
 741     fn should_run(run: ShouldRun) -> ShouldRun {
 742         run.path("src")
 743     }
 744
 745     fn make_run(run: RunConfig) {
 746         run.builder.ensure(Src);
 747     }
 748
 749     /// Creates the `rust-src` installer component
 750     fn run(self, builder: &Builder) -> PathBuf {
 751         let build = builder.build;
 752         println!("Dist src");
 753
 754         let name = pkgname(build, "rust-src");
 755         let image = tmpdir(build).join(format!("{}-image", name));
 756         let _ = fs::remove_dir_all(&image);
 757
 758         let dst = image.join("lib/rustlib/src");
 759         let dst_src = dst.join("rust");
 760         t!(fs::create_dir_all(&dst_src));
 761
 762         let src_files = [
 763             "src/Cargo.lock",
 764         ];
 765         // This is the reduced set of paths which will become the rust-src component
 766         // (essentially libstd and all of its path dependencies)
 767         let std_src_dirs = [
 768             "src/build_helper",
 769             "src/dlmalloc",
 770             "src/liballoc",
 771             "src/liballoc_jemalloc",
 772             "src/liballoc_system",
 773             "src/libbacktrace",
 774             "src/libcompiler_builtins",
 775             "src/libcore",
 776             "src/liblibc",
 777             "src/libpanic_abort",
 778             "src/libpanic_unwind",
 779             "src/librustc_asan",
 780             "src/librustc_lsan",
 781             "src/librustc_msan",
 782             "src/librustc_tsan",
 783             "src/libstd",
 784             "src/libstd_unicode",
 785             "src/libunwind",
 786             "src/rustc/compiler_builtins_shim",
 787             "src/rustc/libc_shim",
 788             "src/rustc/dlmalloc_shim",
 789             "src/libtest",
 790             "src/libterm",
 791             "src/jemalloc",
 792             "src/libprofiler_builtins",
 793         ];
 794         let std_src_dirs_exclude = [
 795             "src/libcompiler_builtins/compiler-rt/test",
 796             "src/jemalloc/test/unit",
 797         ];
 798
 799         copy_src_dirs(build, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);
 800         for file in src_files.iter() {
 801             copy(&build.src.join(file), &dst_src.join(file));
 802         }
 803
 804         // Create source tarball in rust-installer format
 805         let mut cmd = rust_installer(builder);
 806         cmd.arg("generate")
 807            .arg("--product-name=Rust")
 808            .arg("--rel-manifest-dir=rustlib")
 809            .arg("--success-message=Awesome-Source.")
 810            .arg("--image-dir").arg(&image)
 811            .arg("--work-dir").arg(&tmpdir(build))
 812            .arg("--output-dir").arg(&distdir(build))
 813            .arg(format!("--package-name={}", name))
 814            .arg("--component-name=rust-src")
 815            .arg("--legacy-manifest-dirs=rustlib,cargo");
 816         build.run(&mut cmd);
 817
 818         t!(fs::remove_dir_all(&image));
 819         distdir(build).join(&format!("{}.tar.gz", name))
 820     }
 821 }
 822
 823 const CARGO_VENDOR_VERSION: &str = "0.1.4";
 824
 825 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 826 pub struct PlainSourceTarball;
 827
 828 impl Step for PlainSourceTarball {
 829     /// Produces the location of the tarball generated
 830     type Output = PathBuf;
 831     const DEFAULT: bool = true;
 832     const ONLY_HOSTS: bool = true;
 833     const ONLY_BUILD_TARGETS: bool = true;
 834     const ONLY_BUILD: bool = true;
 835
 836     fn should_run(run: ShouldRun) -> ShouldRun {
 837         let builder = run.builder;
 838         run.path("src").default_condition(builder.config.rust_dist_src)
 839     }
 840
 841     fn make_run(run: RunConfig) {
 842         run.builder.ensure(PlainSourceTarball);
 843     }
 844
 845     /// Creates the plain source tarball
 846     fn run(self, builder: &Builder) -> PathBuf {
 847         let build = builder.build;
 848         println!("Create plain source tarball");
 849
 850         // Make sure that the root folder of tarball has the correct name
 851         let plain_name = format!("{}-src", pkgname(build, "rustc"));
 852         let plain_dst_src = tmpdir(build).join(&plain_name);
 853         let _ = fs::remove_dir_all(&plain_dst_src);
 854         t!(fs::create_dir_all(&plain_dst_src));
 855
 856         // This is the set of root paths which will become part of the source package
 857         let src_files = [
 858             "COPYRIGHT",
 859             "LICENSE-APACHE",
 860             "LICENSE-MIT",
 861             "CONTRIBUTING.md",
 862             "README.md",
 863             "RELEASES.md",
 864             "configure",
 865             "x.py",
 866             "config.toml.example",
 867         ];
 868         let src_dirs = [
 869             "src",
 870         ];
 871
 872         copy_src_dirs(build, &src_dirs[..], &[], &plain_dst_src);
 873
 874         // Copy the files normally
 875         for item in &src_files {
 876             copy(&build.src.join(item), &plain_dst_src.join(item));
 877         }
 878
 879         // Create the version file
 880         write_file(&plain_dst_src.join("version"), build.rust_version().as_bytes());
 881         if let Some(sha) = build.rust_sha() {
 882             write_file(&plain_dst_src.join("git-commit-hash"), sha.as_bytes());
 883         }
 884
 885         // If we're building from git sources, we need to vendor a complete distribution.
 886         if build.rust_info.is_git() {
 887             // Get cargo-vendor installed, if it isn't already.
 888             let mut has_cargo_vendor = false;
 889             let mut cmd = Command::new(&build.initial_cargo);
 890             for line in output(cmd.arg("install").arg("--list")).lines() {
 891                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 892             }
 893             if !has_cargo_vendor {
 894                 let mut cmd = Command::new(&build.initial_cargo);
 895                 cmd.arg("install")
 896                    .arg("--force")
 897                    .arg("--debug")
 898                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
 899                    .arg("cargo-vendor")
 900                    .env("RUSTC", &build.initial_rustc);
 901                 build.run(&mut cmd);
 902             }
 903
 904             // Vendor all Cargo dependencies
 905             let mut cmd = Command::new(&build.initial_cargo);
 906             cmd.arg("vendor")
 907                .current_dir(&plain_dst_src.join("src"));
 908             build.run(&mut cmd);
 909         }
 910
 911         // Create plain source tarball
 912         let plain_name = format!("rustc-{}-src", build.rust_package_vers());
 913         let mut tarball = distdir(build).join(&format!("{}.tar.gz", plain_name));
 914         tarball.set_extension(""); // strip .gz
 915         tarball.set_extension(""); // strip .tar
 916         if let Some(dir) = tarball.parent() {
 917             t!(fs::create_dir_all(dir));
 918         }
 919         println!("running installer");
 920         let mut cmd = rust_installer(builder);
 921         cmd.arg("tarball")
 922            .arg("--input").arg(&plain_name)
 923            .arg("--output").arg(&tarball)
 924            .arg("--work-dir=.")
 925            .current_dir(tmpdir(build));
 926         build.run(&mut cmd);
 927         distdir(build).join(&format!("{}.tar.gz", plain_name))
 928     }
 929 }
 930
 931 fn install(src: &Path, dstdir: &Path, perms: u32) {
 932     let dst = dstdir.join(src.file_name().unwrap());
 933     t!(fs::create_dir_all(dstdir));
 934     drop(fs::remove_file(&dst));
 935     {
 936         let mut s = t!(fs::File::open(&src));
 937         let mut d = t!(fs::File::create(&dst));
 938         io::copy(&mut s, &mut d).expect("failed to copy");
 939     }
 940     chmod(&dst, perms);
 941 }
 942
 943 #[cfg(unix)]
 944 fn chmod(path: &Path, perms: u32) {
 945     use std::os::unix::fs::*;
 946     t!(fs::set_permissions(path, fs::Permissions::from_mode(perms)));
 947 }
 948 #[cfg(windows)]
 949 fn chmod(_path: &Path, _perms: u32) {}
 950
 951 // We have to run a few shell scripts, which choke quite a bit on both `\`
 952 // characters and on `C:\` paths, so normalize both of them away.
 953 pub fn sanitize_sh(path: &Path) -> String {
 954     let path = path.to_str().unwrap().replace("\\", "/");
 955     return change_drive(&path).unwrap_or(path);
 956
 957     fn change_drive(s: &str) -> Option<String> {
 958         let mut ch = s.chars();
 959         let drive = ch.next().unwrap_or('C');
 960         if ch.next() != Some(':') {
 961             return None
 962         }
 963         if ch.next() != Some('/') {
 964             return None
 965         }
 966         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
 967     }
 968 }
 969
 970 fn write_file(path: &Path, data: &[u8]) {
 971     let mut vf = t!(fs::File::create(path));
 972     t!(vf.write_all(data));
 973 }
 974
 975 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 976 pub struct Cargo {
 977     pub stage: u32,
 978     pub target: Interned<String>,
 979 }
 980
 981 impl Step for Cargo {
 982     type Output = PathBuf;
 983     const ONLY_BUILD_TARGETS: bool = true;
 984     const ONLY_HOSTS: bool = true;
 985
 986     fn should_run(run: ShouldRun) -> ShouldRun {
 987         run.path("cargo")
 988     }
 989
 990     fn make_run(run: RunConfig) {
 991         run.builder.ensure(Cargo {
 992             stage: run.builder.top_stage,
 993             target: run.target,
 994         });
 995     }
 996
 997     fn run(self, builder: &Builder) -> PathBuf {
 998         let build = builder.build;
 999         let stage = self.stage;
1000         let target = self.target;
1001
1002         println!("Dist cargo stage{} ({})", stage, target);
1003         let src = build.src.join("src/tools/cargo");
1004         let etc = src.join("src/etc");
1005         let release_num = build.release_num("cargo");
1006         let name = pkgname(build, "cargo");
1007         let version = builder.cargo_info.version(build, &release_num);
1008
1009         let tmp = tmpdir(build);
1010         let image = tmp.join("cargo-image");
1011         drop(fs::remove_dir_all(&image));
1012         t!(fs::create_dir_all(&image));
1013
1014         // Prepare the image directory
1015         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
1016         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
1017         let cargo = builder.ensure(tool::Cargo {
1018             compiler: builder.compiler(stage, build.build),
1019             target
1020         });
1021         install(&cargo, &image.join("bin"), 0o755);
1022         for man in t!(etc.join("man").read_dir()) {
1023             let man = t!(man);
1024             install(&man.path(), &image.join("share/man/man1"), 0o644);
1025         }
1026         install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1027         copy(&etc.join("cargo.bashcomp.sh"),
1028              &image.join("etc/bash_completion.d/cargo"));
1029         let doc = image.join("share/doc/cargo");
1030         install(&src.join("README.md"), &doc, 0o644);
1031         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1032         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1033         install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1034
1035         // Prepare the overlay
1036         let overlay = tmp.join("cargo-overlay");
1037         drop(fs::remove_dir_all(&overlay));
1038         t!(fs::create_dir_all(&overlay));
1039         install(&src.join("README.md"), &overlay, 0o644);
1040         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1041         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1042         install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1043         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1044
1045         // Generate the installer tarball
1046         let mut cmd = rust_installer(builder);
1047         cmd.arg("generate")
1048            .arg("--product-name=Rust")
1049            .arg("--rel-manifest-dir=rustlib")
1050            .arg("--success-message=Rust-is-ready-to-roll.")
1051            .arg("--image-dir").arg(&image)
1052            .arg("--work-dir").arg(&tmpdir(build))
1053            .arg("--output-dir").arg(&distdir(build))
1054            .arg("--non-installed-overlay").arg(&overlay)
1055            .arg(format!("--package-name={}-{}", name, target))
1056            .arg("--component-name=cargo")
1057            .arg("--legacy-manifest-dirs=rustlib,cargo");
1058         build.run(&mut cmd);
1059         distdir(build).join(format!("{}-{}.tar.gz", name, target))
1060     }
1061 }
1062
1063 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1064 pub struct Rls {
1065     pub stage: u32,
1066     pub target: Interned<String>,
1067 }
1068
1069 impl Step for Rls {
1070     type Output = Option<PathBuf>;
1071     const ONLY_BUILD_TARGETS: bool = true;
1072     const ONLY_HOSTS: bool = true;
1073
1074     fn should_run(run: ShouldRun) -> ShouldRun {
1075         run.path("rls")
1076     }
1077
1078     fn make_run(run: RunConfig) {
1079         run.builder.ensure(Rls {
1080             stage: run.builder.top_stage,
1081             target: run.target,
1082         });
1083     }
1084
1085     fn run(self, builder: &Builder) -> Option<PathBuf> {
1086         let build = builder.build;
1087         let stage = self.stage;
1088         let target = self.target;
1089         assert!(build.config.extended);
1090
1091         println!("Dist RLS stage{} ({})", stage, target);
1092         let src = build.src.join("src/tools/rls");
1093         let release_num = build.release_num("rls");
1094         let name = pkgname(build, "rls");
1095         let version = build.rls_info.version(build, &release_num);
1096
1097         let tmp = tmpdir(build);
1098         let image = tmp.join("rls-image");
1099         drop(fs::remove_dir_all(&image));
1100         t!(fs::create_dir_all(&image));
1101
1102         // Prepare the image directory
1103         // We expect RLS to build, because we've exited this step above if tool
1104         // state for RLS isn't testing.
1105         let rls = builder.ensure(tool::Rls {
1106             compiler: builder.compiler(stage, build.build),
1107             target
1108         }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
1109
1110         install(&rls, &image.join("bin"), 0o755);
1111         let doc = image.join("share/doc/rls");
1112         install(&src.join("README.md"), &doc, 0o644);
1113         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1114         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1115
1116         // Prepare the overlay
1117         let overlay = tmp.join("rls-overlay");
1118         drop(fs::remove_dir_all(&overlay));
1119         t!(fs::create_dir_all(&overlay));
1120         install(&src.join("README.md"), &overlay, 0o644);
1121         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1122         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1123         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1124
1125         // Generate the installer tarball
1126         let mut cmd = rust_installer(builder);
1127         cmd.arg("generate")
1128            .arg("--product-name=Rust")
1129            .arg("--rel-manifest-dir=rustlib")
1130            .arg("--success-message=RLS-ready-to-serve.")
1131            .arg("--image-dir").arg(&image)
1132            .arg("--work-dir").arg(&tmpdir(build))
1133            .arg("--output-dir").arg(&distdir(build))
1134            .arg("--non-installed-overlay").arg(&overlay)
1135            .arg(format!("--package-name={}-{}", name, target))
1136            .arg("--legacy-manifest-dirs=rustlib,cargo")
1137            .arg("--component-name=rls-preview");
1138
1139         build.run(&mut cmd);
1140         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1141     }
1142 }
1143
1144
1145 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1146 pub struct Rustfmt {
1147     pub stage: u32,
1148     pub target: Interned<String>,
1149 }
1150
1151 impl Step for Rustfmt {
1152     type Output = Option<PathBuf>;
1153     const ONLY_BUILD_TARGETS: bool = true;
1154     const ONLY_HOSTS: bool = true;
1155
1156     fn should_run(run: ShouldRun) -> ShouldRun {
1157         run.path("rustfmt")
1158     }
1159
1160     fn make_run(run: RunConfig) {
1161         run.builder.ensure(Rustfmt {
1162             stage: run.builder.top_stage,
1163             target: run.target,
1164         });
1165     }
1166
1167     fn run(self, builder: &Builder) -> Option<PathBuf> {
1168         let build = builder.build;
1169         let stage = self.stage;
1170         let target = self.target;
1171         assert!(build.config.extended);
1172
1173         println!("Dist Rustfmt stage{} ({})", stage, target);
1174         let src = build.src.join("src/tools/rustfmt");
1175         let release_num = build.release_num("rustfmt");
1176         let name = pkgname(build, "rustfmt");
1177         let version = build.rustfmt_info.version(build, &release_num);
1178
1179         let tmp = tmpdir(build);
1180         let image = tmp.join("rustfmt-image");
1181         drop(fs::remove_dir_all(&image));
1182         t!(fs::create_dir_all(&image));
1183
1184         // Prepare the image directory
1185         let rustfmt = builder.ensure(tool::Rustfmt {
1186             compiler: builder.compiler(stage, build.build),
1187             target
1188         }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
1189         let cargofmt = builder.ensure(tool::Cargofmt {
1190             compiler: builder.compiler(stage, build.build),
1191             target
1192         }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
1193
1194         install(&rustfmt, &image.join("bin"), 0o755);
1195         install(&cargofmt, &image.join("bin"), 0o755);
1196         let doc = image.join("share/doc/rustfmt");
1197         install(&src.join("README.md"), &doc, 0o644);
1198         install(&src.join("LICENSE-MIT"), &doc, 0o644);
1199         install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1200
1201         // Prepare the overlay
1202         let overlay = tmp.join("rustfmt-overlay");
1203         drop(fs::remove_dir_all(&overlay));
1204         t!(fs::create_dir_all(&overlay));
1205         install(&src.join("README.md"), &overlay, 0o644);
1206         install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1207         install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1208         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1209
1210         // Generate the installer tarball
1211         let mut cmd = rust_installer(builder);
1212         cmd.arg("generate")
1213            .arg("--product-name=Rust")
1214            .arg("--rel-manifest-dir=rustlib")
1215            .arg("--success-message=rustfmt-ready-to-fmt.")
1216            .arg("--image-dir").arg(&image)
1217            .arg("--work-dir").arg(&tmpdir(build))
1218            .arg("--output-dir").arg(&distdir(build))
1219            .arg("--non-installed-overlay").arg(&overlay)
1220            .arg(format!("--package-name={}-{}", name, target))
1221            .arg("--legacy-manifest-dirs=rustlib,cargo")
1222            .arg("--component-name=rustfmt-preview");
1223
1224         build.run(&mut cmd);
1225         Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
1226     }
1227 }
1228
1229
1230 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1231 pub struct DontDistWithMiriEnabled;
1232
1233 impl Step for DontDistWithMiriEnabled {
1234     type Output = PathBuf;
1235     const DEFAULT: bool = true;
1236
1237     fn should_run(run: ShouldRun) -> ShouldRun {
1238         let build_miri = run.builder.build.config.test_miri;
1239         run.default_condition(build_miri)
1240     }
1241
1242     fn make_run(run: RunConfig) {
1243         run.builder.ensure(DontDistWithMiriEnabled);
1244     }
1245
1246     fn run(self, _: &Builder) -> PathBuf {
1247         panic!("Do not distribute with miri enabled.\n\
1248                 The distributed libraries would include all MIR (increasing binary size).
1249                 The distributed MIR would include validation statements.");
1250     }
1251 }
1252
1253
1254 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1255 pub struct Extended {
1256     stage: u32,
1257     host: Interned<String>,
1258     target: Interned<String>,
1259 }
1260
1261 impl Step for Extended {
1262     type Output = ();
1263     const DEFAULT: bool = true;
1264     const ONLY_BUILD_TARGETS: bool = true;
1265     const ONLY_HOSTS: bool = true;
1266
1267     fn should_run(run: ShouldRun) -> ShouldRun {
1268         let builder = run.builder;
1269         run.path("extended").default_condition(builder.config.extended)
1270     }
1271
1272     fn make_run(run: RunConfig) {
1273         run.builder.ensure(Extended {
1274             stage: run.builder.top_stage,
1275             host: run.host,
1276             target: run.target,
1277         });
1278     }
1279
1280     /// Creates a combined installer for the specified target in the provided stage.
1281     fn run(self, builder: &Builder) {
1282         let build = builder.build;
1283         let stage = self.stage;
1284         let target = self.target;
1285
1286         println!("Dist extended stage{} ({})", stage, target);
1287
1288         let rustc_installer = builder.ensure(Rustc {
1289             compiler: builder.compiler(stage, target),
1290         });
1291         let cargo_installer = builder.ensure(Cargo { stage, target });
1292         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1293         let rls_installer = builder.ensure(Rls { stage, target });
1294         let mingw_installer = builder.ensure(Mingw { host: target });
1295         let analysis_installer = builder.ensure(Analysis {
1296             compiler: builder.compiler(stage, self.host),
1297             target
1298         });
1299
1300         let docs_installer = builder.ensure(Docs { stage, host: target, });
1301         let std_installer = builder.ensure(Std {
1302             compiler: builder.compiler(stage, self.host),
1303             target,
1304         });
1305
1306         let tmp = tmpdir(build);
1307         let overlay = tmp.join("extended-overlay");
1308         let etc = build.src.join("src/etc/installer");
1309         let work = tmp.join("work");
1310
1311         let _ = fs::remove_dir_all(&overlay);
1312         install(&build.src.join("COPYRIGHT"), &overlay, 0o644);
1313         install(&build.src.join("LICENSE-APACHE"), &overlay, 0o644);
1314         install(&build.src.join("LICENSE-MIT"), &overlay, 0o644);
1315         let version = build.rust_version();
1316         t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
1317         if let Some(sha) = build.rust_sha() {
1318             t!(t!(File::create(overlay.join("git-commit-hash"))).write_all(sha.as_bytes()));
1319         }
1320         install(&etc.join("README.md"), &overlay, 0o644);
1321
1322         // When rust-std package split from rustc, we needed to ensure that during
1323         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1324         // the std files during uninstall. To do this ensure that rustc comes
1325         // before rust-std in the list below.
1326         let mut tarballs = Vec::new();
1327         tarballs.push(rustc_installer);
1328         tarballs.push(cargo_installer);
1329         tarballs.extend(rls_installer.clone());
1330         tarballs.extend(rustfmt_installer.clone());
1331         tarballs.push(analysis_installer);
1332         tarballs.push(std_installer);
1333         if build.config.docs {
1334             tarballs.push(docs_installer);
1335         }
1336         if target.contains("pc-windows-gnu") {
1337             tarballs.push(mingw_installer.unwrap());
1338         }
1339         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1340         for tarball in &tarballs[1..] {
1341             input_tarballs.push(",");
1342             input_tarballs.push(tarball);
1343         }
1344
1345         let mut cmd = rust_installer(builder);
1346         cmd.arg("combine")
1347             .arg("--product-name=Rust")
1348             .arg("--rel-manifest-dir=rustlib")
1349             .arg("--success-message=Rust-is-ready-to-roll.")
1350             .arg("--work-dir").arg(&work)
1351             .arg("--output-dir").arg(&distdir(build))
1352             .arg(format!("--package-name={}-{}", pkgname(build, "rust"), target))
1353             .arg("--legacy-manifest-dirs=rustlib,cargo")
1354             .arg("--input-tarballs").arg(input_tarballs)
1355             .arg("--non-installed-overlay").arg(&overlay);
1356         build.run(&mut cmd);
1357
1358         let mut license = String::new();
1359         t!(t!(File::open(build.src.join("COPYRIGHT"))).read_to_string(&mut license));
1360         license.push_str("\n");
1361         t!(t!(File::open(build.src.join("LICENSE-APACHE"))).read_to_string(&mut license));
1362         license.push_str("\n");
1363         t!(t!(File::open(build.src.join("LICENSE-MIT"))).read_to_string(&mut license));
1364
1365         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1366         let mut rtf = rtf.to_string();
1367         rtf.push_str("\n");
1368         for line in license.lines() {
1369             rtf.push_str(line);
1370             rtf.push_str("\\line ");
1371         }
1372         rtf.push_str("}");
1373
1374         fn filter(contents: &str, marker: &str) -> String {
1375             let start = format!("tool-{}-start", marker);
1376             let end = format!("tool-{}-end", marker);
1377             let mut lines = Vec::new();
1378             let mut omitted = false;
1379             for line in contents.lines() {
1380                 if line.contains(&start) {
1381                     omitted = true;
1382                 } else if line.contains(&end) {
1383                     omitted = false;
1384                 } else if !omitted {
1385                     lines.push(line);
1386                 }
1387             }
1388
1389             lines.join("\n")
1390         }
1391
1392         let xform = |p: &Path| {
1393             let mut contents = String::new();
1394             t!(t!(File::open(p)).read_to_string(&mut contents));
1395             if rls_installer.is_none() {
1396                 contents = filter(&contents, "rls");
1397             }
1398             if rustfmt_installer.is_none() {
1399                 contents = filter(&contents, "rustfmt");
1400             }
1401             let ret = tmp.join(p.file_name().unwrap());
1402             t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
1403             return ret
1404         };
1405
1406         if target.contains("apple-darwin") {
1407             let pkg = tmp.join("pkg");
1408             let _ = fs::remove_dir_all(&pkg);
1409
1410             let pkgbuild = |component: &str| {
1411                 let mut cmd = Command::new("pkgbuild");
1412                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1413                     .arg("--scripts").arg(pkg.join(component))
1414                     .arg("--nopayload")
1415                     .arg(pkg.join(component).with_extension("pkg"));
1416                 build.run(&mut cmd);
1417             };
1418
1419             let prepare = |name: &str| {
1420                 t!(fs::create_dir_all(pkg.join(name)));
1421                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
1422                         &pkg.join(name));
1423                 install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1424                 pkgbuild(name);
1425             };
1426             prepare("rustc");
1427             prepare("cargo");
1428             prepare("rust-docs");
1429             prepare("rust-std");
1430             prepare("rust-analysis");
1431
1432             if rls_installer.is_some() {
1433                 prepare("rls");
1434             }
1435
1436             // create an 'uninstall' package
1437             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1438             pkgbuild("uninstall");
1439
1440             t!(fs::create_dir_all(pkg.join("res")));
1441             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
1442             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1443             let mut cmd = Command::new("productbuild");
1444             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1445                 .arg("--resources").arg(pkg.join("res"))
1446                 .arg(distdir(build).join(format!("{}-{}.pkg",
1447                                                     pkgname(build, "rust"),
1448                                                     target)))
1449                 .arg("--package-path").arg(&pkg);
1450             build.run(&mut cmd);
1451         }
1452
1453         if target.contains("windows") {
1454             let exe = tmp.join("exe");
1455             let _ = fs::remove_dir_all(&exe);
1456
1457             let prepare = |name: &str| {
1458                 t!(fs::create_dir_all(exe.join(name)));
1459                 let dir = if name == "rust-std" || name == "rust-analysis" {
1460                     format!("{}-{}", name, target)
1461                 } else if name == "rls" {
1462                     "rls-preview".to_string()
1463                 } else {
1464                     name.to_string()
1465                 };
1466                 cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
1467                             .join(dir),
1468                         &exe.join(name));
1469                 t!(fs::remove_file(exe.join(name).join("manifest.in")));
1470             };
1471             prepare("rustc");
1472             prepare("cargo");
1473             prepare("rust-analysis");
1474             prepare("rust-docs");
1475             prepare("rust-std");
1476             if rls_installer.is_some() {
1477                 prepare("rls");
1478             }
1479             if target.contains("windows-gnu") {
1480                 prepare("rust-mingw");
1481             }
1482
1483             install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1484             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1485             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1486             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1487             t!(t!(File::create(exe.join("LICENSE.txt"))).write_all(license.as_bytes()));
1488
1489             // Generate exe installer
1490             let mut cmd = Command::new("iscc");
1491             cmd.arg("rust.iss")
1492                 .current_dir(&exe);
1493             if target.contains("windows-gnu") {
1494                 cmd.arg("/dMINGW");
1495             }
1496             add_env(build, &mut cmd, target);
1497             build.run(&mut cmd);
1498             install(&exe.join(format!("{}-{}.exe", pkgname(build, "rust"), target)),
1499                     &distdir(build),
1500                     0o755);
1501
1502             // Generate msi installer
1503             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1504             let heat = wix.join("bin/heat.exe");
1505             let candle = wix.join("bin/candle.exe");
1506             let light = wix.join("bin/light.exe");
1507
1508             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1509             build.run(Command::new(&heat)
1510                             .current_dir(&exe)
1511                             .arg("dir")
1512                             .arg("rustc")
1513                             .args(&heat_flags)
1514                             .arg("-cg").arg("RustcGroup")
1515                             .arg("-dr").arg("Rustc")
1516                             .arg("-var").arg("var.RustcDir")
1517                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1518             build.run(Command::new(&heat)
1519                             .current_dir(&exe)
1520                             .arg("dir")
1521                             .arg("rust-docs")
1522                             .args(&heat_flags)
1523                             .arg("-cg").arg("DocsGroup")
1524                             .arg("-dr").arg("Docs")
1525                             .arg("-var").arg("var.DocsDir")
1526                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1527                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1528             build.run(Command::new(&heat)
1529                             .current_dir(&exe)
1530                             .arg("dir")
1531                             .arg("cargo")
1532                             .args(&heat_flags)
1533                             .arg("-cg").arg("CargoGroup")
1534                             .arg("-dr").arg("Cargo")
1535                             .arg("-var").arg("var.CargoDir")
1536                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1537                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1538             build.run(Command::new(&heat)
1539                             .current_dir(&exe)
1540                             .arg("dir")
1541                             .arg("rust-std")
1542                             .args(&heat_flags)
1543                             .arg("-cg").arg("StdGroup")
1544                             .arg("-dr").arg("Std")
1545                             .arg("-var").arg("var.StdDir")
1546                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1547             if rls_installer.is_some() {
1548                 build.run(Command::new(&heat)
1549                                 .current_dir(&exe)
1550                                 .arg("dir")
1551                                 .arg("rls")
1552                                 .args(&heat_flags)
1553                                 .arg("-cg").arg("RlsGroup")
1554                                 .arg("-dr").arg("Rls")
1555                                 .arg("-var").arg("var.RlsDir")
1556                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1557                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1558             }
1559             build.run(Command::new(&heat)
1560                             .current_dir(&exe)
1561                             .arg("dir")
1562                             .arg("rust-analysis")
1563                             .args(&heat_flags)
1564                             .arg("-cg").arg("AnalysisGroup")
1565                             .arg("-dr").arg("Analysis")
1566                             .arg("-var").arg("var.AnalysisDir")
1567                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1568                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1569             if target.contains("windows-gnu") {
1570                 build.run(Command::new(&heat)
1571                                 .current_dir(&exe)
1572                                 .arg("dir")
1573                                 .arg("rust-mingw")
1574                                 .args(&heat_flags)
1575                                 .arg("-cg").arg("GccGroup")
1576                                 .arg("-dr").arg("Gcc")
1577                                 .arg("-var").arg("var.GccDir")
1578                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1579             }
1580
1581             let candle = |input: &Path| {
1582                 let output = exe.join(input.file_stem().unwrap())
1583                                 .with_extension("wixobj");
1584                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1585                 let mut cmd = Command::new(&candle);
1586                 cmd.current_dir(&exe)
1587                     .arg("-nologo")
1588                     .arg("-dRustcDir=rustc")
1589                     .arg("-dDocsDir=rust-docs")
1590                     .arg("-dCargoDir=cargo")
1591                     .arg("-dStdDir=rust-std")
1592                     .arg("-dAnalysisDir=rust-analysis")
1593                     .arg("-arch").arg(&arch)
1594                     .arg("-out").arg(&output)
1595                     .arg(&input);
1596                 add_env(build, &mut cmd, target);
1597
1598                 if rls_installer.is_some() {
1599                     cmd.arg("-dRlsDir=rls");
1600                 }
1601                 if target.contains("windows-gnu") {
1602                     cmd.arg("-dGccDir=rust-mingw");
1603                 }
1604                 build.run(&mut cmd);
1605             };
1606             candle(&xform(&etc.join("msi/rust.wxs")));
1607             candle(&etc.join("msi/ui.wxs"));
1608             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1609             candle("RustcGroup.wxs".as_ref());
1610             candle("DocsGroup.wxs".as_ref());
1611             candle("CargoGroup.wxs".as_ref());
1612             candle("StdGroup.wxs".as_ref());
1613             if rls_installer.is_some() {
1614                 candle("RlsGroup.wxs".as_ref());
1615             }
1616             candle("AnalysisGroup.wxs".as_ref());
1617
1618             if target.contains("windows-gnu") {
1619                 candle("GccGroup.wxs".as_ref());
1620             }
1621
1622             t!(t!(File::create(exe.join("LICENSE.rtf"))).write_all(rtf.as_bytes()));
1623             install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1624             install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1625
1626             let filename = format!("{}-{}.msi", pkgname(build, "rust"), target);
1627             let mut cmd = Command::new(&light);
1628             cmd.arg("-nologo")
1629                 .arg("-ext").arg("WixUIExtension")
1630                 .arg("-ext").arg("WixUtilExtension")
1631                 .arg("-out").arg(exe.join(&filename))
1632                 .arg("rust.wixobj")
1633                 .arg("ui.wixobj")
1634                 .arg("rustwelcomedlg.wixobj")
1635                 .arg("RustcGroup.wixobj")
1636                 .arg("DocsGroup.wixobj")
1637                 .arg("CargoGroup.wixobj")
1638                 .arg("StdGroup.wixobj")
1639                 .arg("AnalysisGroup.wixobj")
1640                 .current_dir(&exe);
1641
1642             if rls_installer.is_some() {
1643                 cmd.arg("RlsGroup.wixobj");
1644             }
1645
1646             if target.contains("windows-gnu") {
1647                 cmd.arg("GccGroup.wixobj");
1648             }
1649             // ICE57 wrongly complains about the shortcuts
1650             cmd.arg("-sice:ICE57");
1651
1652             build.run(&mut cmd);
1653
1654             t!(fs::rename(exe.join(&filename), distdir(build).join(&filename)));
1655         }
1656     }
1657 }
1658
1659 fn add_env(build: &Build, cmd: &mut Command, target: Interned<String>) {
1660     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1661     cmd.env("CFG_RELEASE_INFO", build.rust_version())
1662        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1663        .env("CFG_RELEASE", build.rust_release())
1664        .env("CFG_VER_MAJOR", parts.next().unwrap())
1665        .env("CFG_VER_MINOR", parts.next().unwrap())
1666        .env("CFG_VER_PATCH", parts.next().unwrap())
1667        .env("CFG_VER_BUILD", "0") // just needed to build
1668        .env("CFG_PACKAGE_VERS", build.rust_package_vers())
1669        .env("CFG_PACKAGE_NAME", pkgname(build, "rust"))
1670        .env("CFG_BUILD", target)
1671        .env("CFG_CHANNEL", &build.config.channel);
1672
1673     if target.contains("windows-gnu") {
1674        cmd.env("CFG_MINGW", "1")
1675           .env("CFG_ABI", "GNU");
1676     } else {
1677        cmd.env("CFG_MINGW", "0")
1678           .env("CFG_ABI", "MSVC");
1679     }
1680
1681     if target.contains("x86_64") {
1682        cmd.env("CFG_PLATFORM", "x64");
1683     } else {
1684        cmd.env("CFG_PLATFORM", "x86");
1685     }
1686 }
1687
1688 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
1689 pub struct HashSign;
1690
1691 impl Step for HashSign {
1692     type Output = ();
1693     const ONLY_BUILD_TARGETS: bool = true;
1694     const ONLY_HOSTS: bool = true;
1695     const ONLY_BUILD: bool = true;
1696
1697     fn should_run(run: ShouldRun) -> ShouldRun {
1698         run.path("hash-and-sign")
1699     }
1700
1701     fn make_run(run: RunConfig) {
1702         run.builder.ensure(HashSign);
1703     }
1704
1705     fn run(self, builder: &Builder) {
1706         let build = builder.build;
1707         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1708         let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1709             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1710         });
1711         let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1712             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1713         });
1714         let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1715             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1716         });
1717         let mut pass = String::new();
1718         t!(t!(File::open(&file)).read_to_string(&mut pass));
1719
1720         let today = output(Command::new("date").arg("+%Y-%m-%d"));
1721
1722         cmd.arg(sign);
1723         cmd.arg(distdir(build));
1724         cmd.arg(today.trim());
1725         cmd.arg(build.rust_package_vers());
1726         cmd.arg(build.package_vers(&build.release_num("cargo")));
1727         cmd.arg(build.package_vers(&build.release_num("rls")));
1728         cmd.arg(build.package_vers(&build.release_num("rustfmt")));
1729         cmd.arg(addr);
1730
1731         t!(fs::create_dir_all(distdir(build)));
1732
1733         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
1734         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
1735         let status = t!(child.wait());
1736         assert!(status.success());
1737     }
1738 }