src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::output;
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{libdir, is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time;
  27
  28 pub fn pkgname(builder: &Builder, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub stage: u32,
  72     pub host: Interned<String>,
  73 }
  74
  75 impl Step for Docs {
  76     type Output = PathBuf;
  77     const DEFAULT: bool = true;
  78
  79     fn should_run(run: ShouldRun) -> ShouldRun {
  80         run.path("src/doc")
  81     }
  82
  83     fn make_run(run: RunConfig) {
  84         run.builder.ensure(Docs {
  85             stage: run.builder.top_stage,
  86             host: run.target,
  87         });
  88     }
  89
  90     /// Builds the `rust-docs` installer component.
  91     fn run(self, builder: &Builder) -> PathBuf {
  92         let host = self.host;
  93
  94         let name = pkgname(builder, "rust-docs");
  95
  96         builder.info(&format!("Dist docs ({})", host));
  97         if !builder.config.docs {
  98             builder.info("\tskipping - docs disabled");
  99             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 100         }
 101
 102         builder.default_doc(None);
 103
 104         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 105         let _ = fs::remove_dir_all(&image);
 106
 107         let dst = image.join("share/doc/rust/html");
 108         t!(fs::create_dir_all(&dst));
 109         let src = builder.doc_out(host);
 110         builder.cp_r(&src, &dst);
 111
 112         let mut cmd = rust_installer(builder);
 113         cmd.arg("generate")
 114            .arg("--product-name=Rust-Documentation")
 115            .arg("--rel-manifest-dir=rustlib")
 116            .arg("--success-message=Rust-documentation-is-installed.")
 117            .arg("--image-dir").arg(&image)
 118            .arg("--work-dir").arg(&tmpdir(builder))
 119            .arg("--output-dir").arg(&distdir(builder))
 120            .arg(format!("--package-name={}-{}", name, host))
 121            .arg("--component-name=rust-docs")
 122            .arg("--legacy-manifest-dirs=rustlib,cargo")
 123            .arg("--bulk-dirs=share/doc/rust/html");
 124         builder.run(&mut cmd);
 125         builder.remove_dir(&image);
 126
 127         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 128     }
 129 }
 130
 131 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 132 pub struct RustcDocs {
 133     pub stage: u32,
 134     pub host: Interned<String>,
 135 }
 136
 137 impl Step for RustcDocs {
 138     type Output = PathBuf;
 139     const DEFAULT: bool = true;
 140
 141     fn should_run(run: ShouldRun) -> ShouldRun {
 142         run.path("src/librustc")
 143     }
 144
 145     fn make_run(run: RunConfig) {
 146         run.builder.ensure(RustcDocs {
 147             stage: run.builder.top_stage,
 148             host: run.target,
 149         });
 150     }
 151
 152     /// Builds the `rustc-docs` installer component.
 153     fn run(self, builder: &Builder) -> PathBuf {
 154         let host = self.host;
 155
 156         let name = pkgname(builder, "rustc-docs");
 157
 158         builder.info(&format!("Dist compiler docs ({})", host));
 159         if !builder.config.compiler_docs {
 160             builder.info("\tskipping - compiler docs disabled");
 161             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 162         }
 163
 164         builder.default_doc(None);
 165
 166         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 167         let _ = fs::remove_dir_all(&image);
 168
 169         let dst = image.join("share/doc/rust/html");
 170         t!(fs::create_dir_all(&dst));
 171         let src = builder.compiler_doc_out(host);
 172         builder.cp_r(&src, &dst);
 173
 174         let mut cmd = rust_installer(builder);
 175         cmd.arg("generate")
 176            .arg("--product-name=Rustc-Documentation")
 177            .arg("--rel-manifest-dir=rustlib")
 178            .arg("--success-message=Rustc-documentation-is-installed.")
 179            .arg("--image-dir").arg(&image)
 180            .arg("--work-dir").arg(&tmpdir(builder))
 181            .arg("--output-dir").arg(&distdir(builder))
 182            .arg(format!("--package-name={}-{}", name, host))
 183            .arg("--component-name=rustc-docs")
 184            .arg("--legacy-manifest-dirs=rustlib,cargo")
 185            .arg("--bulk-dirs=share/doc/rust/html");
 186         builder.run(&mut cmd);
 187         builder.remove_dir(&image);
 188
 189         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 190     }
 191 }
 192
 193 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 194     let mut found = Vec::with_capacity(files.len());
 195
 196     for file in files {
 197         let file_path =
 198             path.iter()
 199                 .map(|dir| dir.join(file))
 200                 .find(|p| p.exists());
 201
 202         if let Some(file_path) = file_path {
 203             found.push(file_path);
 204         } else {
 205             panic!("Could not find '{}' in {:?}", file, path);
 206         }
 207     }
 208
 209     found
 210 }
 211
 212 fn make_win_dist(
 213     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder
 214 ) {
 215     //Ask gcc where it keeps its stuff
 216     let mut cmd = Command::new(builder.cc(target_triple));
 217     cmd.arg("-print-search-dirs");
 218     let gcc_out = output(&mut cmd);
 219
 220     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 221     let mut lib_path = Vec::new();
 222
 223     for line in gcc_out.lines() {
 224         let idx = line.find(':').unwrap();
 225         let key = &line[..idx];
 226         let trim_chars: &[_] = &[' ', '='];
 227         let value =
 228             line[(idx + 1)..]
 229                 .trim_start_matches(trim_chars)
 230                 .split(';')
 231                 .map(PathBuf::from);
 232
 233         if key == "programs" {
 234             bin_path.extend(value);
 235         } else if key == "libraries" {
 236             lib_path.extend(value);
 237         }
 238     }
 239
 240     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 241     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 242     if target_triple.starts_with("i686-") {
 243         rustc_dlls.push("libgcc_s_dw2-1.dll");
 244     } else {
 245         rustc_dlls.push("libgcc_s_seh-1.dll");
 246     }
 247
 248     let target_libs = [ //MinGW libs
 249         "libgcc.a",
 250         "libgcc_eh.a",
 251         "libgcc_s.a",
 252         "libm.a",
 253         "libmingw32.a",
 254         "libmingwex.a",
 255         "libstdc++.a",
 256         "libiconv.a",
 257         "libmoldname.a",
 258         "libpthread.a",
 259         //Windows import libs
 260         "libadvapi32.a",
 261         "libbcrypt.a",
 262         "libcomctl32.a",
 263         "libcomdlg32.a",
 264         "libcredui.a",
 265         "libcrypt32.a",
 266         "libdbghelp.a",
 267         "libgdi32.a",
 268         "libimagehlp.a",
 269         "libiphlpapi.a",
 270         "libkernel32.a",
 271         "libmsimg32.a",
 272         "libmsvcrt.a",
 273         "libodbc32.a",
 274         "libole32.a",
 275         "liboleaut32.a",
 276         "libopengl32.a",
 277         "libpsapi.a",
 278         "librpcrt4.a",
 279         "libsecur32.a",
 280         "libsetupapi.a",
 281         "libshell32.a",
 282         "libsynchronization.a",
 283         "libuser32.a",
 284         "libuserenv.a",
 285         "libuuid.a",
 286         "libwinhttp.a",
 287         "libwinmm.a",
 288         "libwinspool.a",
 289         "libws2_32.a",
 290         "libwsock32.a",
 291     ];
 292
 293     //Find mingw artifacts we want to bundle
 294     let target_tools = find_files(&target_tools, &bin_path);
 295     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 296     let target_libs = find_files(&target_libs, &lib_path);
 297
 298     // Copy runtime dlls next to rustc.exe
 299     let dist_bin_dir = rust_root.join("bin/");
 300     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 301     for src in rustc_dlls {
 302         builder.copy_to_folder(&src, &dist_bin_dir);
 303     }
 304
 305     //Copy platform tools to platform-specific bin directory
 306     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 307     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 308     for src in target_tools {
 309         builder.copy_to_folder(&src, &target_bin_dir);
 310     }
 311
 312     // Warn windows-gnu users that the bundled GCC cannot compile C files
 313     builder.create(
 314         &target_bin_dir.join("GCC-WARNING.txt"),
 315         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 316          used as a linker. In order to be able to compile projects containing C code use\
 317          the GCC provided by MinGW or Cygwin."
 318     );
 319
 320     //Copy platform libs to platform-specific lib directory
 321     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 322     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 323     for src in target_libs {
 324         builder.copy_to_folder(&src, &target_lib_dir);
 325     }
 326 }
 327
 328 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 329 pub struct Mingw {
 330     pub host: Interned<String>,
 331 }
 332
 333 impl Step for Mingw {
 334     type Output = Option<PathBuf>;
 335     const DEFAULT: bool = true;
 336
 337     fn should_run(run: ShouldRun) -> ShouldRun {
 338         run.never()
 339     }
 340
 341     fn make_run(run: RunConfig) {
 342         run.builder.ensure(Mingw { host: run.target });
 343     }
 344
 345     /// Build the `rust-mingw` installer component.
 346     ///
 347     /// This contains all the bits and pieces to run the MinGW Windows targets
 348     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 349     fn run(self, builder: &Builder) -> Option<PathBuf> {
 350         let host = self.host;
 351
 352         if !host.contains("pc-windows-gnu") {
 353             return None;
 354         }
 355
 356         builder.info(&format!("Dist mingw ({})", host));
 357         let name = pkgname(builder, "rust-mingw");
 358         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 359         let _ = fs::remove_dir_all(&image);
 360         t!(fs::create_dir_all(&image));
 361
 362         // The first argument is a "temporary directory" which is just
 363         // thrown away (this contains the runtime DLLs included in the rustc package
 364         // above) and the second argument is where to place all the MinGW components
 365         // (which is what we want).
 366         make_win_dist(&tmpdir(builder), &image, host, &builder);
 367
 368         let mut cmd = rust_installer(builder);
 369         cmd.arg("generate")
 370            .arg("--product-name=Rust-MinGW")
 371            .arg("--rel-manifest-dir=rustlib")
 372            .arg("--success-message=Rust-MinGW-is-installed.")
 373            .arg("--image-dir").arg(&image)
 374            .arg("--work-dir").arg(&tmpdir(builder))
 375            .arg("--output-dir").arg(&distdir(builder))
 376            .arg(format!("--package-name={}-{}", name, host))
 377            .arg("--component-name=rust-mingw")
 378            .arg("--legacy-manifest-dirs=rustlib,cargo");
 379         builder.run(&mut cmd);
 380         t!(fs::remove_dir_all(&image));
 381         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 382     }
 383 }
 384
 385 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct Rustc {
 387     pub compiler: Compiler,
 388 }
 389
 390 impl Step for Rustc {
 391     type Output = PathBuf;
 392     const DEFAULT: bool = true;
 393     const ONLY_HOSTS: bool = true;
 394
 395     fn should_run(run: ShouldRun) -> ShouldRun {
 396         run.path("src/librustc")
 397     }
 398
 399     fn make_run(run: RunConfig) {
 400         run.builder.ensure(Rustc {
 401             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 402         });
 403     }
 404
 405     /// Creates the `rustc` installer component.
 406     fn run(self, builder: &Builder) -> PathBuf {
 407         let compiler = self.compiler;
 408         let host = self.compiler.host;
 409
 410         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 411         let name = pkgname(builder, "rustc");
 412         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 413         let _ = fs::remove_dir_all(&image);
 414         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 415         let _ = fs::remove_dir_all(&overlay);
 416
 417         // Prepare the rustc "image", what will actually end up getting installed
 418         prepare_image(builder, compiler, &image);
 419
 420         // Prepare the overlay which is part of the tarball but won't actually be
 421         // installed
 422         let cp = |file: &str| {
 423             builder.install(&builder.src.join(file), &overlay, 0o644);
 424         };
 425         cp("COPYRIGHT");
 426         cp("LICENSE-APACHE");
 427         cp("LICENSE-MIT");
 428         cp("README.md");
 429         // tiny morsel of metadata is used by rust-packaging
 430         let version = builder.rust_version();
 431         builder.create(&overlay.join("version"), &version);
 432         if let Some(sha) = builder.rust_sha() {
 433             builder.create(&overlay.join("git-commit-hash"), &sha);
 434         }
 435
 436         // On MinGW we've got a few runtime DLL dependencies that we need to
 437         // include. The first argument to this script is where to put these DLLs
 438         // (the image we're creating), and the second argument is a junk directory
 439         // to ignore all other MinGW stuff the script creates.
 440         //
 441         // On 32-bit MinGW we're always including a DLL which needs some extra
 442         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 443         // anything requiring us to distribute a license, but it's likely the
 444         // install will *also* include the rust-mingw package, which also needs
 445         // licenses, so to be safe we just include it here in all MinGW packages.
 446         if host.contains("pc-windows-gnu") {
 447             make_win_dist(&image, &tmpdir(builder), host, builder);
 448
 449             let dst = image.join("share/doc");
 450             t!(fs::create_dir_all(&dst));
 451             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 452         }
 453
 454         // Finally, wrap everything up in a nice tarball!
 455         let mut cmd = rust_installer(builder);
 456         cmd.arg("generate")
 457            .arg("--product-name=Rust")
 458            .arg("--rel-manifest-dir=rustlib")
 459            .arg("--success-message=Rust-is-ready-to-roll.")
 460            .arg("--image-dir").arg(&image)
 461            .arg("--work-dir").arg(&tmpdir(builder))
 462            .arg("--output-dir").arg(&distdir(builder))
 463            .arg("--non-installed-overlay").arg(&overlay)
 464            .arg(format!("--package-name={}-{}", name, host))
 465            .arg("--component-name=rustc")
 466            .arg("--legacy-manifest-dirs=rustlib,cargo");
 467         builder.run(&mut cmd);
 468         builder.remove_dir(&image);
 469         builder.remove_dir(&overlay);
 470
 471         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 472
 473         fn prepare_image(builder: &Builder, compiler: Compiler, image: &Path) {
 474             let host = compiler.host;
 475             let src = builder.sysroot(compiler);
 476             let libdir = libdir(&host);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 483
 484             // Copy runtime DLLs needed by the compiler
 485             if libdir != "bin" {
 486                 for entry in builder.read_dir(&src.join(libdir)) {
 487                     let name = entry.file_name();
 488                     if let Some(s) = name.to_str() {
 489                         if is_dylib(s) {
 490                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 499             let backends_dst = image.join(&backends_rel);
 500             t!(fs::create_dir_all(&backends_dst));
 501             builder.cp_r(&backends_src, &backends_dst);
 502
 503             // Copy libLLVM.so to the lib dir as well, if needed. While not
 504             // technically needed by rustc itself it's needed by lots of other
 505             // components like the llvm tools and LLD. LLD is included below and
 506             // tools/LLDB come later, so let's just throw it in the rustc
 507             // component for now.
 508             maybe_install_llvm_dylib(builder, host, image);
 509
 510             // Copy over lld if it's there
 511             if builder.config.lld_enabled {
 512                 let exe = exe("rust-lld", &compiler.host);
 513                 let src = builder.sysroot_libdir(compiler, host)
 514                     .parent()
 515                     .unwrap()
 516                     .join("bin")
 517                     .join(&exe);
 518                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 519                 let dst = image.join("lib/rustlib")
 520                     .join(&*host)
 521                     .join("bin")
 522                     .join(&exe);
 523                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 524                 builder.copy(&src, &dst);
 525             }
 526
 527             // Man pages
 528             t!(fs::create_dir_all(image.join("share/man/man1")));
 529             let man_src = builder.src.join("src/doc/man");
 530             let man_dst = image.join("share/man/man1");
 531             let month_year = t!(time::strftime("%B %Y", &time::now()));
 532             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 533             // to hardlink, and we don't want to edit the source templates
 534             for file_entry in builder.read_dir(&man_src) {
 535                 let page_src = file_entry.path();
 536                 let page_dst = man_dst.join(file_entry.file_name());
 537                 t!(fs::copy(&page_src, &page_dst));
 538                 // template in month/year and version number
 539                 builder.replace_in_file(&page_dst,
 540                                 &[("<INSERT DATE HERE>", &month_year),
 541                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 542             }
 543
 544             // Debugger scripts
 545             builder.ensure(DebuggerScripts {
 546                 sysroot: INTERNER.intern_path(image.to_owned()),
 547                 host,
 548             });
 549
 550             // Misc license info
 551             let cp = |file: &str| {
 552                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 553             };
 554             cp("COPYRIGHT");
 555             cp("LICENSE-APACHE");
 556             cp("LICENSE-MIT");
 557             cp("README.md");
 558         }
 559     }
 560 }
 561
 562 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 563 pub struct DebuggerScripts {
 564     pub sysroot: Interned<PathBuf>,
 565     pub host: Interned<String>,
 566 }
 567
 568 impl Step for DebuggerScripts {
 569     type Output = ();
 570
 571     fn should_run(run: ShouldRun) -> ShouldRun {
 572         run.path("src/lldb_batchmode.py")
 573     }
 574
 575     fn make_run(run: RunConfig) {
 576         run.builder.ensure(DebuggerScripts {
 577             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 578             host: run.target,
 579         });
 580     }
 581
 582     /// Copies debugger scripts for `target` into the `sysroot` specified.
 583     fn run(self, builder: &Builder) {
 584         let host = self.host;
 585         let sysroot = self.sysroot;
 586         let dst = sysroot.join("lib/rustlib/etc");
 587         t!(fs::create_dir_all(&dst));
 588         let cp_debugger_script = |file: &str| {
 589             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 590         };
 591         if host.contains("windows-msvc") {
 592             // windbg debugger scripts
 593             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 594                 0o755);
 595
 596             cp_debugger_script("natvis/intrinsic.natvis");
 597             cp_debugger_script("natvis/liballoc.natvis");
 598             cp_debugger_script("natvis/libcore.natvis");
 599         } else {
 600             cp_debugger_script("debugger_pretty_printers_common.py");
 601
 602             // gdb debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 604                     0o755);
 605             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 606                     0o755);
 607
 608             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 609             cp_debugger_script("gdb_rust_pretty_printing.py");
 610
 611             // lldb debugger scripts
 612             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 613                     0o755);
 614
 615             cp_debugger_script("lldb_rust_formatters.py");
 616         }
 617     }
 618 }
 619
 620 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 621 pub struct Std {
 622     pub compiler: Compiler,
 623     pub target: Interned<String>,
 624 }
 625
 626 impl Step for Std {
 627     type Output = PathBuf;
 628     const DEFAULT: bool = true;
 629
 630     fn should_run(run: ShouldRun) -> ShouldRun {
 631         run.path("src/libstd")
 632     }
 633
 634     fn make_run(run: RunConfig) {
 635         run.builder.ensure(Std {
 636             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 637             target: run.target,
 638         });
 639     }
 640
 641     fn run(self, builder: &Builder) -> PathBuf {
 642         let compiler = self.compiler;
 643         let target = self.target;
 644
 645         let name = pkgname(builder, "rust-std");
 646         builder.info(&format!("Dist std stage{} ({} -> {})",
 647             compiler.stage, &compiler.host, target));
 648
 649         // The only true set of target libraries came from the build triple, so
 650         // let's reduce redundant work by only producing archives from that host.
 651         if compiler.host != builder.config.build {
 652             builder.info("\tskipping, not a build host");
 653             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 654         }
 655
 656         // We want to package up as many target libraries as possible
 657         // for the `rust-std` package, so if this is a host target we
 658         // depend on librustc and otherwise we just depend on libtest.
 659         if builder.hosts.iter().any(|t| t == target) {
 660             builder.ensure(compile::Rustc { compiler, target });
 661         } else {
 662             if builder.no_std(target) == Some(true) {
 663                 // the `test` doesn't compile for no-std targets
 664                 builder.ensure(compile::Std { compiler, target });
 665             } else {
 666                 builder.ensure(compile::Test { compiler, target });
 667             }
 668         }
 669
 670         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 671         let _ = fs::remove_dir_all(&image);
 672
 673         let dst = image.join("lib/rustlib").join(target);
 674         t!(fs::create_dir_all(&dst));
 675         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 676         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 677         builder.cp_filtered(&src, &dst, &|path| {
 678             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 679                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 680                     return false
 681                 }
 682                 if name == "bin" {
 683                     return false
 684                 }
 685                 if name.contains("LLVM") {
 686                     return false
 687                 }
 688             }
 689             true
 690         });
 691
 692         let mut cmd = rust_installer(builder);
 693         cmd.arg("generate")
 694            .arg("--product-name=Rust")
 695            .arg("--rel-manifest-dir=rustlib")
 696            .arg("--success-message=std-is-standing-at-the-ready.")
 697            .arg("--image-dir").arg(&image)
 698            .arg("--work-dir").arg(&tmpdir(builder))
 699            .arg("--output-dir").arg(&distdir(builder))
 700            .arg(format!("--package-name={}-{}", name, target))
 701            .arg(format!("--component-name=rust-std-{}", target))
 702            .arg("--legacy-manifest-dirs=rustlib,cargo");
 703         builder.run(&mut cmd);
 704         builder.remove_dir(&image);
 705         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 706     }
 707 }
 708
 709 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 710 pub struct Analysis {
 711     pub compiler: Compiler,
 712     pub target: Interned<String>,
 713 }
 714
 715 impl Step for Analysis {
 716     type Output = PathBuf;
 717     const DEFAULT: bool = true;
 718
 719     fn should_run(run: ShouldRun) -> ShouldRun {
 720         let builder = run.builder;
 721         run.path("analysis").default_condition(builder.config.extended)
 722     }
 723
 724     fn make_run(run: RunConfig) {
 725         run.builder.ensure(Analysis {
 726             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 727             target: run.target,
 728         });
 729     }
 730
 731     /// Creates a tarball of save-analysis metadata, if available.
 732     fn run(self, builder: &Builder) -> PathBuf {
 733         let compiler = self.compiler;
 734         let target = self.target;
 735         assert!(builder.config.extended);
 736         builder.info("Dist analysis");
 737         let name = pkgname(builder, "rust-analysis");
 738
 739         if &compiler.host != builder.config.build {
 740             builder.info("\tskipping, not a build host");
 741             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 742         }
 743
 744         builder.ensure(Std { compiler, target });
 745
 746         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 747         // stage2 artifacts is simply copied from stage1 in that case.
 748         let compiler = if builder.force_use_stage1(compiler, target) {
 749             builder.compiler(1, compiler.host)
 750         } else {
 751             compiler.clone()
 752         };
 753
 754         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 755
 756         let src = builder.stage_out(compiler, Mode::Std)
 757             .join(target).join(builder.cargo_dir()).join("deps");
 758
 759         let image_src = src.join("save-analysis");
 760         let dst = image.join("lib/rustlib").join(target).join("analysis");
 761         t!(fs::create_dir_all(&dst));
 762         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 763         builder.cp_r(&image_src, &dst);
 764
 765         let mut cmd = rust_installer(builder);
 766         cmd.arg("generate")
 767            .arg("--product-name=Rust")
 768            .arg("--rel-manifest-dir=rustlib")
 769            .arg("--success-message=save-analysis-saved.")
 770            .arg("--image-dir").arg(&image)
 771            .arg("--work-dir").arg(&tmpdir(builder))
 772            .arg("--output-dir").arg(&distdir(builder))
 773            .arg(format!("--package-name={}-{}", name, target))
 774            .arg(format!("--component-name=rust-analysis-{}", target))
 775            .arg("--legacy-manifest-dirs=rustlib,cargo");
 776         builder.run(&mut cmd);
 777         builder.remove_dir(&image);
 778         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 779     }
 780 }
 781
 782 fn copy_src_dirs(builder: &Builder, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 783     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 784         let spath = match path.to_str() {
 785             Some(path) => path,
 786             None => return false,
 787         };
 788         if spath.ends_with("~") || spath.ends_with(".pyc") {
 789             return false
 790         }
 791
 792         const LLVM_PROJECTS: &[&str] = &[
 793             "llvm-project/clang", "llvm-project\\clang",
 794             "llvm-project/lld", "llvm-project\\lld",
 795             "llvm-project/lldb", "llvm-project\\lldb",
 796             "llvm-project/llvm", "llvm-project\\llvm",
 797         ];
 798         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 799             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 800         {
 801             return false;
 802         }
 803
 804         const LLVM_TEST: &[&str] = &[
 805             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 806             "llvm-emscripten/test", "llvm-emscripten\\test",
 807         ];
 808         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 809             (spath.ends_with(".ll") ||
 810              spath.ends_with(".td") ||
 811              spath.ends_with(".s")) {
 812             return false
 813         }
 814         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 815             return false
 816         }
 817
 818         let full_path = Path::new(dir).join(path);
 819         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 820             return false;
 821         }
 822
 823         let excludes = [
 824             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 825             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 826             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 827             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 828         ];
 829         !path.iter()
 830              .map(|s| s.to_str().unwrap())
 831              .any(|s| excludes.contains(&s))
 832     }
 833
 834     // Copy the directories using our filter
 835     for item in src_dirs {
 836         let dst = &dst_dir.join(item);
 837         t!(fs::create_dir_all(dst));
 838         builder.cp_filtered(
 839             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 840     }
 841 }
 842
 843 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 844 pub struct Src;
 845
 846 impl Step for Src {
 847     /// The output path of the src installer tarball
 848     type Output = PathBuf;
 849     const DEFAULT: bool = true;
 850     const ONLY_HOSTS: bool = true;
 851
 852     fn should_run(run: ShouldRun) -> ShouldRun {
 853         run.path("src")
 854     }
 855
 856     fn make_run(run: RunConfig) {
 857         run.builder.ensure(Src);
 858     }
 859
 860     /// Creates the `rust-src` installer component
 861     fn run(self, builder: &Builder) -> PathBuf {
 862         builder.info("Dist src");
 863
 864         let name = pkgname(builder, "rust-src");
 865         let image = tmpdir(builder).join(format!("{}-image", name));
 866         let _ = fs::remove_dir_all(&image);
 867
 868         let dst = image.join("lib/rustlib/src");
 869         let dst_src = dst.join("rust");
 870         t!(fs::create_dir_all(&dst_src));
 871
 872         let src_files = [
 873             "Cargo.lock",
 874         ];
 875         // This is the reduced set of paths which will become the rust-src component
 876         // (essentially libstd and all of its path dependencies)
 877         let std_src_dirs = [
 878             "src/build_helper",
 879             "src/liballoc",
 880             "src/libcore",
 881             "src/libpanic_abort",
 882             "src/libpanic_unwind",
 883             "src/librustc_asan",
 884             "src/librustc_lsan",
 885             "src/librustc_msan",
 886             "src/librustc_tsan",
 887             "src/libstd",
 888             "src/libunwind",
 889             "src/libtest",
 890             "src/libterm",
 891             "src/libprofiler_builtins",
 892             "src/stdsimd",
 893             "src/libproc_macro",
 894             "src/tools/rustc-std-workspace-core",
 895         ];
 896
 897         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 898         for file in src_files.iter() {
 899             builder.copy(&builder.src.join(file), &dst_src.join(file));
 900         }
 901
 902         // Create source tarball in rust-installer format
 903         let mut cmd = rust_installer(builder);
 904         cmd.arg("generate")
 905            .arg("--product-name=Rust")
 906            .arg("--rel-manifest-dir=rustlib")
 907            .arg("--success-message=Awesome-Source.")
 908            .arg("--image-dir").arg(&image)
 909            .arg("--work-dir").arg(&tmpdir(builder))
 910            .arg("--output-dir").arg(&distdir(builder))
 911            .arg(format!("--package-name={}", name))
 912            .arg("--component-name=rust-src")
 913            .arg("--legacy-manifest-dirs=rustlib,cargo");
 914         builder.run(&mut cmd);
 915
 916         builder.remove_dir(&image);
 917         distdir(builder).join(&format!("{}.tar.gz", name))
 918     }
 919 }
 920
 921 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 922
 923 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 924 pub struct PlainSourceTarball;
 925
 926 impl Step for PlainSourceTarball {
 927     /// Produces the location of the tarball generated
 928     type Output = PathBuf;
 929     const DEFAULT: bool = true;
 930     const ONLY_HOSTS: bool = true;
 931
 932     fn should_run(run: ShouldRun) -> ShouldRun {
 933         let builder = run.builder;
 934         run.path("src").default_condition(builder.config.rust_dist_src)
 935     }
 936
 937     fn make_run(run: RunConfig) {
 938         run.builder.ensure(PlainSourceTarball);
 939     }
 940
 941     /// Creates the plain source tarball
 942     fn run(self, builder: &Builder) -> PathBuf {
 943         builder.info("Create plain source tarball");
 944
 945         // Make sure that the root folder of tarball has the correct name
 946         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 947         let plain_dst_src = tmpdir(builder).join(&plain_name);
 948         let _ = fs::remove_dir_all(&plain_dst_src);
 949         t!(fs::create_dir_all(&plain_dst_src));
 950
 951         // This is the set of root paths which will become part of the source package
 952         let src_files = [
 953             "COPYRIGHT",
 954             "LICENSE-APACHE",
 955             "LICENSE-MIT",
 956             "CONTRIBUTING.md",
 957             "README.md",
 958             "RELEASES.md",
 959             "configure",
 960             "x.py",
 961             "config.toml.example",
 962             "Cargo.toml",
 963             "Cargo.lock",
 964         ];
 965         let src_dirs = [
 966             "src",
 967         ];
 968
 969         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 970
 971         // Copy the files normally
 972         for item in &src_files {
 973             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 974         }
 975
 976         // Create the version file
 977         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 978         if let Some(sha) = builder.rust_sha() {
 979             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 980         }
 981
 982         // If we're building from git sources, we need to vendor a complete distribution.
 983         if builder.rust_info.is_git() {
 984             // Get cargo-vendor installed, if it isn't already.
 985             let mut has_cargo_vendor = false;
 986             let mut cmd = Command::new(&builder.initial_cargo);
 987             for line in output(cmd.arg("install").arg("--list")).lines() {
 988                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
 989             }
 990             if !has_cargo_vendor {
 991                 let mut cmd = builder.cargo(
 992                     builder.compiler(0, builder.config.build),
 993                     Mode::ToolBootstrap,
 994                     builder.config.build,
 995                     "install"
 996                 );
 997                 cmd.arg("--force")
 998                    .arg("--debug")
 999                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1000                    .arg("cargo-vendor");
1001                 builder.run(&mut cmd);
1002             }
1003
1004             // Vendor all Cargo dependencies
1005             let mut cmd = Command::new(&builder.initial_cargo);
1006             cmd.arg("vendor")
1007                .current_dir(&plain_dst_src);
1008             builder.run(&mut cmd);
1009         }
1010
1011         // Create plain source tarball
1012         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1013         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1014         tarball.set_extension(""); // strip .gz
1015         tarball.set_extension(""); // strip .tar
1016         if let Some(dir) = tarball.parent() {
1017             builder.create_dir(&dir);
1018         }
1019         builder.info("running installer");
1020         let mut cmd = rust_installer(builder);
1021         cmd.arg("tarball")
1022            .arg("--input").arg(&plain_name)
1023            .arg("--output").arg(&tarball)
1024            .arg("--work-dir=.")
1025            .current_dir(tmpdir(builder));
1026         builder.run(&mut cmd);
1027         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1028     }
1029 }
1030
1031 // We have to run a few shell scripts, which choke quite a bit on both `\`
1032 // characters and on `C:\` paths, so normalize both of them away.
1033 pub fn sanitize_sh(path: &Path) -> String {
1034     let path = path.to_str().unwrap().replace("\\", "/");
1035     return change_drive(&path).unwrap_or(path);
1036
1037     fn change_drive(s: &str) -> Option<String> {
1038         let mut ch = s.chars();
1039         let drive = ch.next().unwrap_or('C');
1040         if ch.next() != Some(':') {
1041             return None
1042         }
1043         if ch.next() != Some('/') {
1044             return None
1045         }
1046         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1047     }
1048 }
1049
1050 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1051 pub struct Cargo {
1052     pub stage: u32,
1053     pub target: Interned<String>,
1054 }
1055
1056 impl Step for Cargo {
1057     type Output = PathBuf;
1058     const ONLY_HOSTS: bool = true;
1059
1060     fn should_run(run: ShouldRun) -> ShouldRun {
1061         run.path("cargo")
1062     }
1063
1064     fn make_run(run: RunConfig) {
1065         run.builder.ensure(Cargo {
1066             stage: run.builder.top_stage,
1067             target: run.target,
1068         });
1069     }
1070
1071     fn run(self, builder: &Builder) -> PathBuf {
1072         let stage = self.stage;
1073         let target = self.target;
1074
1075         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1076         let src = builder.src.join("src/tools/cargo");
1077         let etc = src.join("src/etc");
1078         let release_num = builder.release_num("cargo");
1079         let name = pkgname(builder, "cargo");
1080         let version = builder.cargo_info.version(builder, &release_num);
1081
1082         let tmp = tmpdir(builder);
1083         let image = tmp.join("cargo-image");
1084         drop(fs::remove_dir_all(&image));
1085         builder.create_dir(&image);
1086
1087         // Prepare the image directory
1088         builder.create_dir(&image.join("share/zsh/site-functions"));
1089         builder.create_dir(&image.join("etc/bash_completion.d"));
1090         let cargo = builder.ensure(tool::Cargo {
1091             compiler: builder.compiler(stage, builder.config.build),
1092             target
1093         });
1094         builder.install(&cargo, &image.join("bin"), 0o755);
1095         for man in t!(etc.join("man").read_dir()) {
1096             let man = t!(man);
1097             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1098         }
1099         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1100         builder.copy(&etc.join("cargo.bashcomp.sh"),
1101              &image.join("etc/bash_completion.d/cargo"));
1102         let doc = image.join("share/doc/cargo");
1103         builder.install(&src.join("README.md"), &doc, 0o644);
1104         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1105         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1106         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1107
1108         // Prepare the overlay
1109         let overlay = tmp.join("cargo-overlay");
1110         drop(fs::remove_dir_all(&overlay));
1111         builder.create_dir(&overlay);
1112         builder.install(&src.join("README.md"), &overlay, 0o644);
1113         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1114         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1115         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1116         builder.create(&overlay.join("version"), &version);
1117
1118         // Generate the installer tarball
1119         let mut cmd = rust_installer(builder);
1120         cmd.arg("generate")
1121            .arg("--product-name=Rust")
1122            .arg("--rel-manifest-dir=rustlib")
1123            .arg("--success-message=Rust-is-ready-to-roll.")
1124            .arg("--image-dir").arg(&image)
1125            .arg("--work-dir").arg(&tmpdir(builder))
1126            .arg("--output-dir").arg(&distdir(builder))
1127            .arg("--non-installed-overlay").arg(&overlay)
1128            .arg(format!("--package-name={}-{}", name, target))
1129            .arg("--component-name=cargo")
1130            .arg("--legacy-manifest-dirs=rustlib,cargo");
1131         builder.run(&mut cmd);
1132         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1133     }
1134 }
1135
1136 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1137 pub struct Rls {
1138     pub stage: u32,
1139     pub target: Interned<String>,
1140 }
1141
1142 impl Step for Rls {
1143     type Output = Option<PathBuf>;
1144     const ONLY_HOSTS: bool = true;
1145
1146     fn should_run(run: ShouldRun) -> ShouldRun {
1147         run.path("rls")
1148     }
1149
1150     fn make_run(run: RunConfig) {
1151         run.builder.ensure(Rls {
1152             stage: run.builder.top_stage,
1153             target: run.target,
1154         });
1155     }
1156
1157     fn run(self, builder: &Builder) -> Option<PathBuf> {
1158         let stage = self.stage;
1159         let target = self.target;
1160         assert!(builder.config.extended);
1161
1162         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1163         let src = builder.src.join("src/tools/rls");
1164         let release_num = builder.release_num("rls");
1165         let name = pkgname(builder, "rls");
1166         let version = builder.rls_info.version(builder, &release_num);
1167
1168         let tmp = tmpdir(builder);
1169         let image = tmp.join("rls-image");
1170         drop(fs::remove_dir_all(&image));
1171         t!(fs::create_dir_all(&image));
1172
1173         // Prepare the image directory
1174         // We expect RLS to build, because we've exited this step above if tool
1175         // state for RLS isn't testing.
1176         let rls = builder.ensure(tool::Rls {
1177             compiler: builder.compiler(stage, builder.config.build),
1178             target, extra_features: Vec::new()
1179         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1180
1181         builder.install(&rls, &image.join("bin"), 0o755);
1182         let doc = image.join("share/doc/rls");
1183         builder.install(&src.join("README.md"), &doc, 0o644);
1184         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1185         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1186
1187         // Prepare the overlay
1188         let overlay = tmp.join("rls-overlay");
1189         drop(fs::remove_dir_all(&overlay));
1190         t!(fs::create_dir_all(&overlay));
1191         builder.install(&src.join("README.md"), &overlay, 0o644);
1192         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1193         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1194         builder.create(&overlay.join("version"), &version);
1195
1196         // Generate the installer tarball
1197         let mut cmd = rust_installer(builder);
1198         cmd.arg("generate")
1199            .arg("--product-name=Rust")
1200            .arg("--rel-manifest-dir=rustlib")
1201            .arg("--success-message=RLS-ready-to-serve.")
1202            .arg("--image-dir").arg(&image)
1203            .arg("--work-dir").arg(&tmpdir(builder))
1204            .arg("--output-dir").arg(&distdir(builder))
1205            .arg("--non-installed-overlay").arg(&overlay)
1206            .arg(format!("--package-name={}-{}", name, target))
1207            .arg("--legacy-manifest-dirs=rustlib,cargo")
1208            .arg("--component-name=rls-preview");
1209
1210         builder.run(&mut cmd);
1211         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1212     }
1213 }
1214
1215 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1216 pub struct Clippy {
1217     pub stage: u32,
1218     pub target: Interned<String>,
1219 }
1220
1221 impl Step for Clippy {
1222     type Output = Option<PathBuf>;
1223     const ONLY_HOSTS: bool = true;
1224
1225     fn should_run(run: ShouldRun) -> ShouldRun {
1226         run.path("clippy")
1227     }
1228
1229     fn make_run(run: RunConfig) {
1230         run.builder.ensure(Clippy {
1231             stage: run.builder.top_stage,
1232             target: run.target,
1233         });
1234     }
1235
1236     fn run(self, builder: &Builder) -> Option<PathBuf> {
1237         let stage = self.stage;
1238         let target = self.target;
1239         assert!(builder.config.extended);
1240
1241         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1242         let src = builder.src.join("src/tools/clippy");
1243         let release_num = builder.release_num("clippy");
1244         let name = pkgname(builder, "clippy");
1245         let version = builder.clippy_info.version(builder, &release_num);
1246
1247         let tmp = tmpdir(builder);
1248         let image = tmp.join("clippy-image");
1249         drop(fs::remove_dir_all(&image));
1250         builder.create_dir(&image);
1251
1252         // Prepare the image directory
1253         // We expect clippy to build, because we've exited this step above if tool
1254         // state for clippy isn't testing.
1255         let clippy = builder.ensure(tool::Clippy {
1256             compiler: builder.compiler(stage, builder.config.build),
1257             target, extra_features: Vec::new()
1258         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1259         let cargoclippy = builder.ensure(tool::CargoClippy {
1260             compiler: builder.compiler(stage, builder.config.build),
1261             target, extra_features: Vec::new()
1262         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1263
1264         builder.install(&clippy, &image.join("bin"), 0o755);
1265         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1266         let doc = image.join("share/doc/clippy");
1267         builder.install(&src.join("README.md"), &doc, 0o644);
1268         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1269         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1270
1271         // Prepare the overlay
1272         let overlay = tmp.join("clippy-overlay");
1273         drop(fs::remove_dir_all(&overlay));
1274         t!(fs::create_dir_all(&overlay));
1275         builder.install(&src.join("README.md"), &overlay, 0o644);
1276         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1277         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1278         builder.create(&overlay.join("version"), &version);
1279
1280         // Generate the installer tarball
1281         let mut cmd = rust_installer(builder);
1282         cmd.arg("generate")
1283            .arg("--product-name=Rust")
1284            .arg("--rel-manifest-dir=rustlib")
1285            .arg("--success-message=clippy-ready-to-serve.")
1286            .arg("--image-dir").arg(&image)
1287            .arg("--work-dir").arg(&tmpdir(builder))
1288            .arg("--output-dir").arg(&distdir(builder))
1289            .arg("--non-installed-overlay").arg(&overlay)
1290            .arg(format!("--package-name={}-{}", name, target))
1291            .arg("--legacy-manifest-dirs=rustlib,cargo")
1292            .arg("--component-name=clippy-preview");
1293
1294         builder.run(&mut cmd);
1295         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1296     }
1297 }
1298
1299 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1300 pub struct Miri {
1301     pub stage: u32,
1302     pub target: Interned<String>,
1303 }
1304
1305 impl Step for Miri {
1306     type Output = Option<PathBuf>;
1307     const ONLY_HOSTS: bool = true;
1308
1309     fn should_run(run: ShouldRun) -> ShouldRun {
1310         run.path("miri")
1311     }
1312
1313     fn make_run(run: RunConfig) {
1314         run.builder.ensure(Miri {
1315             stage: run.builder.top_stage,
1316             target: run.target,
1317         });
1318     }
1319
1320     fn run(self, builder: &Builder) -> Option<PathBuf> {
1321         let stage = self.stage;
1322         let target = self.target;
1323         assert!(builder.config.extended);
1324
1325         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1326         let src = builder.src.join("src/tools/miri");
1327         let release_num = builder.release_num("miri");
1328         let name = pkgname(builder, "miri");
1329         let version = builder.miri_info.version(builder, &release_num);
1330
1331         let tmp = tmpdir(builder);
1332         let image = tmp.join("miri-image");
1333         drop(fs::remove_dir_all(&image));
1334         builder.create_dir(&image);
1335
1336         // Prepare the image directory
1337         // We expect miri to build, because we've exited this step above if tool
1338         // state for miri isn't testing.
1339         let miri = builder.ensure(tool::Miri {
1340             compiler: builder.compiler(stage, builder.config.build),
1341             target, extra_features: Vec::new()
1342         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1343         let cargomiri = builder.ensure(tool::CargoMiri {
1344             compiler: builder.compiler(stage, builder.config.build),
1345             target, extra_features: Vec::new()
1346         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1347
1348         builder.install(&miri, &image.join("bin"), 0o755);
1349         builder.install(&cargomiri, &image.join("bin"), 0o755);
1350         let doc = image.join("share/doc/miri");
1351         builder.install(&src.join("README.md"), &doc, 0o644);
1352         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1353         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1354
1355         // Prepare the overlay
1356         let overlay = tmp.join("miri-overlay");
1357         drop(fs::remove_dir_all(&overlay));
1358         t!(fs::create_dir_all(&overlay));
1359         builder.install(&src.join("README.md"), &overlay, 0o644);
1360         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1361         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1362         builder.create(&overlay.join("version"), &version);
1363
1364         // Generate the installer tarball
1365         let mut cmd = rust_installer(builder);
1366         cmd.arg("generate")
1367            .arg("--product-name=Rust")
1368            .arg("--rel-manifest-dir=rustlib")
1369            .arg("--success-message=miri-ready-to-serve.")
1370            .arg("--image-dir").arg(&image)
1371            .arg("--work-dir").arg(&tmpdir(builder))
1372            .arg("--output-dir").arg(&distdir(builder))
1373            .arg("--non-installed-overlay").arg(&overlay)
1374            .arg(format!("--package-name={}-{}", name, target))
1375            .arg("--legacy-manifest-dirs=rustlib,cargo")
1376            .arg("--component-name=miri-preview");
1377
1378         builder.run(&mut cmd);
1379         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1380     }
1381 }
1382
1383 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1384 pub struct Rustfmt {
1385     pub stage: u32,
1386     pub target: Interned<String>,
1387 }
1388
1389 impl Step for Rustfmt {
1390     type Output = Option<PathBuf>;
1391     const ONLY_HOSTS: bool = true;
1392
1393     fn should_run(run: ShouldRun) -> ShouldRun {
1394         run.path("rustfmt")
1395     }
1396
1397     fn make_run(run: RunConfig) {
1398         run.builder.ensure(Rustfmt {
1399             stage: run.builder.top_stage,
1400             target: run.target,
1401         });
1402     }
1403
1404     fn run(self, builder: &Builder) -> Option<PathBuf> {
1405         let stage = self.stage;
1406         let target = self.target;
1407
1408         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1409         let src = builder.src.join("src/tools/rustfmt");
1410         let release_num = builder.release_num("rustfmt");
1411         let name = pkgname(builder, "rustfmt");
1412         let version = builder.rustfmt_info.version(builder, &release_num);
1413
1414         let tmp = tmpdir(builder);
1415         let image = tmp.join("rustfmt-image");
1416         drop(fs::remove_dir_all(&image));
1417         builder.create_dir(&image);
1418
1419         // Prepare the image directory
1420         let rustfmt = builder.ensure(tool::Rustfmt {
1421             compiler: builder.compiler(stage, builder.config.build),
1422             target, extra_features: Vec::new()
1423         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1424         let cargofmt = builder.ensure(tool::Cargofmt {
1425             compiler: builder.compiler(stage, builder.config.build),
1426             target, extra_features: Vec::new()
1427         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1428
1429         builder.install(&rustfmt, &image.join("bin"), 0o755);
1430         builder.install(&cargofmt, &image.join("bin"), 0o755);
1431         let doc = image.join("share/doc/rustfmt");
1432         builder.install(&src.join("README.md"), &doc, 0o644);
1433         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1434         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1435
1436         // Prepare the overlay
1437         let overlay = tmp.join("rustfmt-overlay");
1438         drop(fs::remove_dir_all(&overlay));
1439         builder.create_dir(&overlay);
1440         builder.install(&src.join("README.md"), &overlay, 0o644);
1441         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1442         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1443         builder.create(&overlay.join("version"), &version);
1444
1445         // Generate the installer tarball
1446         let mut cmd = rust_installer(builder);
1447         cmd.arg("generate")
1448            .arg("--product-name=Rust")
1449            .arg("--rel-manifest-dir=rustlib")
1450            .arg("--success-message=rustfmt-ready-to-fmt.")
1451            .arg("--image-dir").arg(&image)
1452            .arg("--work-dir").arg(&tmpdir(builder))
1453            .arg("--output-dir").arg(&distdir(builder))
1454            .arg("--non-installed-overlay").arg(&overlay)
1455            .arg(format!("--package-name={}-{}", name, target))
1456            .arg("--legacy-manifest-dirs=rustlib,cargo")
1457            .arg("--component-name=rustfmt-preview");
1458
1459         builder.run(&mut cmd);
1460         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1461     }
1462 }
1463
1464 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1465 pub struct Extended {
1466     stage: u32,
1467     host: Interned<String>,
1468     target: Interned<String>,
1469 }
1470
1471 impl Step for Extended {
1472     type Output = ();
1473     const DEFAULT: bool = true;
1474     const ONLY_HOSTS: bool = true;
1475
1476     fn should_run(run: ShouldRun) -> ShouldRun {
1477         let builder = run.builder;
1478         run.path("extended").default_condition(builder.config.extended)
1479     }
1480
1481     fn make_run(run: RunConfig) {
1482         run.builder.ensure(Extended {
1483             stage: run.builder.top_stage,
1484             host: run.builder.config.build,
1485             target: run.target,
1486         });
1487     }
1488
1489     /// Creates a combined installer for the specified target in the provided stage.
1490     fn run(self, builder: &Builder) {
1491         let stage = self.stage;
1492         let target = self.target;
1493
1494         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1495
1496         let rustc_installer = builder.ensure(Rustc {
1497             compiler: builder.compiler(stage, target),
1498         });
1499         let cargo_installer = builder.ensure(Cargo { stage, target });
1500         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1501         let rls_installer = builder.ensure(Rls { stage, target });
1502         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1503         let clippy_installer = builder.ensure(Clippy { stage, target });
1504         let miri_installer = builder.ensure(Miri { stage, target });
1505         let lldb_installer = builder.ensure(Lldb { target });
1506         let mingw_installer = builder.ensure(Mingw { host: target });
1507         let analysis_installer = builder.ensure(Analysis {
1508             compiler: builder.compiler(stage, self.host),
1509             target
1510         });
1511
1512         let docs_installer = builder.ensure(Docs { stage, host: target, });
1513         let std_installer = builder.ensure(Std {
1514             compiler: builder.compiler(stage, self.host),
1515             target,
1516         });
1517
1518         let tmp = tmpdir(builder);
1519         let overlay = tmp.join("extended-overlay");
1520         let etc = builder.src.join("src/etc/installer");
1521         let work = tmp.join("work");
1522
1523         let _ = fs::remove_dir_all(&overlay);
1524         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1525         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1526         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1527         let version = builder.rust_version();
1528         builder.create(&overlay.join("version"), &version);
1529         if let Some(sha) = builder.rust_sha() {
1530             builder.create(&overlay.join("git-commit-hash"), &sha);
1531         }
1532         builder.install(&etc.join("README.md"), &overlay, 0o644);
1533
1534         // When rust-std package split from rustc, we needed to ensure that during
1535         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1536         // the std files during uninstall. To do this ensure that rustc comes
1537         // before rust-std in the list below.
1538         let mut tarballs = Vec::new();
1539         tarballs.push(rustc_installer);
1540         tarballs.push(cargo_installer);
1541         tarballs.extend(rls_installer.clone());
1542         tarballs.extend(clippy_installer.clone());
1543         tarballs.extend(miri_installer.clone());
1544         tarballs.extend(rustfmt_installer.clone());
1545         tarballs.extend(llvm_tools_installer);
1546         tarballs.extend(lldb_installer);
1547         tarballs.push(analysis_installer);
1548         tarballs.push(std_installer);
1549         if builder.config.docs {
1550             tarballs.push(docs_installer);
1551         }
1552         if target.contains("pc-windows-gnu") {
1553             tarballs.push(mingw_installer.unwrap());
1554         }
1555         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1556         for tarball in &tarballs[1..] {
1557             input_tarballs.push(",");
1558             input_tarballs.push(tarball);
1559         }
1560
1561         let mut cmd = rust_installer(builder);
1562         cmd.arg("combine")
1563             .arg("--product-name=Rust")
1564             .arg("--rel-manifest-dir=rustlib")
1565             .arg("--success-message=Rust-is-ready-to-roll.")
1566             .arg("--work-dir").arg(&work)
1567             .arg("--output-dir").arg(&distdir(builder))
1568             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1569             .arg("--legacy-manifest-dirs=rustlib,cargo")
1570             .arg("--input-tarballs").arg(input_tarballs)
1571             .arg("--non-installed-overlay").arg(&overlay);
1572         builder.run(&mut cmd);
1573
1574         let mut license = String::new();
1575         license += &builder.read(&builder.src.join("COPYRIGHT"));
1576         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1577         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1578         license.push_str("\n");
1579         license.push_str("\n");
1580
1581         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1582         let mut rtf = rtf.to_string();
1583         rtf.push_str("\n");
1584         for line in license.lines() {
1585             rtf.push_str(line);
1586             rtf.push_str("\\line ");
1587         }
1588         rtf.push_str("}");
1589
1590         fn filter(contents: &str, marker: &str) -> String {
1591             let start = format!("tool-{}-start", marker);
1592             let end = format!("tool-{}-end", marker);
1593             let mut lines = Vec::new();
1594             let mut omitted = false;
1595             for line in contents.lines() {
1596                 if line.contains(&start) {
1597                     omitted = true;
1598                 } else if line.contains(&end) {
1599                     omitted = false;
1600                 } else if !omitted {
1601                     lines.push(line);
1602                 }
1603             }
1604
1605             lines.join("\n")
1606         }
1607
1608         let xform = |p: &Path| {
1609             let mut contents = t!(fs::read_to_string(p));
1610             if rls_installer.is_none() {
1611                 contents = filter(&contents, "rls");
1612             }
1613             if clippy_installer.is_none() {
1614                 contents = filter(&contents, "clippy");
1615             }
1616             if miri_installer.is_none() {
1617                 contents = filter(&contents, "miri");
1618             }
1619             if rustfmt_installer.is_none() {
1620                 contents = filter(&contents, "rustfmt");
1621             }
1622             let ret = tmp.join(p.file_name().unwrap());
1623             t!(fs::write(&ret, &contents));
1624             ret
1625         };
1626
1627         if target.contains("apple-darwin") {
1628             let pkg = tmp.join("pkg");
1629             let _ = fs::remove_dir_all(&pkg);
1630
1631             let pkgbuild = |component: &str| {
1632                 let mut cmd = Command::new("pkgbuild");
1633                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1634                     .arg("--scripts").arg(pkg.join(component))
1635                     .arg("--nopayload")
1636                     .arg(pkg.join(component).with_extension("pkg"));
1637                 builder.run(&mut cmd);
1638             };
1639
1640             let prepare = |name: &str| {
1641                 builder.create_dir(&pkg.join(name));
1642                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1643                         &pkg.join(name));
1644                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1645                 pkgbuild(name);
1646             };
1647             prepare("rustc");
1648             prepare("cargo");
1649             prepare("rust-docs");
1650             prepare("rust-std");
1651             prepare("rust-analysis");
1652
1653             if rls_installer.is_some() {
1654                 prepare("rls");
1655             }
1656             if clippy_installer.is_some() {
1657                 prepare("clippy");
1658             }
1659             if miri_installer.is_some() {
1660                 prepare("miri");
1661             }
1662
1663             // create an 'uninstall' package
1664             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1665             pkgbuild("uninstall");
1666
1667             builder.create_dir(&pkg.join("res"));
1668             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1669             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1670             let mut cmd = Command::new("productbuild");
1671             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1672                 .arg("--resources").arg(pkg.join("res"))
1673                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1674                                                     pkgname(builder, "rust"),
1675                                                     target)))
1676                 .arg("--package-path").arg(&pkg);
1677             builder.run(&mut cmd);
1678         }
1679
1680         if target.contains("windows") {
1681             let exe = tmp.join("exe");
1682             let _ = fs::remove_dir_all(&exe);
1683
1684             let prepare = |name: &str| {
1685                 builder.create_dir(&exe.join(name));
1686                 let dir = if name == "rust-std" || name == "rust-analysis" {
1687                     format!("{}-{}", name, target)
1688                 } else if name == "rls" {
1689                     "rls-preview".to_string()
1690                 } else if name == "clippy" {
1691                     "clippy-preview".to_string()
1692                 } else if name == "miri" {
1693                     "miri-preview".to_string()
1694                 } else {
1695                     name.to_string()
1696                 };
1697                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1698                             .join(dir),
1699                         &exe.join(name));
1700                 builder.remove(&exe.join(name).join("manifest.in"));
1701             };
1702             prepare("rustc");
1703             prepare("cargo");
1704             prepare("rust-analysis");
1705             prepare("rust-docs");
1706             prepare("rust-std");
1707             if rls_installer.is_some() {
1708                 prepare("rls");
1709             }
1710             if clippy_installer.is_some() {
1711                 prepare("clippy");
1712             }
1713             if miri_installer.is_some() {
1714                 prepare("miri");
1715             }
1716             if target.contains("windows-gnu") {
1717                 prepare("rust-mingw");
1718             }
1719
1720             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1721             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1722             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1723             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1724             builder.create(&exe.join("LICENSE.txt"), &license);
1725
1726             // Generate exe installer
1727             let mut cmd = Command::new("iscc");
1728             cmd.arg("rust.iss")
1729                 .current_dir(&exe);
1730             if target.contains("windows-gnu") {
1731                 cmd.arg("/dMINGW");
1732             }
1733             add_env(builder, &mut cmd, target);
1734             builder.run(&mut cmd);
1735             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1736                     &distdir(builder),
1737                     0o755);
1738
1739             // Generate msi installer
1740             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1741             let heat = wix.join("bin/heat.exe");
1742             let candle = wix.join("bin/candle.exe");
1743             let light = wix.join("bin/light.exe");
1744
1745             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1746             builder.run(Command::new(&heat)
1747                             .current_dir(&exe)
1748                             .arg("dir")
1749                             .arg("rustc")
1750                             .args(&heat_flags)
1751                             .arg("-cg").arg("RustcGroup")
1752                             .arg("-dr").arg("Rustc")
1753                             .arg("-var").arg("var.RustcDir")
1754                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1755             builder.run(Command::new(&heat)
1756                             .current_dir(&exe)
1757                             .arg("dir")
1758                             .arg("rust-docs")
1759                             .args(&heat_flags)
1760                             .arg("-cg").arg("DocsGroup")
1761                             .arg("-dr").arg("Docs")
1762                             .arg("-var").arg("var.DocsDir")
1763                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1764                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1765             builder.run(Command::new(&heat)
1766                             .current_dir(&exe)
1767                             .arg("dir")
1768                             .arg("cargo")
1769                             .args(&heat_flags)
1770                             .arg("-cg").arg("CargoGroup")
1771                             .arg("-dr").arg("Cargo")
1772                             .arg("-var").arg("var.CargoDir")
1773                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1774                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1775             builder.run(Command::new(&heat)
1776                             .current_dir(&exe)
1777                             .arg("dir")
1778                             .arg("rust-std")
1779                             .args(&heat_flags)
1780                             .arg("-cg").arg("StdGroup")
1781                             .arg("-dr").arg("Std")
1782                             .arg("-var").arg("var.StdDir")
1783                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1784             if rls_installer.is_some() {
1785                 builder.run(Command::new(&heat)
1786                                 .current_dir(&exe)
1787                                 .arg("dir")
1788                                 .arg("rls")
1789                                 .args(&heat_flags)
1790                                 .arg("-cg").arg("RlsGroup")
1791                                 .arg("-dr").arg("Rls")
1792                                 .arg("-var").arg("var.RlsDir")
1793                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1794                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1795             }
1796             if clippy_installer.is_some() {
1797                 builder.run(Command::new(&heat)
1798                                 .current_dir(&exe)
1799                                 .arg("dir")
1800                                 .arg("clippy")
1801                                 .args(&heat_flags)
1802                                 .arg("-cg").arg("ClippyGroup")
1803                                 .arg("-dr").arg("Clippy")
1804                                 .arg("-var").arg("var.ClippyDir")
1805                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1806                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1807             }
1808             if miri_installer.is_some() {
1809                 builder.run(Command::new(&heat)
1810                                 .current_dir(&exe)
1811                                 .arg("dir")
1812                                 .arg("miri")
1813                                 .args(&heat_flags)
1814                                 .arg("-cg").arg("MiriGroup")
1815                                 .arg("-dr").arg("Miri")
1816                                 .arg("-var").arg("var.MiriDir")
1817                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1818                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1819             }
1820             builder.run(Command::new(&heat)
1821                             .current_dir(&exe)
1822                             .arg("dir")
1823                             .arg("rust-analysis")
1824                             .args(&heat_flags)
1825                             .arg("-cg").arg("AnalysisGroup")
1826                             .arg("-dr").arg("Analysis")
1827                             .arg("-var").arg("var.AnalysisDir")
1828                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1829                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1830             if target.contains("windows-gnu") {
1831                 builder.run(Command::new(&heat)
1832                                 .current_dir(&exe)
1833                                 .arg("dir")
1834                                 .arg("rust-mingw")
1835                                 .args(&heat_flags)
1836                                 .arg("-cg").arg("GccGroup")
1837                                 .arg("-dr").arg("Gcc")
1838                                 .arg("-var").arg("var.GccDir")
1839                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1840             }
1841
1842             let candle = |input: &Path| {
1843                 let output = exe.join(input.file_stem().unwrap())
1844                                 .with_extension("wixobj");
1845                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1846                 let mut cmd = Command::new(&candle);
1847                 cmd.current_dir(&exe)
1848                     .arg("-nologo")
1849                     .arg("-dRustcDir=rustc")
1850                     .arg("-dDocsDir=rust-docs")
1851                     .arg("-dCargoDir=cargo")
1852                     .arg("-dStdDir=rust-std")
1853                     .arg("-dAnalysisDir=rust-analysis")
1854                     .arg("-arch").arg(&arch)
1855                     .arg("-out").arg(&output)
1856                     .arg(&input);
1857                 add_env(builder, &mut cmd, target);
1858
1859                 if rls_installer.is_some() {
1860                     cmd.arg("-dRlsDir=rls");
1861                 }
1862                 if clippy_installer.is_some() {
1863                     cmd.arg("-dClippyDir=clippy");
1864                 }
1865                 if miri_installer.is_some() {
1866                     cmd.arg("-dMiriDir=miri");
1867                 }
1868                 if target.contains("windows-gnu") {
1869                     cmd.arg("-dGccDir=rust-mingw");
1870                 }
1871                 builder.run(&mut cmd);
1872             };
1873             candle(&xform(&etc.join("msi/rust.wxs")));
1874             candle(&etc.join("msi/ui.wxs"));
1875             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1876             candle("RustcGroup.wxs".as_ref());
1877             candle("DocsGroup.wxs".as_ref());
1878             candle("CargoGroup.wxs".as_ref());
1879             candle("StdGroup.wxs".as_ref());
1880             if rls_installer.is_some() {
1881                 candle("RlsGroup.wxs".as_ref());
1882             }
1883             if clippy_installer.is_some() {
1884                 candle("ClippyGroup.wxs".as_ref());
1885             }
1886             if miri_installer.is_some() {
1887                 candle("MiriGroup.wxs".as_ref());
1888             }
1889             candle("AnalysisGroup.wxs".as_ref());
1890
1891             if target.contains("windows-gnu") {
1892                 candle("GccGroup.wxs".as_ref());
1893             }
1894
1895             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1896             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1897             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1898
1899             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1900             let mut cmd = Command::new(&light);
1901             cmd.arg("-nologo")
1902                 .arg("-ext").arg("WixUIExtension")
1903                 .arg("-ext").arg("WixUtilExtension")
1904                 .arg("-out").arg(exe.join(&filename))
1905                 .arg("rust.wixobj")
1906                 .arg("ui.wixobj")
1907                 .arg("rustwelcomedlg.wixobj")
1908                 .arg("RustcGroup.wixobj")
1909                 .arg("DocsGroup.wixobj")
1910                 .arg("CargoGroup.wixobj")
1911                 .arg("StdGroup.wixobj")
1912                 .arg("AnalysisGroup.wixobj")
1913                 .current_dir(&exe);
1914
1915             if rls_installer.is_some() {
1916                 cmd.arg("RlsGroup.wixobj");
1917             }
1918             if clippy_installer.is_some() {
1919                 cmd.arg("ClippyGroup.wixobj");
1920             }
1921             if miri_installer.is_some() {
1922                 cmd.arg("MiriGroup.wixobj");
1923             }
1924
1925             if target.contains("windows-gnu") {
1926                 cmd.arg("GccGroup.wixobj");
1927             }
1928             // ICE57 wrongly complains about the shortcuts
1929             cmd.arg("-sice:ICE57");
1930
1931             builder.run(&mut cmd);
1932
1933             if !builder.config.dry_run {
1934                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1935             }
1936         }
1937     }
1938 }
1939
1940 fn add_env(builder: &Builder, cmd: &mut Command, target: Interned<String>) {
1941     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1942     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1943        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1944        .env("CFG_RELEASE", builder.rust_release())
1945        .env("CFG_VER_MAJOR", parts.next().unwrap())
1946        .env("CFG_VER_MINOR", parts.next().unwrap())
1947        .env("CFG_VER_PATCH", parts.next().unwrap())
1948        .env("CFG_VER_BUILD", "0") // just needed to build
1949        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1950        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1951        .env("CFG_BUILD", target)
1952        .env("CFG_CHANNEL", &builder.config.channel);
1953
1954     if target.contains("windows-gnu") {
1955        cmd.env("CFG_MINGW", "1")
1956           .env("CFG_ABI", "GNU");
1957     } else {
1958        cmd.env("CFG_MINGW", "0")
1959           .env("CFG_ABI", "MSVC");
1960     }
1961
1962     if target.contains("x86_64") {
1963        cmd.env("CFG_PLATFORM", "x64");
1964     } else {
1965        cmd.env("CFG_PLATFORM", "x86");
1966     }
1967 }
1968
1969 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1970 pub struct HashSign;
1971
1972 impl Step for HashSign {
1973     type Output = ();
1974     const ONLY_HOSTS: bool = true;
1975
1976     fn should_run(run: ShouldRun) -> ShouldRun {
1977         run.path("hash-and-sign")
1978     }
1979
1980     fn make_run(run: RunConfig) {
1981         run.builder.ensure(HashSign);
1982     }
1983
1984     fn run(self, builder: &Builder) {
1985         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1986         if builder.config.dry_run {
1987             return;
1988         }
1989         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
1990             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
1991         });
1992         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
1993             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
1994         });
1995         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
1996             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
1997         });
1998         let pass = t!(fs::read_to_string(&file));
1999
2000         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2001
2002         cmd.arg(sign);
2003         cmd.arg(distdir(builder));
2004         cmd.arg(today.trim());
2005         cmd.arg(builder.rust_package_vers());
2006         cmd.arg(addr);
2007         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2008         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2009         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2010         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2011         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2012         cmd.arg(builder.llvm_tools_package_vers());
2013         cmd.arg(builder.lldb_package_vers());
2014
2015         builder.create_dir(&distdir(builder));
2016
2017         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2018         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2019         let status = t!(child.wait());
2020         assert!(status.success());
2021     }
2022 }
2023
2024 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2025 // LLVM tools are linked dynamically.
2026 // Note: This function does no yet support Windows but we also don't support
2027 //       linking LLVM tools dynamically on Windows yet.
2028 pub fn maybe_install_llvm_dylib(builder: &Builder,
2029                                 target: Interned<String>,
2030                                 sysroot: &Path) {
2031     let src_libdir = builder
2032         .llvm_out(target)
2033         .join("lib");
2034     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2035     t!(fs::create_dir_all(&dst_libdir));
2036
2037     if target.contains("apple-darwin") {
2038         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2039         if llvm_dylib_path.exists() {
2040             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2041         }
2042         return
2043     }
2044
2045     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2046     // Since tools link to the latter rather than the former, we have to
2047     // follow the symlink to find out what to distribute.
2048     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2049     if llvm_dylib_path.exists() {
2050         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2051             panic!("dist: Error calling canonicalize path `{}`: {}",
2052                    llvm_dylib_path.display(), e);
2053         });
2054
2055
2056         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2057     }
2058 }
2059
2060 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2061 pub struct LlvmTools {
2062     pub stage: u32,
2063     pub target: Interned<String>,
2064 }
2065
2066 impl Step for LlvmTools {
2067     type Output = Option<PathBuf>;
2068     const ONLY_HOSTS: bool = true;
2069
2070     fn should_run(run: ShouldRun) -> ShouldRun {
2071         run.path("llvm-tools")
2072     }
2073
2074     fn make_run(run: RunConfig) {
2075         run.builder.ensure(LlvmTools {
2076             stage: run.builder.top_stage,
2077             target: run.target,
2078         });
2079     }
2080
2081     fn run(self, builder: &Builder) -> Option<PathBuf> {
2082         let stage = self.stage;
2083         let target = self.target;
2084         assert!(builder.config.extended);
2085
2086         /* run only if llvm-config isn't used */
2087         if let Some(config) = builder.config.target_config.get(&target) {
2088             if let Some(ref _s) = config.llvm_config {
2089                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2090                     stage, target));
2091                 return None;
2092             }
2093         }
2094
2095         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2096         let src = builder.src.join("src/llvm-project/llvm");
2097         let name = pkgname(builder, "llvm-tools");
2098
2099         let tmp = tmpdir(builder);
2100         let image = tmp.join("llvm-tools-image");
2101         drop(fs::remove_dir_all(&image));
2102
2103         // Prepare the image directory
2104         let src_bindir = builder
2105             .llvm_out(target)
2106             .join("bin");
2107         let dst_bindir = image.join("lib/rustlib")
2108             .join(&*target)
2109             .join("bin");
2110         t!(fs::create_dir_all(&dst_bindir));
2111         for tool in LLVM_TOOLS {
2112             let exe = src_bindir.join(exe(tool, &target));
2113             builder.install(&exe, &dst_bindir, 0o755);
2114         }
2115
2116         // Prepare the overlay
2117         let overlay = tmp.join("llvm-tools-overlay");
2118         drop(fs::remove_dir_all(&overlay));
2119         builder.create_dir(&overlay);
2120         builder.install(&src.join("README.txt"), &overlay, 0o644);
2121         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2122         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2123
2124         // Generate the installer tarball
2125         let mut cmd = rust_installer(builder);
2126         cmd.arg("generate")
2127             .arg("--product-name=Rust")
2128             .arg("--rel-manifest-dir=rustlib")
2129             .arg("--success-message=llvm-tools-installed.")
2130             .arg("--image-dir").arg(&image)
2131             .arg("--work-dir").arg(&tmpdir(builder))
2132             .arg("--output-dir").arg(&distdir(builder))
2133             .arg("--non-installed-overlay").arg(&overlay)
2134             .arg(format!("--package-name={}-{}", name, target))
2135             .arg("--legacy-manifest-dirs=rustlib,cargo")
2136             .arg("--component-name=llvm-tools-preview");
2137
2138
2139         builder.run(&mut cmd);
2140         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2141     }
2142 }
2143
2144 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2145 pub struct Lldb {
2146     pub target: Interned<String>,
2147 }
2148
2149 impl Step for Lldb {
2150     type Output = Option<PathBuf>;
2151     const ONLY_HOSTS: bool = true;
2152     const DEFAULT: bool = true;
2153
2154     fn should_run(run: ShouldRun) -> ShouldRun {
2155         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2156     }
2157
2158     fn make_run(run: RunConfig) {
2159         run.builder.ensure(Lldb {
2160             target: run.target,
2161         });
2162     }
2163
2164     fn run(self, builder: &Builder) -> Option<PathBuf> {
2165         let target = self.target;
2166
2167         if builder.config.dry_run {
2168             return None;
2169         }
2170
2171         let bindir = builder
2172             .llvm_out(target)
2173             .join("bin");
2174         let lldb_exe = bindir.join(exe("lldb", &target));
2175         if !lldb_exe.exists() {
2176             return None;
2177         }
2178
2179         builder.info(&format!("Dist Lldb ({})", target));
2180         let src = builder.src.join("src/llvm-project/lldb");
2181         let name = pkgname(builder, "lldb");
2182
2183         let tmp = tmpdir(builder);
2184         let image = tmp.join("lldb-image");
2185         drop(fs::remove_dir_all(&image));
2186
2187         // Prepare the image directory
2188         let root = image.join("lib/rustlib").join(&*target);
2189         let dst = root.join("bin");
2190         t!(fs::create_dir_all(&dst));
2191         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2192             let exe = bindir.join(exe(program, &target));
2193             builder.install(&exe, &dst, 0o755);
2194         }
2195
2196         // The libraries.
2197         let libdir = builder.llvm_out(target).join("lib");
2198         let dst = root.join("lib");
2199         t!(fs::create_dir_all(&dst));
2200         for entry in t!(fs::read_dir(&libdir)) {
2201             let entry = entry.unwrap();
2202             if let Ok(name) = entry.file_name().into_string() {
2203                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2204                     if t!(entry.file_type()).is_symlink() {
2205                         builder.copy_to_folder(&entry.path(), &dst);
2206                     } else {
2207                        builder.install(&entry.path(), &dst, 0o755);
2208                     }
2209                 }
2210             }
2211         }
2212
2213         // The lldb scripts might be installed in lib/python$version
2214         // or in lib64/python$version.  If lib64 exists, use it;
2215         // otherwise lib.
2216         let libdir = builder.llvm_out(target).join("lib64");
2217         let (libdir, libdir_name) = if libdir.exists() {
2218             (libdir, "lib64")
2219         } else {
2220             (builder.llvm_out(target).join("lib"), "lib")
2221         };
2222         for entry in t!(fs::read_dir(&libdir)) {
2223             let entry = t!(entry);
2224             if let Ok(name) = entry.file_name().into_string() {
2225                 if name.starts_with("python") {
2226                     let dst = root.join(libdir_name)
2227                         .join(entry.file_name());
2228                     t!(fs::create_dir_all(&dst));
2229                     builder.cp_r(&entry.path(), &dst);
2230                     break;
2231                 }
2232             }
2233         }
2234
2235         // Prepare the overlay
2236         let overlay = tmp.join("lldb-overlay");
2237         drop(fs::remove_dir_all(&overlay));
2238         builder.create_dir(&overlay);
2239         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2240         builder.create(&overlay.join("version"), &builder.lldb_vers());
2241
2242         // Generate the installer tarball
2243         let mut cmd = rust_installer(builder);
2244         cmd.arg("generate")
2245             .arg("--product-name=Rust")
2246             .arg("--rel-manifest-dir=rustlib")
2247             .arg("--success-message=lldb-installed.")
2248             .arg("--image-dir").arg(&image)
2249             .arg("--work-dir").arg(&tmpdir(builder))
2250             .arg("--output-dir").arg(&distdir(builder))
2251             .arg("--non-installed-overlay").arg(&overlay)
2252             .arg(format!("--package-name={}-{}", name, target))
2253             .arg("--legacy-manifest-dirs=rustlib,cargo")
2254             .arg("--component-name=lldb-preview");
2255
2256
2257         builder.run(&mut cmd);
2258         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2259     }
2260 }