src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub host: Interned<String>,
  72 }
  73
  74 impl Step for Docs {
  75     type Output = PathBuf;
  76     const DEFAULT: bool = true;
  77
  78     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  79         run.path("src/doc")
  80     }
  81
  82     fn make_run(run: RunConfig<'_>) {
  83         run.builder.ensure(Docs {
  84             host: run.target,
  85         });
  86     }
  87
  88     /// Builds the `rust-docs` installer component.
  89     fn run(self, builder: &Builder<'_>) -> PathBuf {
  90         let host = self.host;
  91
  92         let name = pkgname(builder, "rust-docs");
  93
  94         builder.info(&format!("Dist docs ({})", host));
  95         if !builder.config.docs {
  96             builder.info("\tskipping - docs disabled");
  97             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
  98         }
  99
 100         builder.default_doc(None);
 101
 102         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 103         let _ = fs::remove_dir_all(&image);
 104
 105         let dst = image.join("share/doc/rust/html");
 106         t!(fs::create_dir_all(&dst));
 107         let src = builder.doc_out(host);
 108         builder.cp_r(&src, &dst);
 109
 110         let mut cmd = rust_installer(builder);
 111         cmd.arg("generate")
 112            .arg("--product-name=Rust-Documentation")
 113            .arg("--rel-manifest-dir=rustlib")
 114            .arg("--success-message=Rust-documentation-is-installed.")
 115            .arg("--image-dir").arg(&image)
 116            .arg("--work-dir").arg(&tmpdir(builder))
 117            .arg("--output-dir").arg(&distdir(builder))
 118            .arg(format!("--package-name={}-{}", name, host))
 119            .arg("--component-name=rust-docs")
 120            .arg("--legacy-manifest-dirs=rustlib,cargo")
 121            .arg("--bulk-dirs=share/doc/rust/html");
 122         builder.run(&mut cmd);
 123         builder.remove_dir(&image);
 124
 125         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 126     }
 127 }
 128
 129 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 130 pub struct RustcDocs {
 131     pub host: Interned<String>,
 132 }
 133
 134 impl Step for RustcDocs {
 135     type Output = PathBuf;
 136     const DEFAULT: bool = true;
 137
 138     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 139         run.path("src/librustc")
 140     }
 141
 142     fn make_run(run: RunConfig<'_>) {
 143         run.builder.ensure(RustcDocs {
 144             host: run.target,
 145         });
 146     }
 147
 148     /// Builds the `rustc-docs` installer component.
 149     fn run(self, builder: &Builder<'_>) -> PathBuf {
 150         let host = self.host;
 151
 152         let name = pkgname(builder, "rustc-docs");
 153
 154         builder.info(&format!("Dist compiler docs ({})", host));
 155         if !builder.config.compiler_docs {
 156             builder.info("\tskipping - compiler docs disabled");
 157             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 158         }
 159
 160         builder.default_doc(None);
 161
 162         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 163         let _ = fs::remove_dir_all(&image);
 164
 165         let dst = image.join("share/doc/rust/html");
 166         t!(fs::create_dir_all(&dst));
 167         let src = builder.compiler_doc_out(host);
 168         builder.cp_r(&src, &dst);
 169
 170         let mut cmd = rust_installer(builder);
 171         cmd.arg("generate")
 172            .arg("--product-name=Rustc-Documentation")
 173            .arg("--rel-manifest-dir=rustlib")
 174            .arg("--success-message=Rustc-documentation-is-installed.")
 175            .arg("--image-dir").arg(&image)
 176            .arg("--work-dir").arg(&tmpdir(builder))
 177            .arg("--output-dir").arg(&distdir(builder))
 178            .arg(format!("--package-name={}-{}", name, host))
 179            .arg("--component-name=rustc-docs")
 180            .arg("--legacy-manifest-dirs=rustlib,cargo")
 181            .arg("--bulk-dirs=share/doc/rust/html");
 182         builder.run(&mut cmd);
 183         builder.remove_dir(&image);
 184
 185         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 186     }
 187 }
 188
 189 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 190     let mut found = Vec::with_capacity(files.len());
 191
 192     for file in files {
 193         let file_path =
 194             path.iter()
 195                 .map(|dir| dir.join(file))
 196                 .find(|p| p.exists());
 197
 198         if let Some(file_path) = file_path {
 199             found.push(file_path);
 200         } else {
 201             panic!("Could not find '{}' in {:?}", file, path);
 202         }
 203     }
 204
 205     found
 206 }
 207
 208 fn make_win_dist(
 209     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 210 ) {
 211     //Ask gcc where it keeps its stuff
 212     let mut cmd = Command::new(builder.cc(target_triple));
 213     cmd.arg("-print-search-dirs");
 214     let gcc_out = output(&mut cmd);
 215
 216     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 217     let mut lib_path = Vec::new();
 218
 219     for line in gcc_out.lines() {
 220         let idx = line.find(':').unwrap();
 221         let key = &line[..idx];
 222         let trim_chars: &[_] = &[' ', '='];
 223         let value =
 224             line[(idx + 1)..]
 225                 .trim_start_matches(trim_chars)
 226                 .split(';')
 227                 .map(PathBuf::from);
 228
 229         if key == "programs" {
 230             bin_path.extend(value);
 231         } else if key == "libraries" {
 232             lib_path.extend(value);
 233         }
 234     }
 235
 236     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 237     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 238     if target_triple.starts_with("i686-") {
 239         rustc_dlls.push("libgcc_s_dw2-1.dll");
 240     } else {
 241         rustc_dlls.push("libgcc_s_seh-1.dll");
 242     }
 243
 244     let target_libs = [ //MinGW libs
 245         "libgcc.a",
 246         "libgcc_eh.a",
 247         "libgcc_s.a",
 248         "libm.a",
 249         "libmingw32.a",
 250         "libmingwex.a",
 251         "libstdc++.a",
 252         "libiconv.a",
 253         "libmoldname.a",
 254         "libpthread.a",
 255         //Windows import libs
 256         "libadvapi32.a",
 257         "libbcrypt.a",
 258         "libcomctl32.a",
 259         "libcomdlg32.a",
 260         "libcredui.a",
 261         "libcrypt32.a",
 262         "libdbghelp.a",
 263         "libgdi32.a",
 264         "libimagehlp.a",
 265         "libiphlpapi.a",
 266         "libkernel32.a",
 267         "libmsimg32.a",
 268         "libmsvcrt.a",
 269         "libodbc32.a",
 270         "libole32.a",
 271         "liboleaut32.a",
 272         "libopengl32.a",
 273         "libpsapi.a",
 274         "librpcrt4.a",
 275         "libsecur32.a",
 276         "libsetupapi.a",
 277         "libshell32.a",
 278         "libsynchronization.a",
 279         "libuser32.a",
 280         "libuserenv.a",
 281         "libuuid.a",
 282         "libwinhttp.a",
 283         "libwinmm.a",
 284         "libwinspool.a",
 285         "libws2_32.a",
 286         "libwsock32.a",
 287     ];
 288
 289     //Find mingw artifacts we want to bundle
 290     let target_tools = find_files(&target_tools, &bin_path);
 291     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 292     let target_libs = find_files(&target_libs, &lib_path);
 293
 294     // Copy runtime dlls next to rustc.exe
 295     let dist_bin_dir = rust_root.join("bin/");
 296     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 297     for src in rustc_dlls {
 298         builder.copy_to_folder(&src, &dist_bin_dir);
 299     }
 300
 301     //Copy platform tools to platform-specific bin directory
 302     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 303     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 304     for src in target_tools {
 305         builder.copy_to_folder(&src, &target_bin_dir);
 306     }
 307
 308     // Warn windows-gnu users that the bundled GCC cannot compile C files
 309     builder.create(
 310         &target_bin_dir.join("GCC-WARNING.txt"),
 311         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 312          used as a linker. In order to be able to compile projects containing C code use\
 313          the GCC provided by MinGW or Cygwin."
 314     );
 315
 316     //Copy platform libs to platform-specific lib directory
 317     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 318     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 319     for src in target_libs {
 320         builder.copy_to_folder(&src, &target_lib_dir);
 321     }
 322 }
 323
 324 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 325 pub struct Mingw {
 326     pub host: Interned<String>,
 327 }
 328
 329 impl Step for Mingw {
 330     type Output = Option<PathBuf>;
 331     const DEFAULT: bool = true;
 332
 333     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 334         run.never()
 335     }
 336
 337     fn make_run(run: RunConfig<'_>) {
 338         run.builder.ensure(Mingw { host: run.target });
 339     }
 340
 341     /// Builds the `rust-mingw` installer component.
 342     ///
 343     /// This contains all the bits and pieces to run the MinGW Windows targets
 344     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 345     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 346         let host = self.host;
 347
 348         if !host.contains("pc-windows-gnu") {
 349             return None;
 350         }
 351
 352         builder.info(&format!("Dist mingw ({})", host));
 353         let name = pkgname(builder, "rust-mingw");
 354         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 355         let _ = fs::remove_dir_all(&image);
 356         t!(fs::create_dir_all(&image));
 357
 358         // The first argument is a "temporary directory" which is just
 359         // thrown away (this contains the runtime DLLs included in the rustc package
 360         // above) and the second argument is where to place all the MinGW components
 361         // (which is what we want).
 362         make_win_dist(&tmpdir(builder), &image, host, &builder);
 363
 364         let mut cmd = rust_installer(builder);
 365         cmd.arg("generate")
 366            .arg("--product-name=Rust-MinGW")
 367            .arg("--rel-manifest-dir=rustlib")
 368            .arg("--success-message=Rust-MinGW-is-installed.")
 369            .arg("--image-dir").arg(&image)
 370            .arg("--work-dir").arg(&tmpdir(builder))
 371            .arg("--output-dir").arg(&distdir(builder))
 372            .arg(format!("--package-name={}-{}", name, host))
 373            .arg("--component-name=rust-mingw")
 374            .arg("--legacy-manifest-dirs=rustlib,cargo");
 375         builder.run(&mut cmd);
 376         t!(fs::remove_dir_all(&image));
 377         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 378     }
 379 }
 380
 381 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 382 pub struct Rustc {
 383     pub compiler: Compiler,
 384 }
 385
 386 impl Step for Rustc {
 387     type Output = PathBuf;
 388     const DEFAULT: bool = true;
 389     const ONLY_HOSTS: bool = true;
 390
 391     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 392         run.path("src/librustc")
 393     }
 394
 395     fn make_run(run: RunConfig<'_>) {
 396         run.builder.ensure(Rustc {
 397             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 398         });
 399     }
 400
 401     /// Creates the `rustc` installer component.
 402     fn run(self, builder: &Builder<'_>) -> PathBuf {
 403         let compiler = self.compiler;
 404         let host = self.compiler.host;
 405
 406         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 407         let name = pkgname(builder, "rustc");
 408         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 409         let _ = fs::remove_dir_all(&image);
 410         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 411         let _ = fs::remove_dir_all(&overlay);
 412
 413         // Prepare the rustc "image", what will actually end up getting installed
 414         prepare_image(builder, compiler, &image);
 415
 416         // Prepare the overlay which is part of the tarball but won't actually be
 417         // installed
 418         let cp = |file: &str| {
 419             builder.install(&builder.src.join(file), &overlay, 0o644);
 420         };
 421         cp("COPYRIGHT");
 422         cp("LICENSE-APACHE");
 423         cp("LICENSE-MIT");
 424         cp("README.md");
 425         // tiny morsel of metadata is used by rust-packaging
 426         let version = builder.rust_version();
 427         builder.create(&overlay.join("version"), &version);
 428         if let Some(sha) = builder.rust_sha() {
 429             builder.create(&overlay.join("git-commit-hash"), &sha);
 430         }
 431
 432         // On MinGW we've got a few runtime DLL dependencies that we need to
 433         // include. The first argument to this script is where to put these DLLs
 434         // (the image we're creating), and the second argument is a junk directory
 435         // to ignore all other MinGW stuff the script creates.
 436         //
 437         // On 32-bit MinGW we're always including a DLL which needs some extra
 438         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 439         // anything requiring us to distribute a license, but it's likely the
 440         // install will *also* include the rust-mingw package, which also needs
 441         // licenses, so to be safe we just include it here in all MinGW packages.
 442         if host.contains("pc-windows-gnu") {
 443             make_win_dist(&image, &tmpdir(builder), host, builder);
 444
 445             let dst = image.join("share/doc");
 446             t!(fs::create_dir_all(&dst));
 447             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 448         }
 449
 450         // Finally, wrap everything up in a nice tarball!
 451         let mut cmd = rust_installer(builder);
 452         cmd.arg("generate")
 453            .arg("--product-name=Rust")
 454            .arg("--rel-manifest-dir=rustlib")
 455            .arg("--success-message=Rust-is-ready-to-roll.")
 456            .arg("--image-dir").arg(&image)
 457            .arg("--work-dir").arg(&tmpdir(builder))
 458            .arg("--output-dir").arg(&distdir(builder))
 459            .arg("--non-installed-overlay").arg(&overlay)
 460            .arg(format!("--package-name={}-{}", name, host))
 461            .arg("--component-name=rustc")
 462            .arg("--legacy-manifest-dirs=rustlib,cargo");
 463         builder.run(&mut cmd);
 464         builder.remove_dir(&image);
 465         builder.remove_dir(&overlay);
 466
 467         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 468
 469         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 470             let host = compiler.host;
 471             let src = builder.sysroot(compiler);
 472             let libdir = builder.rustc_libdir(compiler);
 473
 474             // Copy rustc/rustdoc binaries
 475             t!(fs::create_dir_all(image.join("bin")));
 476             builder.cp_r(&src.join("bin"), &image.join("bin"));
 477
 478             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 479
 480             let libdir_relative = builder.libdir_relative(compiler);
 481
 482             // Copy runtime DLLs needed by the compiler
 483             if libdir_relative.to_str() != Some("bin") {
 484                 for entry in builder.read_dir(&libdir) {
 485                     let name = entry.file_name();
 486                     if let Some(s) = name.to_str() {
 487                         if is_dylib(s) {
 488                             builder.install(&entry.path(), &image.join(&libdir_relative), 0o644);
 489                         }
 490                     }
 491                 }
 492             }
 493
 494             // Copy over the codegen backends
 495             let backends_src = builder.sysroot_codegen_backends(compiler);
 496             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 497             let backends_dst = image.join(&backends_rel);
 498             t!(fs::create_dir_all(&backends_dst));
 499             builder.cp_r(&backends_src, &backends_dst);
 500
 501             // Copy libLLVM.so to the lib dir as well, if needed. While not
 502             // technically needed by rustc itself it's needed by lots of other
 503             // components like the llvm tools and LLD. LLD is included below and
 504             // tools/LLDB come later, so let's just throw it in the rustc
 505             // component for now.
 506             maybe_install_llvm_dylib(builder, host, image);
 507
 508             // Copy over lld if it's there
 509             if builder.config.lld_enabled {
 510                 let exe = exe("rust-lld", &compiler.host);
 511                 let src = builder.sysroot_libdir(compiler, host)
 512                     .parent()
 513                     .unwrap()
 514                     .join("bin")
 515                     .join(&exe);
 516                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 517                 let dst = image.join("lib/rustlib")
 518                     .join(&*host)
 519                     .join("bin")
 520                     .join(&exe);
 521                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 522                 builder.copy(&src, &dst);
 523             }
 524
 525             // Man pages
 526             t!(fs::create_dir_all(image.join("share/man/man1")));
 527             let man_src = builder.src.join("src/doc/man");
 528             let man_dst = image.join("share/man/man1");
 529
 530             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 531             let time = env::var("SOURCE_DATE_EPOCH")
 532                 .map(|timestamp| {
 533                     let epoch = timestamp.parse().map_err(|err| {
 534                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 535                     }).unwrap();
 536
 537                     time::at(Timespec::new(epoch, 0))
 538                 })
 539                 .unwrap_or_else(|_| time::now());
 540
 541             let month_year = t!(time::strftime("%B %Y", &time));
 542             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 543             // to hardlink, and we don't want to edit the source templates
 544             for file_entry in builder.read_dir(&man_src) {
 545                 let page_src = file_entry.path();
 546                 let page_dst = man_dst.join(file_entry.file_name());
 547                 t!(fs::copy(&page_src, &page_dst));
 548                 // template in month/year and version number
 549                 builder.replace_in_file(&page_dst,
 550                                 &[("<INSERT DATE HERE>", &month_year),
 551                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 552             }
 553
 554             // Debugger scripts
 555             builder.ensure(DebuggerScripts {
 556                 sysroot: INTERNER.intern_path(image.to_owned()),
 557                 host,
 558             });
 559
 560             // Misc license info
 561             let cp = |file: &str| {
 562                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 563             };
 564             cp("COPYRIGHT");
 565             cp("LICENSE-APACHE");
 566             cp("LICENSE-MIT");
 567             cp("README.md");
 568         }
 569     }
 570 }
 571
 572 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 573 pub struct DebuggerScripts {
 574     pub sysroot: Interned<PathBuf>,
 575     pub host: Interned<String>,
 576 }
 577
 578 impl Step for DebuggerScripts {
 579     type Output = ();
 580
 581     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 582         run.path("src/lldb_batchmode.py")
 583     }
 584
 585     fn make_run(run: RunConfig<'_>) {
 586         run.builder.ensure(DebuggerScripts {
 587             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 588             host: run.target,
 589         });
 590     }
 591
 592     /// Copies debugger scripts for `target` into the `sysroot` specified.
 593     fn run(self, builder: &Builder<'_>) {
 594         let host = self.host;
 595         let sysroot = self.sysroot;
 596         let dst = sysroot.join("lib/rustlib/etc");
 597         t!(fs::create_dir_all(&dst));
 598         let cp_debugger_script = |file: &str| {
 599             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 600         };
 601         if host.contains("windows-msvc") {
 602             // windbg debugger scripts
 603             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 604                 0o755);
 605
 606             cp_debugger_script("natvis/intrinsic.natvis");
 607             cp_debugger_script("natvis/liballoc.natvis");
 608             cp_debugger_script("natvis/libcore.natvis");
 609         } else {
 610             cp_debugger_script("debugger_pretty_printers_common.py");
 611
 612             // gdb debugger scripts
 613             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 614                     0o755);
 615             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 616                     0o755);
 617
 618             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 619             cp_debugger_script("gdb_rust_pretty_printing.py");
 620
 621             // lldb debugger scripts
 622             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 623                     0o755);
 624
 625             cp_debugger_script("lldb_rust_formatters.py");
 626         }
 627     }
 628 }
 629
 630 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 631 pub struct Std {
 632     pub compiler: Compiler,
 633     pub target: Interned<String>,
 634 }
 635
 636 impl Step for Std {
 637     type Output = PathBuf;
 638     const DEFAULT: bool = true;
 639
 640     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 641         run.path("src/libstd")
 642     }
 643
 644     fn make_run(run: RunConfig<'_>) {
 645         run.builder.ensure(Std {
 646             compiler: run.builder.compiler_for(
 647                 run.builder.top_stage,
 648                 run.builder.config.build,
 649                 run.target,
 650             ),
 651             target: run.target,
 652         });
 653     }
 654
 655     fn run(self, builder: &Builder<'_>) -> PathBuf {
 656         let compiler = self.compiler;
 657         let target = self.target;
 658
 659         let name = pkgname(builder, "rust-std");
 660         builder.info(&format!("Dist std stage{} ({} -> {})",
 661             compiler.stage, &compiler.host, target));
 662
 663         // The only true set of target libraries came from the build triple, so
 664         // let's reduce redundant work by only producing archives from that host.
 665         if compiler.host != builder.config.build {
 666             builder.info("\tskipping, not a build host");
 667             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 668         }
 669
 670         // We want to package up as many target libraries as possible
 671         // for the `rust-std` package, so if this is a host target we
 672         // depend on librustc and otherwise we just depend on libtest.
 673         if builder.hosts.iter().any(|t| t == target) {
 674             builder.ensure(compile::Rustc { compiler, target });
 675         } else {
 676             if builder.no_std(target) == Some(true) {
 677                 // the `test` doesn't compile for no-std targets
 678                 builder.ensure(compile::Std { compiler, target });
 679             } else {
 680                 builder.ensure(compile::Test { compiler, target });
 681             }
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             // Find the actual compiler (handling the full bootstrap option) which
 741             // produced the save-analysis data because that data isn't copied
 742             // through the sysroot uplifting.
 743             compiler: run.builder.compiler_for(
 744                 run.builder.top_stage,
 745                 run.builder.config.build,
 746                 run.target,
 747             ),
 748             target: run.target,
 749         });
 750     }
 751
 752     /// Creates a tarball of save-analysis metadata, if available.
 753     fn run(self, builder: &Builder<'_>) -> PathBuf {
 754         let compiler = self.compiler;
 755         let target = self.target;
 756         assert!(builder.config.extended);
 757         builder.info("Dist analysis");
 758         let name = pkgname(builder, "rust-analysis");
 759
 760         if &compiler.host != builder.config.build {
 761             builder.info("\tskipping, not a build host");
 762             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 763         }
 764
 765         builder.ensure(Std { compiler, target });
 766
 767         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 768
 769         let src = builder.stage_out(compiler, Mode::Std)
 770             .join(target).join(builder.cargo_dir()).join("deps");
 771
 772         let image_src = src.join("save-analysis");
 773         let dst = image.join("lib/rustlib").join(target).join("analysis");
 774         t!(fs::create_dir_all(&dst));
 775         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 776         builder.cp_r(&image_src, &dst);
 777
 778         let mut cmd = rust_installer(builder);
 779         cmd.arg("generate")
 780            .arg("--product-name=Rust")
 781            .arg("--rel-manifest-dir=rustlib")
 782            .arg("--success-message=save-analysis-saved.")
 783            .arg("--image-dir").arg(&image)
 784            .arg("--work-dir").arg(&tmpdir(builder))
 785            .arg("--output-dir").arg(&distdir(builder))
 786            .arg(format!("--package-name={}-{}", name, target))
 787            .arg(format!("--component-name=rust-analysis-{}", target))
 788            .arg("--legacy-manifest-dirs=rustlib,cargo");
 789         builder.run(&mut cmd);
 790         builder.remove_dir(&image);
 791         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 792     }
 793 }
 794
 795 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 796     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 797         let spath = match path.to_str() {
 798             Some(path) => path,
 799             None => return false,
 800         };
 801         if spath.ends_with("~") || spath.ends_with(".pyc") {
 802             return false
 803         }
 804
 805         const LLVM_PROJECTS: &[&str] = &[
 806             "llvm-project/clang", "llvm-project\\clang",
 807             "llvm-project/libunwind", "llvm-project\\libunwind",
 808             "llvm-project/lld", "llvm-project\\lld",
 809             "llvm-project/lldb", "llvm-project\\lldb",
 810             "llvm-project/llvm", "llvm-project\\llvm",
 811         ];
 812         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 813             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 814         {
 815             return false;
 816         }
 817
 818         const LLVM_TEST: &[&str] = &[
 819             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 820             "llvm-emscripten/test", "llvm-emscripten\\test",
 821         ];
 822         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 823             (spath.ends_with(".ll") ||
 824              spath.ends_with(".td") ||
 825              spath.ends_with(".s")) {
 826             return false
 827         }
 828         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 829             return false
 830         }
 831
 832         let full_path = Path::new(dir).join(path);
 833         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 834             return false;
 835         }
 836
 837         let excludes = [
 838             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 839             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 840             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 841             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 842         ];
 843         !path.iter()
 844              .map(|s| s.to_str().unwrap())
 845              .any(|s| excludes.contains(&s))
 846     }
 847
 848     // Copy the directories using our filter
 849     for item in src_dirs {
 850         let dst = &dst_dir.join(item);
 851         t!(fs::create_dir_all(dst));
 852         builder.cp_filtered(
 853             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 854     }
 855 }
 856
 857 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 858 pub struct Src;
 859
 860 impl Step for Src {
 861     /// The output path of the src installer tarball
 862     type Output = PathBuf;
 863     const DEFAULT: bool = true;
 864     const ONLY_HOSTS: bool = true;
 865
 866     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 867         run.path("src")
 868     }
 869
 870     fn make_run(run: RunConfig<'_>) {
 871         run.builder.ensure(Src);
 872     }
 873
 874     /// Creates the `rust-src` installer component
 875     fn run(self, builder: &Builder<'_>) -> PathBuf {
 876         builder.info("Dist src");
 877
 878         let name = pkgname(builder, "rust-src");
 879         let image = tmpdir(builder).join(format!("{}-image", name));
 880         let _ = fs::remove_dir_all(&image);
 881
 882         let dst = image.join("lib/rustlib/src");
 883         let dst_src = dst.join("rust");
 884         t!(fs::create_dir_all(&dst_src));
 885
 886         let src_files = [
 887             "Cargo.lock",
 888         ];
 889         // This is the reduced set of paths which will become the rust-src component
 890         // (essentially libstd and all of its path dependencies)
 891         let std_src_dirs = [
 892             "src/build_helper",
 893             "src/liballoc",
 894             "src/libcore",
 895             "src/libpanic_abort",
 896             "src/libpanic_unwind",
 897             "src/librustc_asan",
 898             "src/librustc_lsan",
 899             "src/librustc_msan",
 900             "src/librustc_tsan",
 901             "src/libstd",
 902             "src/libunwind",
 903             "src/libtest",
 904             "src/libterm",
 905             "src/libprofiler_builtins",
 906             "src/stdarch",
 907             "src/libproc_macro",
 908             "src/tools/rustc-std-workspace-core",
 909             "src/tools/rustc-std-workspace-alloc",
 910             "src/librustc",
 911             "src/libsyntax",
 912         ];
 913
 914         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 915         for file in src_files.iter() {
 916             builder.copy(&builder.src.join(file), &dst_src.join(file));
 917         }
 918
 919         // Create source tarball in rust-installer format
 920         let mut cmd = rust_installer(builder);
 921         cmd.arg("generate")
 922            .arg("--product-name=Rust")
 923            .arg("--rel-manifest-dir=rustlib")
 924            .arg("--success-message=Awesome-Source.")
 925            .arg("--image-dir").arg(&image)
 926            .arg("--work-dir").arg(&tmpdir(builder))
 927            .arg("--output-dir").arg(&distdir(builder))
 928            .arg(format!("--package-name={}", name))
 929            .arg("--component-name=rust-src")
 930            .arg("--legacy-manifest-dirs=rustlib,cargo");
 931         builder.run(&mut cmd);
 932
 933         builder.remove_dir(&image);
 934         distdir(builder).join(&format!("{}.tar.gz", name))
 935     }
 936 }
 937
 938 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 939 pub struct PlainSourceTarball;
 940
 941 impl Step for PlainSourceTarball {
 942     /// Produces the location of the tarball generated
 943     type Output = PathBuf;
 944     const DEFAULT: bool = true;
 945     const ONLY_HOSTS: bool = true;
 946
 947     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 948         let builder = run.builder;
 949         run.path("src").default_condition(builder.config.rust_dist_src)
 950     }
 951
 952     fn make_run(run: RunConfig<'_>) {
 953         run.builder.ensure(PlainSourceTarball);
 954     }
 955
 956     /// Creates the plain source tarball
 957     fn run(self, builder: &Builder<'_>) -> PathBuf {
 958         builder.info("Create plain source tarball");
 959
 960         // Make sure that the root folder of tarball has the correct name
 961         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 962         let plain_dst_src = tmpdir(builder).join(&plain_name);
 963         let _ = fs::remove_dir_all(&plain_dst_src);
 964         t!(fs::create_dir_all(&plain_dst_src));
 965
 966         // This is the set of root paths which will become part of the source package
 967         let src_files = [
 968             "COPYRIGHT",
 969             "LICENSE-APACHE",
 970             "LICENSE-MIT",
 971             "CONTRIBUTING.md",
 972             "README.md",
 973             "RELEASES.md",
 974             "configure",
 975             "x.py",
 976             "config.toml.example",
 977             "Cargo.toml",
 978             "Cargo.lock",
 979         ];
 980         let src_dirs = [
 981             "src",
 982         ];
 983
 984         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 985
 986         // Copy the files normally
 987         for item in &src_files {
 988             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 989         }
 990
 991         // Create the version file
 992         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 993         if let Some(sha) = builder.rust_sha() {
 994             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 995         }
 996
 997         // If we're building from git sources, we need to vendor a complete distribution.
 998         if builder.rust_info.is_git() {
 999             // Vendor all Cargo dependencies
1000             let mut cmd = Command::new(&builder.initial_cargo);
1001             cmd.arg("vendor")
1002                .current_dir(&plain_dst_src);
1003             builder.run(&mut cmd);
1004         }
1005
1006         // Create plain source tarball
1007         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1008         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1009         tarball.set_extension(""); // strip .gz
1010         tarball.set_extension(""); // strip .tar
1011         if let Some(dir) = tarball.parent() {
1012             builder.create_dir(&dir);
1013         }
1014         builder.info("running installer");
1015         let mut cmd = rust_installer(builder);
1016         cmd.arg("tarball")
1017            .arg("--input").arg(&plain_name)
1018            .arg("--output").arg(&tarball)
1019            .arg("--work-dir=.")
1020            .current_dir(tmpdir(builder));
1021         builder.run(&mut cmd);
1022         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1023     }
1024 }
1025
1026 // We have to run a few shell scripts, which choke quite a bit on both `\`
1027 // characters and on `C:\` paths, so normalize both of them away.
1028 pub fn sanitize_sh(path: &Path) -> String {
1029     let path = path.to_str().unwrap().replace("\\", "/");
1030     return change_drive(&path).unwrap_or(path);
1031
1032     fn change_drive(s: &str) -> Option<String> {
1033         let mut ch = s.chars();
1034         let drive = ch.next().unwrap_or('C');
1035         if ch.next() != Some(':') {
1036             return None
1037         }
1038         if ch.next() != Some('/') {
1039             return None
1040         }
1041         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1042     }
1043 }
1044
1045 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1046 pub struct Cargo {
1047     pub compiler: Compiler,
1048     pub target: Interned<String>,
1049 }
1050
1051 impl Step for Cargo {
1052     type Output = PathBuf;
1053     const ONLY_HOSTS: bool = true;
1054
1055     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1056         run.path("cargo")
1057     }
1058
1059     fn make_run(run: RunConfig<'_>) {
1060         run.builder.ensure(Cargo {
1061             compiler: run.builder.compiler_for(
1062                 run.builder.top_stage,
1063                 run.builder.config.build,
1064                 run.target,
1065             ),
1066             target: run.target,
1067         });
1068     }
1069
1070     fn run(self, builder: &Builder<'_>) -> PathBuf {
1071         let compiler = self.compiler;
1072         let target = self.target;
1073
1074         builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target));
1075         let src = builder.src.join("src/tools/cargo");
1076         let etc = src.join("src/etc");
1077         let release_num = builder.release_num("cargo");
1078         let name = pkgname(builder, "cargo");
1079         let version = builder.cargo_info.version(builder, &release_num);
1080
1081         let tmp = tmpdir(builder);
1082         let image = tmp.join("cargo-image");
1083         drop(fs::remove_dir_all(&image));
1084         builder.create_dir(&image);
1085
1086         // Prepare the image directory
1087         builder.create_dir(&image.join("share/zsh/site-functions"));
1088         builder.create_dir(&image.join("etc/bash_completion.d"));
1089         let cargo = builder.ensure(tool::Cargo { compiler, target });
1090         builder.install(&cargo, &image.join("bin"), 0o755);
1091         for man in t!(etc.join("man").read_dir()) {
1092             let man = t!(man);
1093             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1094         }
1095         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1096         builder.copy(&etc.join("cargo.bashcomp.sh"),
1097              &image.join("etc/bash_completion.d/cargo"));
1098         let doc = image.join("share/doc/cargo");
1099         builder.install(&src.join("README.md"), &doc, 0o644);
1100         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1101         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1102         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1103
1104         // Prepare the overlay
1105         let overlay = tmp.join("cargo-overlay");
1106         drop(fs::remove_dir_all(&overlay));
1107         builder.create_dir(&overlay);
1108         builder.install(&src.join("README.md"), &overlay, 0o644);
1109         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1110         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1111         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1112         builder.create(&overlay.join("version"), &version);
1113
1114         // Generate the installer tarball
1115         let mut cmd = rust_installer(builder);
1116         cmd.arg("generate")
1117            .arg("--product-name=Rust")
1118            .arg("--rel-manifest-dir=rustlib")
1119            .arg("--success-message=Rust-is-ready-to-roll.")
1120            .arg("--image-dir").arg(&image)
1121            .arg("--work-dir").arg(&tmpdir(builder))
1122            .arg("--output-dir").arg(&distdir(builder))
1123            .arg("--non-installed-overlay").arg(&overlay)
1124            .arg(format!("--package-name={}-{}", name, target))
1125            .arg("--component-name=cargo")
1126            .arg("--legacy-manifest-dirs=rustlib,cargo");
1127         builder.run(&mut cmd);
1128         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1129     }
1130 }
1131
1132 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1133 pub struct Rls {
1134     pub compiler: Compiler,
1135     pub target: Interned<String>,
1136 }
1137
1138 impl Step for Rls {
1139     type Output = Option<PathBuf>;
1140     const ONLY_HOSTS: bool = true;
1141
1142     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1143         run.path("rls")
1144     }
1145
1146     fn make_run(run: RunConfig<'_>) {
1147         run.builder.ensure(Rls {
1148             compiler: run.builder.compiler_for(
1149                 run.builder.top_stage,
1150                 run.builder.config.build,
1151                 run.target,
1152             ),
1153             target: run.target,
1154         });
1155     }
1156
1157     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1158         let compiler = self.compiler;
1159         let target = self.target;
1160         assert!(builder.config.extended);
1161
1162         builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target));
1163         let src = builder.src.join("src/tools/rls");
1164         let release_num = builder.release_num("rls");
1165         let name = pkgname(builder, "rls");
1166         let version = builder.rls_info.version(builder, &release_num);
1167
1168         let tmp = tmpdir(builder);
1169         let image = tmp.join("rls-image");
1170         drop(fs::remove_dir_all(&image));
1171         t!(fs::create_dir_all(&image));
1172
1173         // Prepare the image directory
1174         // We expect RLS to build, because we've exited this step above if tool
1175         // state for RLS isn't testing.
1176         let rls = builder.ensure(tool::Rls {
1177             compiler,
1178             target,
1179             extra_features: Vec::new(),
1180         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1181
1182         builder.install(&rls, &image.join("bin"), 0o755);
1183         let doc = image.join("share/doc/rls");
1184         builder.install(&src.join("README.md"), &doc, 0o644);
1185         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1186         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1187
1188         // Prepare the overlay
1189         let overlay = tmp.join("rls-overlay");
1190         drop(fs::remove_dir_all(&overlay));
1191         t!(fs::create_dir_all(&overlay));
1192         builder.install(&src.join("README.md"), &overlay, 0o644);
1193         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1194         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1195         builder.create(&overlay.join("version"), &version);
1196
1197         // Generate the installer tarball
1198         let mut cmd = rust_installer(builder);
1199         cmd.arg("generate")
1200            .arg("--product-name=Rust")
1201            .arg("--rel-manifest-dir=rustlib")
1202            .arg("--success-message=RLS-ready-to-serve.")
1203            .arg("--image-dir").arg(&image)
1204            .arg("--work-dir").arg(&tmpdir(builder))
1205            .arg("--output-dir").arg(&distdir(builder))
1206            .arg("--non-installed-overlay").arg(&overlay)
1207            .arg(format!("--package-name={}-{}", name, target))
1208            .arg("--legacy-manifest-dirs=rustlib,cargo")
1209            .arg("--component-name=rls-preview");
1210
1211         builder.run(&mut cmd);
1212         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1213     }
1214 }
1215
1216 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1217 pub struct Clippy {
1218     pub compiler: Compiler,
1219     pub target: Interned<String>,
1220 }
1221
1222 impl Step for Clippy {
1223     type Output = Option<PathBuf>;
1224     const ONLY_HOSTS: bool = true;
1225
1226     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1227         run.path("clippy")
1228     }
1229
1230     fn make_run(run: RunConfig<'_>) {
1231         run.builder.ensure(Clippy {
1232             compiler: run.builder.compiler_for(
1233                 run.builder.top_stage,
1234                 run.builder.config.build,
1235                 run.target,
1236             ),
1237             target: run.target,
1238         });
1239     }
1240
1241     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1242         let compiler = self.compiler;
1243         let target = self.target;
1244         assert!(builder.config.extended);
1245
1246         builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target));
1247         let src = builder.src.join("src/tools/clippy");
1248         let release_num = builder.release_num("clippy");
1249         let name = pkgname(builder, "clippy");
1250         let version = builder.clippy_info.version(builder, &release_num);
1251
1252         let tmp = tmpdir(builder);
1253         let image = tmp.join("clippy-image");
1254         drop(fs::remove_dir_all(&image));
1255         builder.create_dir(&image);
1256
1257         // Prepare the image directory
1258         // We expect clippy to build, because we've exited this step above if tool
1259         // state for clippy isn't testing.
1260         let clippy = builder.ensure(tool::Clippy {
1261             compiler,
1262             target,
1263             extra_features: Vec::new(),
1264         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1265         let cargoclippy = builder.ensure(tool::CargoClippy {
1266             compiler,
1267             target, extra_features: Vec::new()
1268         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1269
1270         builder.install(&clippy, &image.join("bin"), 0o755);
1271         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1272         let doc = image.join("share/doc/clippy");
1273         builder.install(&src.join("README.md"), &doc, 0o644);
1274         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1275         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1276
1277         // Prepare the overlay
1278         let overlay = tmp.join("clippy-overlay");
1279         drop(fs::remove_dir_all(&overlay));
1280         t!(fs::create_dir_all(&overlay));
1281         builder.install(&src.join("README.md"), &overlay, 0o644);
1282         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1283         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1284         builder.create(&overlay.join("version"), &version);
1285
1286         // Generate the installer tarball
1287         let mut cmd = rust_installer(builder);
1288         cmd.arg("generate")
1289            .arg("--product-name=Rust")
1290            .arg("--rel-manifest-dir=rustlib")
1291            .arg("--success-message=clippy-ready-to-serve.")
1292            .arg("--image-dir").arg(&image)
1293            .arg("--work-dir").arg(&tmpdir(builder))
1294            .arg("--output-dir").arg(&distdir(builder))
1295            .arg("--non-installed-overlay").arg(&overlay)
1296            .arg(format!("--package-name={}-{}", name, target))
1297            .arg("--legacy-manifest-dirs=rustlib,cargo")
1298            .arg("--component-name=clippy-preview");
1299
1300         builder.run(&mut cmd);
1301         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1302     }
1303 }
1304
1305 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1306 pub struct Miri {
1307     pub compiler: Compiler,
1308     pub target: Interned<String>,
1309 }
1310
1311 impl Step for Miri {
1312     type Output = Option<PathBuf>;
1313     const ONLY_HOSTS: bool = true;
1314
1315     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1316         run.path("miri")
1317     }
1318
1319     fn make_run(run: RunConfig<'_>) {
1320         run.builder.ensure(Miri {
1321             compiler: run.builder.compiler_for(
1322                 run.builder.top_stage,
1323                 run.builder.config.build,
1324                 run.target,
1325             ),
1326             target: run.target,
1327         });
1328     }
1329
1330     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1331         let compiler = self.compiler;
1332         let target = self.target;
1333         assert!(builder.config.extended);
1334
1335         builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target));
1336         let src = builder.src.join("src/tools/miri");
1337         let release_num = builder.release_num("miri");
1338         let name = pkgname(builder, "miri");
1339         let version = builder.miri_info.version(builder, &release_num);
1340
1341         let tmp = tmpdir(builder);
1342         let image = tmp.join("miri-image");
1343         drop(fs::remove_dir_all(&image));
1344         builder.create_dir(&image);
1345
1346         // Prepare the image directory
1347         // We expect miri to build, because we've exited this step above if tool
1348         // state for miri isn't testing.
1349         let miri = builder.ensure(tool::Miri {
1350             compiler,
1351             target,
1352             extra_features: Vec::new(),
1353         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1354         let cargomiri = builder.ensure(tool::CargoMiri {
1355             compiler,
1356             target,
1357             extra_features: Vec::new()
1358         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1359
1360         builder.install(&miri, &image.join("bin"), 0o755);
1361         builder.install(&cargomiri, &image.join("bin"), 0o755);
1362         let doc = image.join("share/doc/miri");
1363         builder.install(&src.join("README.md"), &doc, 0o644);
1364         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1365         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1366
1367         // Prepare the overlay
1368         let overlay = tmp.join("miri-overlay");
1369         drop(fs::remove_dir_all(&overlay));
1370         t!(fs::create_dir_all(&overlay));
1371         builder.install(&src.join("README.md"), &overlay, 0o644);
1372         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1373         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1374         builder.create(&overlay.join("version"), &version);
1375
1376         // Generate the installer tarball
1377         let mut cmd = rust_installer(builder);
1378         cmd.arg("generate")
1379            .arg("--product-name=Rust")
1380            .arg("--rel-manifest-dir=rustlib")
1381            .arg("--success-message=miri-ready-to-serve.")
1382            .arg("--image-dir").arg(&image)
1383            .arg("--work-dir").arg(&tmpdir(builder))
1384            .arg("--output-dir").arg(&distdir(builder))
1385            .arg("--non-installed-overlay").arg(&overlay)
1386            .arg(format!("--package-name={}-{}", name, target))
1387            .arg("--legacy-manifest-dirs=rustlib,cargo")
1388            .arg("--component-name=miri-preview");
1389
1390         builder.run(&mut cmd);
1391         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1392     }
1393 }
1394
1395 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1396 pub struct Rustfmt {
1397     pub compiler: Compiler,
1398     pub target: Interned<String>,
1399 }
1400
1401 impl Step for Rustfmt {
1402     type Output = Option<PathBuf>;
1403     const ONLY_HOSTS: bool = true;
1404
1405     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1406         run.path("rustfmt")
1407     }
1408
1409     fn make_run(run: RunConfig<'_>) {
1410         run.builder.ensure(Rustfmt {
1411             compiler: run.builder.compiler_for(
1412                 run.builder.top_stage,
1413                 run.builder.config.build,
1414                 run.target,
1415             ),
1416             target: run.target,
1417         });
1418     }
1419
1420     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1421         let compiler = self.compiler;
1422         let target = self.target;
1423
1424         builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target));
1425         let src = builder.src.join("src/tools/rustfmt");
1426         let release_num = builder.release_num("rustfmt");
1427         let name = pkgname(builder, "rustfmt");
1428         let version = builder.rustfmt_info.version(builder, &release_num);
1429
1430         let tmp = tmpdir(builder);
1431         let image = tmp.join("rustfmt-image");
1432         drop(fs::remove_dir_all(&image));
1433         builder.create_dir(&image);
1434
1435         // Prepare the image directory
1436         let rustfmt = builder.ensure(tool::Rustfmt {
1437             compiler,
1438             target,
1439             extra_features: Vec::new(),
1440         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1441         let cargofmt = builder.ensure(tool::Cargofmt {
1442             compiler,
1443             target,
1444             extra_features: Vec::new(),
1445         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1446
1447         builder.install(&rustfmt, &image.join("bin"), 0o755);
1448         builder.install(&cargofmt, &image.join("bin"), 0o755);
1449         let doc = image.join("share/doc/rustfmt");
1450         builder.install(&src.join("README.md"), &doc, 0o644);
1451         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1452         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1453
1454         // Prepare the overlay
1455         let overlay = tmp.join("rustfmt-overlay");
1456         drop(fs::remove_dir_all(&overlay));
1457         builder.create_dir(&overlay);
1458         builder.install(&src.join("README.md"), &overlay, 0o644);
1459         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1460         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1461         builder.create(&overlay.join("version"), &version);
1462
1463         // Generate the installer tarball
1464         let mut cmd = rust_installer(builder);
1465         cmd.arg("generate")
1466            .arg("--product-name=Rust")
1467            .arg("--rel-manifest-dir=rustlib")
1468            .arg("--success-message=rustfmt-ready-to-fmt.")
1469            .arg("--image-dir").arg(&image)
1470            .arg("--work-dir").arg(&tmpdir(builder))
1471            .arg("--output-dir").arg(&distdir(builder))
1472            .arg("--non-installed-overlay").arg(&overlay)
1473            .arg(format!("--package-name={}-{}", name, target))
1474            .arg("--legacy-manifest-dirs=rustlib,cargo")
1475            .arg("--component-name=rustfmt-preview");
1476
1477         builder.run(&mut cmd);
1478         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1479     }
1480 }
1481
1482 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1483 pub struct Extended {
1484     stage: u32,
1485     host: Interned<String>,
1486     target: Interned<String>,
1487 }
1488
1489 impl Step for Extended {
1490     type Output = ();
1491     const DEFAULT: bool = true;
1492     const ONLY_HOSTS: bool = true;
1493
1494     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1495         let builder = run.builder;
1496         run.path("extended").default_condition(builder.config.extended)
1497     }
1498
1499     fn make_run(run: RunConfig<'_>) {
1500         run.builder.ensure(Extended {
1501             stage: run.builder.top_stage,
1502             host: run.builder.config.build,
1503             target: run.target,
1504         });
1505     }
1506
1507     /// Creates a combined installer for the specified target in the provided stage.
1508     fn run(self, builder: &Builder<'_>) {
1509         let target = self.target;
1510         let stage = self.stage;
1511         let compiler = builder.compiler_for(self.stage, self.host, self.target);
1512
1513         builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target));
1514
1515         let rustc_installer = builder.ensure(Rustc {
1516             compiler: builder.compiler(stage, target),
1517         });
1518         let cargo_installer = builder.ensure(Cargo { compiler, target });
1519         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
1520         let rls_installer = builder.ensure(Rls { compiler, target });
1521         let llvm_tools_installer = builder.ensure(LlvmTools { target });
1522         let clippy_installer = builder.ensure(Clippy { compiler, target });
1523         let miri_installer = builder.ensure(Miri { compiler, target });
1524         let lldb_installer = builder.ensure(Lldb { target });
1525         let mingw_installer = builder.ensure(Mingw { host: target });
1526         let analysis_installer = builder.ensure(Analysis { compiler, target });
1527
1528         let docs_installer = builder.ensure(Docs { host: target, });
1529         let std_installer = builder.ensure(Std {
1530             compiler: builder.compiler(stage, target),
1531             target,
1532         });
1533
1534         let tmp = tmpdir(builder);
1535         let overlay = tmp.join("extended-overlay");
1536         let etc = builder.src.join("src/etc/installer");
1537         let work = tmp.join("work");
1538
1539         let _ = fs::remove_dir_all(&overlay);
1540         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1541         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1542         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1543         let version = builder.rust_version();
1544         builder.create(&overlay.join("version"), &version);
1545         if let Some(sha) = builder.rust_sha() {
1546             builder.create(&overlay.join("git-commit-hash"), &sha);
1547         }
1548         builder.install(&etc.join("README.md"), &overlay, 0o644);
1549
1550         // When rust-std package split from rustc, we needed to ensure that during
1551         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1552         // the std files during uninstall. To do this ensure that rustc comes
1553         // before rust-std in the list below.
1554         let mut tarballs = Vec::new();
1555         tarballs.push(rustc_installer);
1556         tarballs.push(cargo_installer);
1557         tarballs.extend(rls_installer.clone());
1558         tarballs.extend(clippy_installer.clone());
1559         tarballs.extend(miri_installer.clone());
1560         tarballs.extend(rustfmt_installer.clone());
1561         tarballs.extend(llvm_tools_installer);
1562         tarballs.extend(lldb_installer);
1563         tarballs.push(analysis_installer);
1564         tarballs.push(std_installer);
1565         if builder.config.docs {
1566             tarballs.push(docs_installer);
1567         }
1568         if target.contains("pc-windows-gnu") {
1569             tarballs.push(mingw_installer.unwrap());
1570         }
1571         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1572         for tarball in &tarballs[1..] {
1573             input_tarballs.push(",");
1574             input_tarballs.push(tarball);
1575         }
1576
1577         let mut cmd = rust_installer(builder);
1578         cmd.arg("combine")
1579             .arg("--product-name=Rust")
1580             .arg("--rel-manifest-dir=rustlib")
1581             .arg("--success-message=Rust-is-ready-to-roll.")
1582             .arg("--work-dir").arg(&work)
1583             .arg("--output-dir").arg(&distdir(builder))
1584             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1585             .arg("--legacy-manifest-dirs=rustlib,cargo")
1586             .arg("--input-tarballs").arg(input_tarballs)
1587             .arg("--non-installed-overlay").arg(&overlay);
1588         builder.run(&mut cmd);
1589
1590         let mut license = String::new();
1591         license += &builder.read(&builder.src.join("COPYRIGHT"));
1592         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1593         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1594         license.push_str("\n");
1595         license.push_str("\n");
1596
1597         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1598         let mut rtf = rtf.to_string();
1599         rtf.push_str("\n");
1600         for line in license.lines() {
1601             rtf.push_str(line);
1602             rtf.push_str("\\line ");
1603         }
1604         rtf.push_str("}");
1605
1606         fn filter(contents: &str, marker: &str) -> String {
1607             let start = format!("tool-{}-start", marker);
1608             let end = format!("tool-{}-end", marker);
1609             let mut lines = Vec::new();
1610             let mut omitted = false;
1611             for line in contents.lines() {
1612                 if line.contains(&start) {
1613                     omitted = true;
1614                 } else if line.contains(&end) {
1615                     omitted = false;
1616                 } else if !omitted {
1617                     lines.push(line);
1618                 }
1619             }
1620
1621             lines.join("\n")
1622         }
1623
1624         let xform = |p: &Path| {
1625             let mut contents = t!(fs::read_to_string(p));
1626             if rls_installer.is_none() {
1627                 contents = filter(&contents, "rls");
1628             }
1629             if clippy_installer.is_none() {
1630                 contents = filter(&contents, "clippy");
1631             }
1632             if miri_installer.is_none() {
1633                 contents = filter(&contents, "miri");
1634             }
1635             if rustfmt_installer.is_none() {
1636                 contents = filter(&contents, "rustfmt");
1637             }
1638             let ret = tmp.join(p.file_name().unwrap());
1639             t!(fs::write(&ret, &contents));
1640             ret
1641         };
1642
1643         if target.contains("apple-darwin") {
1644             let pkg = tmp.join("pkg");
1645             let _ = fs::remove_dir_all(&pkg);
1646
1647             let pkgbuild = |component: &str| {
1648                 let mut cmd = Command::new("pkgbuild");
1649                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1650                     .arg("--scripts").arg(pkg.join(component))
1651                     .arg("--nopayload")
1652                     .arg(pkg.join(component).with_extension("pkg"));
1653                 builder.run(&mut cmd);
1654             };
1655
1656             let prepare = |name: &str| {
1657                 builder.create_dir(&pkg.join(name));
1658                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1659                         &pkg.join(name));
1660                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1661                 pkgbuild(name);
1662             };
1663             prepare("rustc");
1664             prepare("cargo");
1665             prepare("rust-docs");
1666             prepare("rust-std");
1667             prepare("rust-analysis");
1668
1669             if rls_installer.is_some() {
1670                 prepare("rls");
1671             }
1672             if clippy_installer.is_some() {
1673                 prepare("clippy");
1674             }
1675             if miri_installer.is_some() {
1676                 prepare("miri");
1677             }
1678
1679             // create an 'uninstall' package
1680             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1681             pkgbuild("uninstall");
1682
1683             builder.create_dir(&pkg.join("res"));
1684             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1685             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1686             let mut cmd = Command::new("productbuild");
1687             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1688                 .arg("--resources").arg(pkg.join("res"))
1689                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1690                                                     pkgname(builder, "rust"),
1691                                                     target)))
1692                 .arg("--package-path").arg(&pkg);
1693             builder.run(&mut cmd);
1694         }
1695
1696         if target.contains("windows") {
1697             let exe = tmp.join("exe");
1698             let _ = fs::remove_dir_all(&exe);
1699
1700             let prepare = |name: &str| {
1701                 builder.create_dir(&exe.join(name));
1702                 let dir = if name == "rust-std" || name == "rust-analysis" {
1703                     format!("{}-{}", name, target)
1704                 } else if name == "rls" {
1705                     "rls-preview".to_string()
1706                 } else if name == "clippy" {
1707                     "clippy-preview".to_string()
1708                 } else if name == "miri" {
1709                     "miri-preview".to_string()
1710                 } else {
1711                     name.to_string()
1712                 };
1713                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1714                             .join(dir),
1715                         &exe.join(name));
1716                 builder.remove(&exe.join(name).join("manifest.in"));
1717             };
1718             prepare("rustc");
1719             prepare("cargo");
1720             prepare("rust-analysis");
1721             prepare("rust-docs");
1722             prepare("rust-std");
1723             if rls_installer.is_some() {
1724                 prepare("rls");
1725             }
1726             if clippy_installer.is_some() {
1727                 prepare("clippy");
1728             }
1729             if miri_installer.is_some() {
1730                 prepare("miri");
1731             }
1732             if target.contains("windows-gnu") {
1733                 prepare("rust-mingw");
1734             }
1735
1736             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1737             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1738             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1739             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1740             builder.create(&exe.join("LICENSE.txt"), &license);
1741
1742             // Generate exe installer
1743             let mut cmd = Command::new("iscc");
1744             cmd.arg("rust.iss")
1745                 .current_dir(&exe);
1746             if target.contains("windows-gnu") {
1747                 cmd.arg("/dMINGW");
1748             }
1749             add_env(builder, &mut cmd, target);
1750             builder.run(&mut cmd);
1751             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1752                     &distdir(builder),
1753                     0o755);
1754
1755             // Generate msi installer
1756             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1757             let heat = wix.join("bin/heat.exe");
1758             let candle = wix.join("bin/candle.exe");
1759             let light = wix.join("bin/light.exe");
1760
1761             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1762             builder.run(Command::new(&heat)
1763                             .current_dir(&exe)
1764                             .arg("dir")
1765                             .arg("rustc")
1766                             .args(&heat_flags)
1767                             .arg("-cg").arg("RustcGroup")
1768                             .arg("-dr").arg("Rustc")
1769                             .arg("-var").arg("var.RustcDir")
1770                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1771             builder.run(Command::new(&heat)
1772                             .current_dir(&exe)
1773                             .arg("dir")
1774                             .arg("rust-docs")
1775                             .args(&heat_flags)
1776                             .arg("-cg").arg("DocsGroup")
1777                             .arg("-dr").arg("Docs")
1778                             .arg("-var").arg("var.DocsDir")
1779                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1780                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1781             builder.run(Command::new(&heat)
1782                             .current_dir(&exe)
1783                             .arg("dir")
1784                             .arg("cargo")
1785                             .args(&heat_flags)
1786                             .arg("-cg").arg("CargoGroup")
1787                             .arg("-dr").arg("Cargo")
1788                             .arg("-var").arg("var.CargoDir")
1789                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1790                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1791             builder.run(Command::new(&heat)
1792                             .current_dir(&exe)
1793                             .arg("dir")
1794                             .arg("rust-std")
1795                             .args(&heat_flags)
1796                             .arg("-cg").arg("StdGroup")
1797                             .arg("-dr").arg("Std")
1798                             .arg("-var").arg("var.StdDir")
1799                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1800             if rls_installer.is_some() {
1801                 builder.run(Command::new(&heat)
1802                                 .current_dir(&exe)
1803                                 .arg("dir")
1804                                 .arg("rls")
1805                                 .args(&heat_flags)
1806                                 .arg("-cg").arg("RlsGroup")
1807                                 .arg("-dr").arg("Rls")
1808                                 .arg("-var").arg("var.RlsDir")
1809                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1810                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1811             }
1812             if clippy_installer.is_some() {
1813                 builder.run(Command::new(&heat)
1814                                 .current_dir(&exe)
1815                                 .arg("dir")
1816                                 .arg("clippy")
1817                                 .args(&heat_flags)
1818                                 .arg("-cg").arg("ClippyGroup")
1819                                 .arg("-dr").arg("Clippy")
1820                                 .arg("-var").arg("var.ClippyDir")
1821                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1822                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1823             }
1824             if miri_installer.is_some() {
1825                 builder.run(Command::new(&heat)
1826                                 .current_dir(&exe)
1827                                 .arg("dir")
1828                                 .arg("miri")
1829                                 .args(&heat_flags)
1830                                 .arg("-cg").arg("MiriGroup")
1831                                 .arg("-dr").arg("Miri")
1832                                 .arg("-var").arg("var.MiriDir")
1833                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1834                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1835             }
1836             builder.run(Command::new(&heat)
1837                             .current_dir(&exe)
1838                             .arg("dir")
1839                             .arg("rust-analysis")
1840                             .args(&heat_flags)
1841                             .arg("-cg").arg("AnalysisGroup")
1842                             .arg("-dr").arg("Analysis")
1843                             .arg("-var").arg("var.AnalysisDir")
1844                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1845                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1846             if target.contains("windows-gnu") {
1847                 builder.run(Command::new(&heat)
1848                                 .current_dir(&exe)
1849                                 .arg("dir")
1850                                 .arg("rust-mingw")
1851                                 .args(&heat_flags)
1852                                 .arg("-cg").arg("GccGroup")
1853                                 .arg("-dr").arg("Gcc")
1854                                 .arg("-var").arg("var.GccDir")
1855                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1856             }
1857
1858             let candle = |input: &Path| {
1859                 let output = exe.join(input.file_stem().unwrap())
1860                                 .with_extension("wixobj");
1861                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1862                 let mut cmd = Command::new(&candle);
1863                 cmd.current_dir(&exe)
1864                     .arg("-nologo")
1865                     .arg("-dRustcDir=rustc")
1866                     .arg("-dDocsDir=rust-docs")
1867                     .arg("-dCargoDir=cargo")
1868                     .arg("-dStdDir=rust-std")
1869                     .arg("-dAnalysisDir=rust-analysis")
1870                     .arg("-arch").arg(&arch)
1871                     .arg("-out").arg(&output)
1872                     .arg(&input);
1873                 add_env(builder, &mut cmd, target);
1874
1875                 if rls_installer.is_some() {
1876                     cmd.arg("-dRlsDir=rls");
1877                 }
1878                 if clippy_installer.is_some() {
1879                     cmd.arg("-dClippyDir=clippy");
1880                 }
1881                 if miri_installer.is_some() {
1882                     cmd.arg("-dMiriDir=miri");
1883                 }
1884                 if target.contains("windows-gnu") {
1885                     cmd.arg("-dGccDir=rust-mingw");
1886                 }
1887                 builder.run(&mut cmd);
1888             };
1889             candle(&xform(&etc.join("msi/rust.wxs")));
1890             candle(&etc.join("msi/ui.wxs"));
1891             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1892             candle("RustcGroup.wxs".as_ref());
1893             candle("DocsGroup.wxs".as_ref());
1894             candle("CargoGroup.wxs".as_ref());
1895             candle("StdGroup.wxs".as_ref());
1896             if rls_installer.is_some() {
1897                 candle("RlsGroup.wxs".as_ref());
1898             }
1899             if clippy_installer.is_some() {
1900                 candle("ClippyGroup.wxs".as_ref());
1901             }
1902             if miri_installer.is_some() {
1903                 candle("MiriGroup.wxs".as_ref());
1904             }
1905             candle("AnalysisGroup.wxs".as_ref());
1906
1907             if target.contains("windows-gnu") {
1908                 candle("GccGroup.wxs".as_ref());
1909             }
1910
1911             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1912             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1913             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1914
1915             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1916             let mut cmd = Command::new(&light);
1917             cmd.arg("-nologo")
1918                 .arg("-ext").arg("WixUIExtension")
1919                 .arg("-ext").arg("WixUtilExtension")
1920                 .arg("-out").arg(exe.join(&filename))
1921                 .arg("rust.wixobj")
1922                 .arg("ui.wixobj")
1923                 .arg("rustwelcomedlg.wixobj")
1924                 .arg("RustcGroup.wixobj")
1925                 .arg("DocsGroup.wixobj")
1926                 .arg("CargoGroup.wixobj")
1927                 .arg("StdGroup.wixobj")
1928                 .arg("AnalysisGroup.wixobj")
1929                 .current_dir(&exe);
1930
1931             if rls_installer.is_some() {
1932                 cmd.arg("RlsGroup.wixobj");
1933             }
1934             if clippy_installer.is_some() {
1935                 cmd.arg("ClippyGroup.wixobj");
1936             }
1937             if miri_installer.is_some() {
1938                 cmd.arg("MiriGroup.wixobj");
1939             }
1940
1941             if target.contains("windows-gnu") {
1942                 cmd.arg("GccGroup.wixobj");
1943             }
1944             // ICE57 wrongly complains about the shortcuts
1945             cmd.arg("-sice:ICE57");
1946
1947             builder.run(&mut cmd);
1948
1949             if !builder.config.dry_run {
1950                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1951             }
1952         }
1953     }
1954 }
1955
1956 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1957     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1958     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1959        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1960        .env("CFG_RELEASE", builder.rust_release())
1961        .env("CFG_VER_MAJOR", parts.next().unwrap())
1962        .env("CFG_VER_MINOR", parts.next().unwrap())
1963        .env("CFG_VER_PATCH", parts.next().unwrap())
1964        .env("CFG_VER_BUILD", "0") // just needed to build
1965        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1966        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1967        .env("CFG_BUILD", target)
1968        .env("CFG_CHANNEL", &builder.config.channel);
1969
1970     if target.contains("windows-gnu") {
1971        cmd.env("CFG_MINGW", "1")
1972           .env("CFG_ABI", "GNU");
1973     } else {
1974        cmd.env("CFG_MINGW", "0")
1975           .env("CFG_ABI", "MSVC");
1976     }
1977
1978     if target.contains("x86_64") {
1979        cmd.env("CFG_PLATFORM", "x64");
1980     } else {
1981        cmd.env("CFG_PLATFORM", "x86");
1982     }
1983 }
1984
1985 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1986 pub struct HashSign;
1987
1988 impl Step for HashSign {
1989     type Output = ();
1990     const ONLY_HOSTS: bool = true;
1991
1992     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1993         run.path("hash-and-sign")
1994     }
1995
1996     fn make_run(run: RunConfig<'_>) {
1997         run.builder.ensure(HashSign);
1998     }
1999
2000     fn run(self, builder: &Builder<'_>) {
2001         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2002         if builder.config.dry_run {
2003             return;
2004         }
2005         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2006             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2007         });
2008         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2009             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2010         });
2011         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2012             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2013         });
2014         let pass = t!(fs::read_to_string(&file));
2015
2016         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2017
2018         cmd.arg(sign);
2019         cmd.arg(distdir(builder));
2020         cmd.arg(today.trim());
2021         cmd.arg(builder.rust_package_vers());
2022         cmd.arg(addr);
2023         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2024         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2025         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2026         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2027         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2028         cmd.arg(builder.llvm_tools_package_vers());
2029         cmd.arg(builder.lldb_package_vers());
2030
2031         builder.create_dir(&distdir(builder));
2032
2033         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2034         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2035         let status = t!(child.wait());
2036         assert!(status.success());
2037     }
2038 }
2039
2040 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2041 // LLVM tools are linked dynamically.
2042 // Note: This function does no yet support Windows but we also don't support
2043 //       linking LLVM tools dynamically on Windows yet.
2044 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2045                                 target: Interned<String>,
2046                                 sysroot: &Path) {
2047     let src_libdir = builder
2048         .llvm_out(target)
2049         .join("lib");
2050     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2051     t!(fs::create_dir_all(&dst_libdir));
2052
2053     if target.contains("apple-darwin") {
2054         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2055         if llvm_dylib_path.exists() {
2056             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2057         }
2058         return
2059     }
2060
2061     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2062     // Since tools link to the latter rather than the former, we have to
2063     // follow the symlink to find out what to distribute.
2064     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2065     if llvm_dylib_path.exists() {
2066         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2067             panic!("dist: Error calling canonicalize path `{}`: {}",
2068                    llvm_dylib_path.display(), e);
2069         });
2070
2071
2072         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2073     }
2074 }
2075
2076 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2077 pub struct LlvmTools {
2078     pub target: Interned<String>,
2079 }
2080
2081 impl Step for LlvmTools {
2082     type Output = Option<PathBuf>;
2083     const ONLY_HOSTS: bool = true;
2084
2085     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2086         run.path("llvm-tools")
2087     }
2088
2089     fn make_run(run: RunConfig<'_>) {
2090         run.builder.ensure(LlvmTools {
2091             target: run.target,
2092         });
2093     }
2094
2095     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2096         let target = self.target;
2097         assert!(builder.config.extended);
2098
2099         /* run only if llvm-config isn't used */
2100         if let Some(config) = builder.config.target_config.get(&target) {
2101             if let Some(ref _s) = config.llvm_config {
2102                 builder.info(&format!("Skipping LlvmTools ({}): external LLVM",
2103                     target));
2104                 return None;
2105             }
2106         }
2107
2108         builder.info(&format!("Dist LlvmTools ({})", target));
2109         let src = builder.src.join("src/llvm-project/llvm");
2110         let name = pkgname(builder, "llvm-tools");
2111
2112         let tmp = tmpdir(builder);
2113         let image = tmp.join("llvm-tools-image");
2114         drop(fs::remove_dir_all(&image));
2115
2116         // Prepare the image directory
2117         let src_bindir = builder
2118             .llvm_out(target)
2119             .join("bin");
2120         let dst_bindir = image.join("lib/rustlib")
2121             .join(&*target)
2122             .join("bin");
2123         t!(fs::create_dir_all(&dst_bindir));
2124         for tool in LLVM_TOOLS {
2125             let exe = src_bindir.join(exe(tool, &target));
2126             builder.install(&exe, &dst_bindir, 0o755);
2127         }
2128
2129         // Prepare the overlay
2130         let overlay = tmp.join("llvm-tools-overlay");
2131         drop(fs::remove_dir_all(&overlay));
2132         builder.create_dir(&overlay);
2133         builder.install(&src.join("README.txt"), &overlay, 0o644);
2134         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2135         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2136
2137         // Generate the installer tarball
2138         let mut cmd = rust_installer(builder);
2139         cmd.arg("generate")
2140             .arg("--product-name=Rust")
2141             .arg("--rel-manifest-dir=rustlib")
2142             .arg("--success-message=llvm-tools-installed.")
2143             .arg("--image-dir").arg(&image)
2144             .arg("--work-dir").arg(&tmpdir(builder))
2145             .arg("--output-dir").arg(&distdir(builder))
2146             .arg("--non-installed-overlay").arg(&overlay)
2147             .arg(format!("--package-name={}-{}", name, target))
2148             .arg("--legacy-manifest-dirs=rustlib,cargo")
2149             .arg("--component-name=llvm-tools-preview");
2150
2151
2152         builder.run(&mut cmd);
2153         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2154     }
2155 }
2156
2157 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2158 pub struct Lldb {
2159     pub target: Interned<String>,
2160 }
2161
2162 impl Step for Lldb {
2163     type Output = Option<PathBuf>;
2164     const ONLY_HOSTS: bool = true;
2165     const DEFAULT: bool = true;
2166
2167     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2168         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2169     }
2170
2171     fn make_run(run: RunConfig<'_>) {
2172         run.builder.ensure(Lldb {
2173             target: run.target,
2174         });
2175     }
2176
2177     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2178         let target = self.target;
2179
2180         if builder.config.dry_run {
2181             return None;
2182         }
2183
2184         let bindir = builder
2185             .llvm_out(target)
2186             .join("bin");
2187         let lldb_exe = bindir.join(exe("lldb", &target));
2188         if !lldb_exe.exists() {
2189             return None;
2190         }
2191
2192         builder.info(&format!("Dist Lldb ({})", target));
2193         let src = builder.src.join("src/llvm-project/lldb");
2194         let name = pkgname(builder, "lldb");
2195
2196         let tmp = tmpdir(builder);
2197         let image = tmp.join("lldb-image");
2198         drop(fs::remove_dir_all(&image));
2199
2200         // Prepare the image directory
2201         let root = image.join("lib/rustlib").join(&*target);
2202         let dst = root.join("bin");
2203         t!(fs::create_dir_all(&dst));
2204         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2205             let exe = bindir.join(exe(program, &target));
2206             builder.install(&exe, &dst, 0o755);
2207         }
2208
2209         // The libraries.
2210         let libdir = builder.llvm_out(target).join("lib");
2211         let dst = root.join("lib");
2212         t!(fs::create_dir_all(&dst));
2213         for entry in t!(fs::read_dir(&libdir)) {
2214             let entry = entry.unwrap();
2215             if let Ok(name) = entry.file_name().into_string() {
2216                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2217                     if t!(entry.file_type()).is_symlink() {
2218                         builder.copy_to_folder(&entry.path(), &dst);
2219                     } else {
2220                        builder.install(&entry.path(), &dst, 0o755);
2221                     }
2222                 }
2223             }
2224         }
2225
2226         // The lldb scripts might be installed in lib/python$version
2227         // or in lib64/python$version.  If lib64 exists, use it;
2228         // otherwise lib.
2229         let libdir = builder.llvm_out(target).join("lib64");
2230         let (libdir, libdir_name) = if libdir.exists() {
2231             (libdir, "lib64")
2232         } else {
2233             (builder.llvm_out(target).join("lib"), "lib")
2234         };
2235         for entry in t!(fs::read_dir(&libdir)) {
2236             let entry = t!(entry);
2237             if let Ok(name) = entry.file_name().into_string() {
2238                 if name.starts_with("python") {
2239                     let dst = root.join(libdir_name)
2240                         .join(entry.file_name());
2241                     t!(fs::create_dir_all(&dst));
2242                     builder.cp_r(&entry.path(), &dst);
2243                     break;
2244                 }
2245             }
2246         }
2247
2248         // Prepare the overlay
2249         let overlay = tmp.join("lldb-overlay");
2250         drop(fs::remove_dir_all(&overlay));
2251         builder.create_dir(&overlay);
2252         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2253         builder.create(&overlay.join("version"), &builder.lldb_vers());
2254
2255         // Generate the installer tarball
2256         let mut cmd = rust_installer(builder);
2257         cmd.arg("generate")
2258             .arg("--product-name=Rust")
2259             .arg("--rel-manifest-dir=rustlib")
2260             .arg("--success-message=lldb-installed.")
2261             .arg("--image-dir").arg(&image)
2262             .arg("--work-dir").arg(&tmpdir(builder))
2263             .arg("--output-dir").arg(&distdir(builder))
2264             .arg("--non-installed-overlay").arg(&overlay)
2265             .arg(format!("--package-name={}-{}", name, target))
2266             .arg("--legacy-manifest-dirs=rustlib,cargo")
2267             .arg("--component-name=lldb-preview");
2268
2269
2270         builder.run(&mut cmd);
2271         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2272     }
2273 }