src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::{output, t};
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub stage: u32,
  72     pub host: Interned<String>,
  73 }
  74
  75 impl Step for Docs {
  76     type Output = PathBuf;
  77     const DEFAULT: bool = true;
  78
  79     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  80         run.path("src/doc")
  81     }
  82
  83     fn make_run(run: RunConfig<'_>) {
  84         run.builder.ensure(Docs {
  85             stage: run.builder.top_stage,
  86             host: run.target,
  87         });
  88     }
  89
  90     /// Builds the `rust-docs` installer component.
  91     fn run(self, builder: &Builder<'_>) -> PathBuf {
  92         let host = self.host;
  93
  94         let name = pkgname(builder, "rust-docs");
  95
  96         builder.info(&format!("Dist docs ({})", host));
  97         if !builder.config.docs {
  98             builder.info("\tskipping - docs disabled");
  99             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 100         }
 101
 102         builder.default_doc(None);
 103
 104         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 105         let _ = fs::remove_dir_all(&image);
 106
 107         let dst = image.join("share/doc/rust/html");
 108         t!(fs::create_dir_all(&dst));
 109         let src = builder.doc_out(host);
 110         builder.cp_r(&src, &dst);
 111
 112         let mut cmd = rust_installer(builder);
 113         cmd.arg("generate")
 114            .arg("--product-name=Rust-Documentation")
 115            .arg("--rel-manifest-dir=rustlib")
 116            .arg("--success-message=Rust-documentation-is-installed.")
 117            .arg("--image-dir").arg(&image)
 118            .arg("--work-dir").arg(&tmpdir(builder))
 119            .arg("--output-dir").arg(&distdir(builder))
 120            .arg(format!("--package-name={}-{}", name, host))
 121            .arg("--component-name=rust-docs")
 122            .arg("--legacy-manifest-dirs=rustlib,cargo")
 123            .arg("--bulk-dirs=share/doc/rust/html");
 124         builder.run(&mut cmd);
 125         builder.remove_dir(&image);
 126
 127         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 128     }
 129 }
 130
 131 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 132 pub struct RustcDocs {
 133     pub stage: u32,
 134     pub host: Interned<String>,
 135 }
 136
 137 impl Step for RustcDocs {
 138     type Output = PathBuf;
 139     const DEFAULT: bool = true;
 140
 141     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 142         run.path("src/librustc")
 143     }
 144
 145     fn make_run(run: RunConfig<'_>) {
 146         run.builder.ensure(RustcDocs {
 147             stage: run.builder.top_stage,
 148             host: run.target,
 149         });
 150     }
 151
 152     /// Builds the `rustc-docs` installer component.
 153     fn run(self, builder: &Builder<'_>) -> PathBuf {
 154         let host = self.host;
 155
 156         let name = pkgname(builder, "rustc-docs");
 157
 158         builder.info(&format!("Dist compiler docs ({})", host));
 159         if !builder.config.compiler_docs {
 160             builder.info("\tskipping - compiler docs disabled");
 161             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 162         }
 163
 164         builder.default_doc(None);
 165
 166         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 167         let _ = fs::remove_dir_all(&image);
 168
 169         let dst = image.join("share/doc/rust/html");
 170         t!(fs::create_dir_all(&dst));
 171         let src = builder.compiler_doc_out(host);
 172         builder.cp_r(&src, &dst);
 173
 174         let mut cmd = rust_installer(builder);
 175         cmd.arg("generate")
 176            .arg("--product-name=Rustc-Documentation")
 177            .arg("--rel-manifest-dir=rustlib")
 178            .arg("--success-message=Rustc-documentation-is-installed.")
 179            .arg("--image-dir").arg(&image)
 180            .arg("--work-dir").arg(&tmpdir(builder))
 181            .arg("--output-dir").arg(&distdir(builder))
 182            .arg(format!("--package-name={}-{}", name, host))
 183            .arg("--component-name=rustc-docs")
 184            .arg("--legacy-manifest-dirs=rustlib,cargo")
 185            .arg("--bulk-dirs=share/doc/rust/html");
 186         builder.run(&mut cmd);
 187         builder.remove_dir(&image);
 188
 189         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 190     }
 191 }
 192
 193 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 194     let mut found = Vec::with_capacity(files.len());
 195
 196     for file in files {
 197         let file_path =
 198             path.iter()
 199                 .map(|dir| dir.join(file))
 200                 .find(|p| p.exists());
 201
 202         if let Some(file_path) = file_path {
 203             found.push(file_path);
 204         } else {
 205             panic!("Could not find '{}' in {:?}", file, path);
 206         }
 207     }
 208
 209     found
 210 }
 211
 212 fn make_win_dist(
 213     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 214 ) {
 215     //Ask gcc where it keeps its stuff
 216     let mut cmd = Command::new(builder.cc(target_triple));
 217     cmd.arg("-print-search-dirs");
 218     let gcc_out = output(&mut cmd);
 219
 220     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 221     let mut lib_path = Vec::new();
 222
 223     for line in gcc_out.lines() {
 224         let idx = line.find(':').unwrap();
 225         let key = &line[..idx];
 226         let trim_chars: &[_] = &[' ', '='];
 227         let value =
 228             line[(idx + 1)..]
 229                 .trim_start_matches(trim_chars)
 230                 .split(';')
 231                 .map(PathBuf::from);
 232
 233         if key == "programs" {
 234             bin_path.extend(value);
 235         } else if key == "libraries" {
 236             lib_path.extend(value);
 237         }
 238     }
 239
 240     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 241     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 242     if target_triple.starts_with("i686-") {
 243         rustc_dlls.push("libgcc_s_dw2-1.dll");
 244     } else {
 245         rustc_dlls.push("libgcc_s_seh-1.dll");
 246     }
 247
 248     let target_libs = [ //MinGW libs
 249         "libgcc.a",
 250         "libgcc_eh.a",
 251         "libgcc_s.a",
 252         "libm.a",
 253         "libmingw32.a",
 254         "libmingwex.a",
 255         "libstdc++.a",
 256         "libiconv.a",
 257         "libmoldname.a",
 258         "libpthread.a",
 259         //Windows import libs
 260         "libadvapi32.a",
 261         "libbcrypt.a",
 262         "libcomctl32.a",
 263         "libcomdlg32.a",
 264         "libcredui.a",
 265         "libcrypt32.a",
 266         "libdbghelp.a",
 267         "libgdi32.a",
 268         "libimagehlp.a",
 269         "libiphlpapi.a",
 270         "libkernel32.a",
 271         "libmsimg32.a",
 272         "libmsvcrt.a",
 273         "libodbc32.a",
 274         "libole32.a",
 275         "liboleaut32.a",
 276         "libopengl32.a",
 277         "libpsapi.a",
 278         "librpcrt4.a",
 279         "libsecur32.a",
 280         "libsetupapi.a",
 281         "libshell32.a",
 282         "libsynchronization.a",
 283         "libuser32.a",
 284         "libuserenv.a",
 285         "libuuid.a",
 286         "libwinhttp.a",
 287         "libwinmm.a",
 288         "libwinspool.a",
 289         "libws2_32.a",
 290         "libwsock32.a",
 291     ];
 292
 293     //Find mingw artifacts we want to bundle
 294     let target_tools = find_files(&target_tools, &bin_path);
 295     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 296     let target_libs = find_files(&target_libs, &lib_path);
 297
 298     // Copy runtime dlls next to rustc.exe
 299     let dist_bin_dir = rust_root.join("bin/");
 300     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 301     for src in rustc_dlls {
 302         builder.copy_to_folder(&src, &dist_bin_dir);
 303     }
 304
 305     //Copy platform tools to platform-specific bin directory
 306     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 307     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 308     for src in target_tools {
 309         builder.copy_to_folder(&src, &target_bin_dir);
 310     }
 311
 312     // Warn windows-gnu users that the bundled GCC cannot compile C files
 313     builder.create(
 314         &target_bin_dir.join("GCC-WARNING.txt"),
 315         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 316          used as a linker. In order to be able to compile projects containing C code use\
 317          the GCC provided by MinGW or Cygwin."
 318     );
 319
 320     //Copy platform libs to platform-specific lib directory
 321     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 322     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 323     for src in target_libs {
 324         builder.copy_to_folder(&src, &target_lib_dir);
 325     }
 326 }
 327
 328 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 329 pub struct Mingw {
 330     pub host: Interned<String>,
 331 }
 332
 333 impl Step for Mingw {
 334     type Output = Option<PathBuf>;
 335     const DEFAULT: bool = true;
 336
 337     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 338         run.never()
 339     }
 340
 341     fn make_run(run: RunConfig<'_>) {
 342         run.builder.ensure(Mingw { host: run.target });
 343     }
 344
 345     /// Builds the `rust-mingw` installer component.
 346     ///
 347     /// This contains all the bits and pieces to run the MinGW Windows targets
 348     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 349     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 350         let host = self.host;
 351
 352         if !host.contains("pc-windows-gnu") {
 353             return None;
 354         }
 355
 356         builder.info(&format!("Dist mingw ({})", host));
 357         let name = pkgname(builder, "rust-mingw");
 358         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 359         let _ = fs::remove_dir_all(&image);
 360         t!(fs::create_dir_all(&image));
 361
 362         // The first argument is a "temporary directory" which is just
 363         // thrown away (this contains the runtime DLLs included in the rustc package
 364         // above) and the second argument is where to place all the MinGW components
 365         // (which is what we want).
 366         make_win_dist(&tmpdir(builder), &image, host, &builder);
 367
 368         let mut cmd = rust_installer(builder);
 369         cmd.arg("generate")
 370            .arg("--product-name=Rust-MinGW")
 371            .arg("--rel-manifest-dir=rustlib")
 372            .arg("--success-message=Rust-MinGW-is-installed.")
 373            .arg("--image-dir").arg(&image)
 374            .arg("--work-dir").arg(&tmpdir(builder))
 375            .arg("--output-dir").arg(&distdir(builder))
 376            .arg(format!("--package-name={}-{}", name, host))
 377            .arg("--component-name=rust-mingw")
 378            .arg("--legacy-manifest-dirs=rustlib,cargo");
 379         builder.run(&mut cmd);
 380         t!(fs::remove_dir_all(&image));
 381         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 382     }
 383 }
 384
 385 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct Rustc {
 387     pub compiler: Compiler,
 388 }
 389
 390 impl Step for Rustc {
 391     type Output = PathBuf;
 392     const DEFAULT: bool = true;
 393     const ONLY_HOSTS: bool = true;
 394
 395     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 396         run.path("src/librustc")
 397     }
 398
 399     fn make_run(run: RunConfig<'_>) {
 400         run.builder.ensure(Rustc {
 401             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 402         });
 403     }
 404
 405     /// Creates the `rustc` installer component.
 406     fn run(self, builder: &Builder<'_>) -> PathBuf {
 407         let compiler = self.compiler;
 408         let host = self.compiler.host;
 409
 410         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 411         let name = pkgname(builder, "rustc");
 412         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 413         let _ = fs::remove_dir_all(&image);
 414         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 415         let _ = fs::remove_dir_all(&overlay);
 416
 417         // Prepare the rustc "image", what will actually end up getting installed
 418         prepare_image(builder, compiler, &image);
 419
 420         // Prepare the overlay which is part of the tarball but won't actually be
 421         // installed
 422         let cp = |file: &str| {
 423             builder.install(&builder.src.join(file), &overlay, 0o644);
 424         };
 425         cp("COPYRIGHT");
 426         cp("LICENSE-APACHE");
 427         cp("LICENSE-MIT");
 428         cp("README.md");
 429         // tiny morsel of metadata is used by rust-packaging
 430         let version = builder.rust_version();
 431         builder.create(&overlay.join("version"), &version);
 432         if let Some(sha) = builder.rust_sha() {
 433             builder.create(&overlay.join("git-commit-hash"), &sha);
 434         }
 435
 436         // On MinGW we've got a few runtime DLL dependencies that we need to
 437         // include. The first argument to this script is where to put these DLLs
 438         // (the image we're creating), and the second argument is a junk directory
 439         // to ignore all other MinGW stuff the script creates.
 440         //
 441         // On 32-bit MinGW we're always including a DLL which needs some extra
 442         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 443         // anything requiring us to distribute a license, but it's likely the
 444         // install will *also* include the rust-mingw package, which also needs
 445         // licenses, so to be safe we just include it here in all MinGW packages.
 446         if host.contains("pc-windows-gnu") {
 447             make_win_dist(&image, &tmpdir(builder), host, builder);
 448
 449             let dst = image.join("share/doc");
 450             t!(fs::create_dir_all(&dst));
 451             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 452         }
 453
 454         // Finally, wrap everything up in a nice tarball!
 455         let mut cmd = rust_installer(builder);
 456         cmd.arg("generate")
 457            .arg("--product-name=Rust")
 458            .arg("--rel-manifest-dir=rustlib")
 459            .arg("--success-message=Rust-is-ready-to-roll.")
 460            .arg("--image-dir").arg(&image)
 461            .arg("--work-dir").arg(&tmpdir(builder))
 462            .arg("--output-dir").arg(&distdir(builder))
 463            .arg("--non-installed-overlay").arg(&overlay)
 464            .arg(format!("--package-name={}-{}", name, host))
 465            .arg("--component-name=rustc")
 466            .arg("--legacy-manifest-dirs=rustlib,cargo");
 467         builder.run(&mut cmd);
 468         builder.remove_dir(&image);
 469         builder.remove_dir(&overlay);
 470
 471         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 472
 473         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 474             let host = compiler.host;
 475             let src = builder.sysroot(compiler);
 476             let libdir = builder.rustc_libdir(compiler);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);
 483
 484             let libdir_relative = builder.libdir_relative(compiler);
 485
 486             // Copy runtime DLLs needed by the compiler
 487             if libdir_relative.to_str() != Some("bin") {
 488                 for entry in builder.read_dir(&libdir) {
 489                     let name = entry.file_name();
 490                     if let Some(s) = name.to_str() {
 491                         if is_dylib(s) {
 492                             builder.install(&entry.path(), &image.join(&libdir_relative), 0o644);
 493                         }
 494                     }
 495                 }
 496             }
 497
 498             // Copy over the codegen backends
 499             let backends_src = builder.sysroot_codegen_backends(compiler);
 500             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 501             let backends_dst = image.join(&backends_rel);
 502             t!(fs::create_dir_all(&backends_dst));
 503             builder.cp_r(&backends_src, &backends_dst);
 504
 505             // Copy libLLVM.so to the lib dir as well, if needed. While not
 506             // technically needed by rustc itself it's needed by lots of other
 507             // components like the llvm tools and LLD. LLD is included below and
 508             // tools/LLDB come later, so let's just throw it in the rustc
 509             // component for now.
 510             maybe_install_llvm_dylib(builder, host, image);
 511
 512             // Copy over lld if it's there
 513             if builder.config.lld_enabled {
 514                 let exe = exe("rust-lld", &compiler.host);
 515                 let src = builder.sysroot_libdir(compiler, host)
 516                     .parent()
 517                     .unwrap()
 518                     .join("bin")
 519                     .join(&exe);
 520                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 521                 let dst = image.join("lib/rustlib")
 522                     .join(&*host)
 523                     .join("bin")
 524                     .join(&exe);
 525                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 526                 builder.copy(&src, &dst);
 527             }
 528
 529             // Man pages
 530             t!(fs::create_dir_all(image.join("share/man/man1")));
 531             let man_src = builder.src.join("src/doc/man");
 532             let man_dst = image.join("share/man/man1");
 533
 534             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 535             let time = env::var("SOURCE_DATE_EPOCH")
 536                 .map(|timestamp| {
 537                     let epoch = timestamp.parse().map_err(|err| {
 538                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 539                     }).unwrap();
 540
 541                     time::at(Timespec::new(epoch, 0))
 542                 })
 543                 .unwrap_or_else(|_| time::now());
 544
 545             let month_year = t!(time::strftime("%B %Y", &time));
 546             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 547             // to hardlink, and we don't want to edit the source templates
 548             for file_entry in builder.read_dir(&man_src) {
 549                 let page_src = file_entry.path();
 550                 let page_dst = man_dst.join(file_entry.file_name());
 551                 t!(fs::copy(&page_src, &page_dst));
 552                 // template in month/year and version number
 553                 builder.replace_in_file(&page_dst,
 554                                 &[("<INSERT DATE HERE>", &month_year),
 555                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 556             }
 557
 558             // Debugger scripts
 559             builder.ensure(DebuggerScripts {
 560                 sysroot: INTERNER.intern_path(image.to_owned()),
 561                 host,
 562             });
 563
 564             // Misc license info
 565             let cp = |file: &str| {
 566                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 567             };
 568             cp("COPYRIGHT");
 569             cp("LICENSE-APACHE");
 570             cp("LICENSE-MIT");
 571             cp("README.md");
 572         }
 573     }
 574 }
 575
 576 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 577 pub struct DebuggerScripts {
 578     pub sysroot: Interned<PathBuf>,
 579     pub host: Interned<String>,
 580 }
 581
 582 impl Step for DebuggerScripts {
 583     type Output = ();
 584
 585     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 586         run.path("src/lldb_batchmode.py")
 587     }
 588
 589     fn make_run(run: RunConfig<'_>) {
 590         run.builder.ensure(DebuggerScripts {
 591             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 592             host: run.target,
 593         });
 594     }
 595
 596     /// Copies debugger scripts for `target` into the `sysroot` specified.
 597     fn run(self, builder: &Builder<'_>) {
 598         let host = self.host;
 599         let sysroot = self.sysroot;
 600         let dst = sysroot.join("lib/rustlib/etc");
 601         t!(fs::create_dir_all(&dst));
 602         let cp_debugger_script = |file: &str| {
 603             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 604         };
 605         if host.contains("windows-msvc") {
 606             // windbg debugger scripts
 607             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 608                 0o755);
 609
 610             cp_debugger_script("natvis/intrinsic.natvis");
 611             cp_debugger_script("natvis/liballoc.natvis");
 612             cp_debugger_script("natvis/libcore.natvis");
 613         } else {
 614             cp_debugger_script("debugger_pretty_printers_common.py");
 615
 616             // gdb debugger scripts
 617             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 618                     0o755);
 619             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 620                     0o755);
 621
 622             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 623             cp_debugger_script("gdb_rust_pretty_printing.py");
 624
 625             // lldb debugger scripts
 626             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 627                     0o755);
 628
 629             cp_debugger_script("lldb_rust_formatters.py");
 630         }
 631     }
 632 }
 633
 634 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 635 pub struct Std {
 636     pub compiler: Compiler,
 637     pub target: Interned<String>,
 638 }
 639
 640 impl Step for Std {
 641     type Output = PathBuf;
 642     const DEFAULT: bool = true;
 643
 644     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 645         run.path("src/libstd")
 646     }
 647
 648     fn make_run(run: RunConfig<'_>) {
 649         run.builder.ensure(Std {
 650             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 651             target: run.target,
 652         });
 653     }
 654
 655     fn run(self, builder: &Builder<'_>) -> PathBuf {
 656         let compiler = self.compiler;
 657         let target = self.target;
 658
 659         let name = pkgname(builder, "rust-std");
 660         builder.info(&format!("Dist std stage{} ({} -> {})",
 661             compiler.stage, &compiler.host, target));
 662
 663         // The only true set of target libraries came from the build triple, so
 664         // let's reduce redundant work by only producing archives from that host.
 665         if compiler.host != builder.config.build {
 666             builder.info("\tskipping, not a build host");
 667             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 668         }
 669
 670         // We want to package up as many target libraries as possible
 671         // for the `rust-std` package, so if this is a host target we
 672         // depend on librustc and otherwise we just depend on libtest.
 673         if builder.hosts.iter().any(|t| t == target) {
 674             builder.ensure(compile::Rustc { compiler, target });
 675         } else {
 676             if builder.no_std(target) == Some(true) {
 677                 // the `test` doesn't compile for no-std targets
 678                 builder.ensure(compile::Std { compiler, target });
 679             } else {
 680                 builder.ensure(compile::Test { compiler, target });
 681             }
 682         }
 683
 684         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 685         let _ = fs::remove_dir_all(&image);
 686
 687         let dst = image.join("lib/rustlib").join(target);
 688         t!(fs::create_dir_all(&dst));
 689         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 690         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 691         builder.cp_filtered(&src, &dst, &|path| {
 692             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 693                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 694                     return false
 695                 }
 696                 if name == "bin" {
 697                     return false
 698                 }
 699                 if name.contains("LLVM") {
 700                     return false
 701                 }
 702             }
 703             true
 704         });
 705
 706         let mut cmd = rust_installer(builder);
 707         cmd.arg("generate")
 708            .arg("--product-name=Rust")
 709            .arg("--rel-manifest-dir=rustlib")
 710            .arg("--success-message=std-is-standing-at-the-ready.")
 711            .arg("--image-dir").arg(&image)
 712            .arg("--work-dir").arg(&tmpdir(builder))
 713            .arg("--output-dir").arg(&distdir(builder))
 714            .arg(format!("--package-name={}-{}", name, target))
 715            .arg(format!("--component-name=rust-std-{}", target))
 716            .arg("--legacy-manifest-dirs=rustlib,cargo");
 717         builder.run(&mut cmd);
 718         builder.remove_dir(&image);
 719         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 720     }
 721 }
 722
 723 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 724 pub struct Analysis {
 725     pub compiler: Compiler,
 726     pub target: Interned<String>,
 727 }
 728
 729 impl Step for Analysis {
 730     type Output = PathBuf;
 731     const DEFAULT: bool = true;
 732
 733     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 734         let builder = run.builder;
 735         run.path("analysis").default_condition(builder.config.extended)
 736     }
 737
 738     fn make_run(run: RunConfig<'_>) {
 739         run.builder.ensure(Analysis {
 740             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 741             target: run.target,
 742         });
 743     }
 744
 745     /// Creates a tarball of save-analysis metadata, if available.
 746     fn run(self, builder: &Builder<'_>) -> PathBuf {
 747         let compiler = self.compiler;
 748         let target = self.target;
 749         assert!(builder.config.extended);
 750         builder.info("Dist analysis");
 751         let name = pkgname(builder, "rust-analysis");
 752
 753         if &compiler.host != builder.config.build {
 754             builder.info("\tskipping, not a build host");
 755             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 756         }
 757
 758         builder.ensure(Std { compiler, target });
 759
 760         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 761         // stage2 artifacts is simply copied from stage1 in that case.
 762         let compiler = if builder.force_use_stage1(compiler, target) {
 763             builder.compiler(1, compiler.host)
 764         } else {
 765             compiler.clone()
 766         };
 767
 768         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 769
 770         let src = builder.stage_out(compiler, Mode::Std)
 771             .join(target).join(builder.cargo_dir()).join("deps");
 772
 773         let image_src = src.join("save-analysis");
 774         let dst = image.join("lib/rustlib").join(target).join("analysis");
 775         t!(fs::create_dir_all(&dst));
 776         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 777         builder.cp_r(&image_src, &dst);
 778
 779         let mut cmd = rust_installer(builder);
 780         cmd.arg("generate")
 781            .arg("--product-name=Rust")
 782            .arg("--rel-manifest-dir=rustlib")
 783            .arg("--success-message=save-analysis-saved.")
 784            .arg("--image-dir").arg(&image)
 785            .arg("--work-dir").arg(&tmpdir(builder))
 786            .arg("--output-dir").arg(&distdir(builder))
 787            .arg(format!("--package-name={}-{}", name, target))
 788            .arg(format!("--component-name=rust-analysis-{}", target))
 789            .arg("--legacy-manifest-dirs=rustlib,cargo");
 790         builder.run(&mut cmd);
 791         builder.remove_dir(&image);
 792         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 793     }
 794 }
 795
 796 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 797     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 798         let spath = match path.to_str() {
 799             Some(path) => path,
 800             None => return false,
 801         };
 802         if spath.ends_with("~") || spath.ends_with(".pyc") {
 803             return false
 804         }
 805
 806         const LLVM_PROJECTS: &[&str] = &[
 807             "llvm-project/clang", "llvm-project\\clang",
 808             "llvm-project/lld", "llvm-project\\lld",
 809             "llvm-project/lldb", "llvm-project\\lldb",
 810             "llvm-project/llvm", "llvm-project\\llvm",
 811         ];
 812         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 813             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 814         {
 815             return false;
 816         }
 817
 818         const LLVM_TEST: &[&str] = &[
 819             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 820             "llvm-emscripten/test", "llvm-emscripten\\test",
 821         ];
 822         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 823             (spath.ends_with(".ll") ||
 824              spath.ends_with(".td") ||
 825              spath.ends_with(".s")) {
 826             return false
 827         }
 828         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 829             return false
 830         }
 831
 832         let full_path = Path::new(dir).join(path);
 833         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 834             return false;
 835         }
 836
 837         let excludes = [
 838             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 839             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 840             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 841             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 842         ];
 843         !path.iter()
 844              .map(|s| s.to_str().unwrap())
 845              .any(|s| excludes.contains(&s))
 846     }
 847
 848     // Copy the directories using our filter
 849     for item in src_dirs {
 850         let dst = &dst_dir.join(item);
 851         t!(fs::create_dir_all(dst));
 852         builder.cp_filtered(
 853             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 854     }
 855 }
 856
 857 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 858 pub struct Src;
 859
 860 impl Step for Src {
 861     /// The output path of the src installer tarball
 862     type Output = PathBuf;
 863     const DEFAULT: bool = true;
 864     const ONLY_HOSTS: bool = true;
 865
 866     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 867         run.path("src")
 868     }
 869
 870     fn make_run(run: RunConfig<'_>) {
 871         run.builder.ensure(Src);
 872     }
 873
 874     /// Creates the `rust-src` installer component
 875     fn run(self, builder: &Builder<'_>) -> PathBuf {
 876         builder.info("Dist src");
 877
 878         let name = pkgname(builder, "rust-src");
 879         let image = tmpdir(builder).join(format!("{}-image", name));
 880         let _ = fs::remove_dir_all(&image);
 881
 882         let dst = image.join("lib/rustlib/src");
 883         let dst_src = dst.join("rust");
 884         t!(fs::create_dir_all(&dst_src));
 885
 886         let src_files = [
 887             "Cargo.lock",
 888         ];
 889         // This is the reduced set of paths which will become the rust-src component
 890         // (essentially libstd and all of its path dependencies)
 891         let std_src_dirs = [
 892             "src/build_helper",
 893             "src/liballoc",
 894             "src/libcore",
 895             "src/libpanic_abort",
 896             "src/libpanic_unwind",
 897             "src/librustc_asan",
 898             "src/librustc_lsan",
 899             "src/librustc_msan",
 900             "src/librustc_tsan",
 901             "src/libstd",
 902             "src/libunwind",
 903             "src/libtest",
 904             "src/libterm",
 905             "src/libprofiler_builtins",
 906             "src/stdsimd",
 907             "src/libproc_macro",
 908             "src/tools/rustc-std-workspace-core",
 909             "src/tools/rustc-std-workspace-alloc",
 910             "src/librustc",
 911             "src/libsyntax",
 912         ];
 913
 914         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 915         for file in src_files.iter() {
 916             builder.copy(&builder.src.join(file), &dst_src.join(file));
 917         }
 918
 919         // Create source tarball in rust-installer format
 920         let mut cmd = rust_installer(builder);
 921         cmd.arg("generate")
 922            .arg("--product-name=Rust")
 923            .arg("--rel-manifest-dir=rustlib")
 924            .arg("--success-message=Awesome-Source.")
 925            .arg("--image-dir").arg(&image)
 926            .arg("--work-dir").arg(&tmpdir(builder))
 927            .arg("--output-dir").arg(&distdir(builder))
 928            .arg(format!("--package-name={}", name))
 929            .arg("--component-name=rust-src")
 930            .arg("--legacy-manifest-dirs=rustlib,cargo");
 931         builder.run(&mut cmd);
 932
 933         builder.remove_dir(&image);
 934         distdir(builder).join(&format!("{}.tar.gz", name))
 935     }
 936 }
 937
 938 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 939
 940 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 941 pub struct PlainSourceTarball;
 942
 943 impl Step for PlainSourceTarball {
 944     /// Produces the location of the tarball generated
 945     type Output = PathBuf;
 946     const DEFAULT: bool = true;
 947     const ONLY_HOSTS: bool = true;
 948
 949     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 950         let builder = run.builder;
 951         run.path("src").default_condition(builder.config.rust_dist_src)
 952     }
 953
 954     fn make_run(run: RunConfig<'_>) {
 955         run.builder.ensure(PlainSourceTarball);
 956     }
 957
 958     /// Creates the plain source tarball
 959     fn run(self, builder: &Builder<'_>) -> PathBuf {
 960         builder.info("Create plain source tarball");
 961
 962         // Make sure that the root folder of tarball has the correct name
 963         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 964         let plain_dst_src = tmpdir(builder).join(&plain_name);
 965         let _ = fs::remove_dir_all(&plain_dst_src);
 966         t!(fs::create_dir_all(&plain_dst_src));
 967
 968         // This is the set of root paths which will become part of the source package
 969         let src_files = [
 970             "COPYRIGHT",
 971             "LICENSE-APACHE",
 972             "LICENSE-MIT",
 973             "CONTRIBUTING.md",
 974             "README.md",
 975             "RELEASES.md",
 976             "configure",
 977             "x.py",
 978             "config.toml.example",
 979             "Cargo.toml",
 980             "Cargo.lock",
 981         ];
 982         let src_dirs = [
 983             "src",
 984         ];
 985
 986         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 987
 988         // Copy the files normally
 989         for item in &src_files {
 990             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 991         }
 992
 993         // Create the version file
 994         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 995         if let Some(sha) = builder.rust_sha() {
 996             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 997         }
 998
 999         // If we're building from git sources, we need to vendor a complete distribution.
1000         if builder.rust_info.is_git() {
1001             // Get cargo-vendor installed, if it isn't already.
1002             let mut has_cargo_vendor = false;
1003             let mut cmd = Command::new(&builder.initial_cargo);
1004             for line in output(cmd.arg("install").arg("--list")).lines() {
1005                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
1006             }
1007             if !has_cargo_vendor {
1008                 let mut cmd = builder.cargo(
1009                     builder.compiler(0, builder.config.build),
1010                     Mode::ToolBootstrap,
1011                     builder.config.build,
1012                     "install"
1013                 );
1014                 cmd.arg("--force")
1015                    .arg("--debug")
1016                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1017                    .arg("cargo-vendor");
1018                 builder.run(&mut cmd);
1019             }
1020
1021             // Vendor all Cargo dependencies
1022             let mut cmd = Command::new(&builder.initial_cargo);
1023             cmd.arg("vendor")
1024                .current_dir(&plain_dst_src);
1025             builder.run(&mut cmd);
1026         }
1027
1028         // Create plain source tarball
1029         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1030         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1031         tarball.set_extension(""); // strip .gz
1032         tarball.set_extension(""); // strip .tar
1033         if let Some(dir) = tarball.parent() {
1034             builder.create_dir(&dir);
1035         }
1036         builder.info("running installer");
1037         let mut cmd = rust_installer(builder);
1038         cmd.arg("tarball")
1039            .arg("--input").arg(&plain_name)
1040            .arg("--output").arg(&tarball)
1041            .arg("--work-dir=.")
1042            .current_dir(tmpdir(builder));
1043         builder.run(&mut cmd);
1044         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1045     }
1046 }
1047
1048 // We have to run a few shell scripts, which choke quite a bit on both `\`
1049 // characters and on `C:\` paths, so normalize both of them away.
1050 pub fn sanitize_sh(path: &Path) -> String {
1051     let path = path.to_str().unwrap().replace("\\", "/");
1052     return change_drive(&path).unwrap_or(path);
1053
1054     fn change_drive(s: &str) -> Option<String> {
1055         let mut ch = s.chars();
1056         let drive = ch.next().unwrap_or('C');
1057         if ch.next() != Some(':') {
1058             return None
1059         }
1060         if ch.next() != Some('/') {
1061             return None
1062         }
1063         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1064     }
1065 }
1066
1067 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1068 pub struct Cargo {
1069     pub stage: u32,
1070     pub target: Interned<String>,
1071 }
1072
1073 impl Step for Cargo {
1074     type Output = PathBuf;
1075     const ONLY_HOSTS: bool = true;
1076
1077     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1078         run.path("cargo")
1079     }
1080
1081     fn make_run(run: RunConfig<'_>) {
1082         run.builder.ensure(Cargo {
1083             stage: run.builder.top_stage,
1084             target: run.target,
1085         });
1086     }
1087
1088     fn run(self, builder: &Builder<'_>) -> PathBuf {
1089         let stage = self.stage;
1090         let target = self.target;
1091
1092         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1093         let src = builder.src.join("src/tools/cargo");
1094         let etc = src.join("src/etc");
1095         let release_num = builder.release_num("cargo");
1096         let name = pkgname(builder, "cargo");
1097         let version = builder.cargo_info.version(builder, &release_num);
1098
1099         let tmp = tmpdir(builder);
1100         let image = tmp.join("cargo-image");
1101         drop(fs::remove_dir_all(&image));
1102         builder.create_dir(&image);
1103
1104         // Prepare the image directory
1105         builder.create_dir(&image.join("share/zsh/site-functions"));
1106         builder.create_dir(&image.join("etc/bash_completion.d"));
1107         let cargo = builder.ensure(tool::Cargo {
1108             compiler: builder.compiler(stage, builder.config.build),
1109             target
1110         });
1111         builder.install(&cargo, &image.join("bin"), 0o755);
1112         for man in t!(etc.join("man").read_dir()) {
1113             let man = t!(man);
1114             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1115         }
1116         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1117         builder.copy(&etc.join("cargo.bashcomp.sh"),
1118              &image.join("etc/bash_completion.d/cargo"));
1119         let doc = image.join("share/doc/cargo");
1120         builder.install(&src.join("README.md"), &doc, 0o644);
1121         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1122         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1123         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1124
1125         // Prepare the overlay
1126         let overlay = tmp.join("cargo-overlay");
1127         drop(fs::remove_dir_all(&overlay));
1128         builder.create_dir(&overlay);
1129         builder.install(&src.join("README.md"), &overlay, 0o644);
1130         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1131         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1132         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1133         builder.create(&overlay.join("version"), &version);
1134
1135         // Generate the installer tarball
1136         let mut cmd = rust_installer(builder);
1137         cmd.arg("generate")
1138            .arg("--product-name=Rust")
1139            .arg("--rel-manifest-dir=rustlib")
1140            .arg("--success-message=Rust-is-ready-to-roll.")
1141            .arg("--image-dir").arg(&image)
1142            .arg("--work-dir").arg(&tmpdir(builder))
1143            .arg("--output-dir").arg(&distdir(builder))
1144            .arg("--non-installed-overlay").arg(&overlay)
1145            .arg(format!("--package-name={}-{}", name, target))
1146            .arg("--component-name=cargo")
1147            .arg("--legacy-manifest-dirs=rustlib,cargo");
1148         builder.run(&mut cmd);
1149         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1150     }
1151 }
1152
1153 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1154 pub struct Rls {
1155     pub stage: u32,
1156     pub target: Interned<String>,
1157 }
1158
1159 impl Step for Rls {
1160     type Output = Option<PathBuf>;
1161     const ONLY_HOSTS: bool = true;
1162
1163     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1164         run.path("rls")
1165     }
1166
1167     fn make_run(run: RunConfig<'_>) {
1168         run.builder.ensure(Rls {
1169             stage: run.builder.top_stage,
1170             target: run.target,
1171         });
1172     }
1173
1174     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1175         let stage = self.stage;
1176         let target = self.target;
1177         assert!(builder.config.extended);
1178
1179         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1180         let src = builder.src.join("src/tools/rls");
1181         let release_num = builder.release_num("rls");
1182         let name = pkgname(builder, "rls");
1183         let version = builder.rls_info.version(builder, &release_num);
1184
1185         let tmp = tmpdir(builder);
1186         let image = tmp.join("rls-image");
1187         drop(fs::remove_dir_all(&image));
1188         t!(fs::create_dir_all(&image));
1189
1190         // Prepare the image directory
1191         // We expect RLS to build, because we've exited this step above if tool
1192         // state for RLS isn't testing.
1193         let rls = builder.ensure(tool::Rls {
1194             compiler: builder.compiler(stage, builder.config.build),
1195             target, extra_features: Vec::new()
1196         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1197
1198         builder.install(&rls, &image.join("bin"), 0o755);
1199         let doc = image.join("share/doc/rls");
1200         builder.install(&src.join("README.md"), &doc, 0o644);
1201         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1202         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1203
1204         // Prepare the overlay
1205         let overlay = tmp.join("rls-overlay");
1206         drop(fs::remove_dir_all(&overlay));
1207         t!(fs::create_dir_all(&overlay));
1208         builder.install(&src.join("README.md"), &overlay, 0o644);
1209         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1210         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1211         builder.create(&overlay.join("version"), &version);
1212
1213         // Generate the installer tarball
1214         let mut cmd = rust_installer(builder);
1215         cmd.arg("generate")
1216            .arg("--product-name=Rust")
1217            .arg("--rel-manifest-dir=rustlib")
1218            .arg("--success-message=RLS-ready-to-serve.")
1219            .arg("--image-dir").arg(&image)
1220            .arg("--work-dir").arg(&tmpdir(builder))
1221            .arg("--output-dir").arg(&distdir(builder))
1222            .arg("--non-installed-overlay").arg(&overlay)
1223            .arg(format!("--package-name={}-{}", name, target))
1224            .arg("--legacy-manifest-dirs=rustlib,cargo")
1225            .arg("--component-name=rls-preview");
1226
1227         builder.run(&mut cmd);
1228         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1229     }
1230 }
1231
1232 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1233 pub struct Clippy {
1234     pub stage: u32,
1235     pub target: Interned<String>,
1236 }
1237
1238 impl Step for Clippy {
1239     type Output = Option<PathBuf>;
1240     const ONLY_HOSTS: bool = true;
1241
1242     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1243         run.path("clippy")
1244     }
1245
1246     fn make_run(run: RunConfig<'_>) {
1247         run.builder.ensure(Clippy {
1248             stage: run.builder.top_stage,
1249             target: run.target,
1250         });
1251     }
1252
1253     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1254         let stage = self.stage;
1255         let target = self.target;
1256         assert!(builder.config.extended);
1257
1258         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1259         let src = builder.src.join("src/tools/clippy");
1260         let release_num = builder.release_num("clippy");
1261         let name = pkgname(builder, "clippy");
1262         let version = builder.clippy_info.version(builder, &release_num);
1263
1264         let tmp = tmpdir(builder);
1265         let image = tmp.join("clippy-image");
1266         drop(fs::remove_dir_all(&image));
1267         builder.create_dir(&image);
1268
1269         // Prepare the image directory
1270         // We expect clippy to build, because we've exited this step above if tool
1271         // state for clippy isn't testing.
1272         let clippy = builder.ensure(tool::Clippy {
1273             compiler: builder.compiler(stage, builder.config.build),
1274             target, extra_features: Vec::new()
1275         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1276         let cargoclippy = builder.ensure(tool::CargoClippy {
1277             compiler: builder.compiler(stage, builder.config.build),
1278             target, extra_features: Vec::new()
1279         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1280
1281         builder.install(&clippy, &image.join("bin"), 0o755);
1282         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1283         let doc = image.join("share/doc/clippy");
1284         builder.install(&src.join("README.md"), &doc, 0o644);
1285         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1286         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1287
1288         // Prepare the overlay
1289         let overlay = tmp.join("clippy-overlay");
1290         drop(fs::remove_dir_all(&overlay));
1291         t!(fs::create_dir_all(&overlay));
1292         builder.install(&src.join("README.md"), &overlay, 0o644);
1293         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1294         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1295         builder.create(&overlay.join("version"), &version);
1296
1297         // Generate the installer tarball
1298         let mut cmd = rust_installer(builder);
1299         cmd.arg("generate")
1300            .arg("--product-name=Rust")
1301            .arg("--rel-manifest-dir=rustlib")
1302            .arg("--success-message=clippy-ready-to-serve.")
1303            .arg("--image-dir").arg(&image)
1304            .arg("--work-dir").arg(&tmpdir(builder))
1305            .arg("--output-dir").arg(&distdir(builder))
1306            .arg("--non-installed-overlay").arg(&overlay)
1307            .arg(format!("--package-name={}-{}", name, target))
1308            .arg("--legacy-manifest-dirs=rustlib,cargo")
1309            .arg("--component-name=clippy-preview");
1310
1311         builder.run(&mut cmd);
1312         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1313     }
1314 }
1315
1316 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1317 pub struct Miri {
1318     pub stage: u32,
1319     pub target: Interned<String>,
1320 }
1321
1322 impl Step for Miri {
1323     type Output = Option<PathBuf>;
1324     const ONLY_HOSTS: bool = true;
1325
1326     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1327         run.path("miri")
1328     }
1329
1330     fn make_run(run: RunConfig<'_>) {
1331         run.builder.ensure(Miri {
1332             stage: run.builder.top_stage,
1333             target: run.target,
1334         });
1335     }
1336
1337     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1338         let stage = self.stage;
1339         let target = self.target;
1340         assert!(builder.config.extended);
1341
1342         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1343         let src = builder.src.join("src/tools/miri");
1344         let release_num = builder.release_num("miri");
1345         let name = pkgname(builder, "miri");
1346         let version = builder.miri_info.version(builder, &release_num);
1347
1348         let tmp = tmpdir(builder);
1349         let image = tmp.join("miri-image");
1350         drop(fs::remove_dir_all(&image));
1351         builder.create_dir(&image);
1352
1353         // Prepare the image directory
1354         // We expect miri to build, because we've exited this step above if tool
1355         // state for miri isn't testing.
1356         let miri = builder.ensure(tool::Miri {
1357             compiler: builder.compiler(stage, builder.config.build),
1358             target, extra_features: Vec::new()
1359         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1360         let cargomiri = builder.ensure(tool::CargoMiri {
1361             compiler: builder.compiler(stage, builder.config.build),
1362             target, extra_features: Vec::new()
1363         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1364
1365         builder.install(&miri, &image.join("bin"), 0o755);
1366         builder.install(&cargomiri, &image.join("bin"), 0o755);
1367         let doc = image.join("share/doc/miri");
1368         builder.install(&src.join("README.md"), &doc, 0o644);
1369         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1370         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1371
1372         // Prepare the overlay
1373         let overlay = tmp.join("miri-overlay");
1374         drop(fs::remove_dir_all(&overlay));
1375         t!(fs::create_dir_all(&overlay));
1376         builder.install(&src.join("README.md"), &overlay, 0o644);
1377         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1378         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1379         builder.create(&overlay.join("version"), &version);
1380
1381         // Generate the installer tarball
1382         let mut cmd = rust_installer(builder);
1383         cmd.arg("generate")
1384            .arg("--product-name=Rust")
1385            .arg("--rel-manifest-dir=rustlib")
1386            .arg("--success-message=miri-ready-to-serve.")
1387            .arg("--image-dir").arg(&image)
1388            .arg("--work-dir").arg(&tmpdir(builder))
1389            .arg("--output-dir").arg(&distdir(builder))
1390            .arg("--non-installed-overlay").arg(&overlay)
1391            .arg(format!("--package-name={}-{}", name, target))
1392            .arg("--legacy-manifest-dirs=rustlib,cargo")
1393            .arg("--component-name=miri-preview");
1394
1395         builder.run(&mut cmd);
1396         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1397     }
1398 }
1399
1400 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1401 pub struct Rustfmt {
1402     pub stage: u32,
1403     pub target: Interned<String>,
1404 }
1405
1406 impl Step for Rustfmt {
1407     type Output = Option<PathBuf>;
1408     const ONLY_HOSTS: bool = true;
1409
1410     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1411         run.path("rustfmt")
1412     }
1413
1414     fn make_run(run: RunConfig<'_>) {
1415         run.builder.ensure(Rustfmt {
1416             stage: run.builder.top_stage,
1417             target: run.target,
1418         });
1419     }
1420
1421     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1422         let stage = self.stage;
1423         let target = self.target;
1424
1425         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1426         let src = builder.src.join("src/tools/rustfmt");
1427         let release_num = builder.release_num("rustfmt");
1428         let name = pkgname(builder, "rustfmt");
1429         let version = builder.rustfmt_info.version(builder, &release_num);
1430
1431         let tmp = tmpdir(builder);
1432         let image = tmp.join("rustfmt-image");
1433         drop(fs::remove_dir_all(&image));
1434         builder.create_dir(&image);
1435
1436         // Prepare the image directory
1437         let rustfmt = builder.ensure(tool::Rustfmt {
1438             compiler: builder.compiler(stage, builder.config.build),
1439             target, extra_features: Vec::new()
1440         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1441         let cargofmt = builder.ensure(tool::Cargofmt {
1442             compiler: builder.compiler(stage, builder.config.build),
1443             target, extra_features: Vec::new()
1444         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1445
1446         builder.install(&rustfmt, &image.join("bin"), 0o755);
1447         builder.install(&cargofmt, &image.join("bin"), 0o755);
1448         let doc = image.join("share/doc/rustfmt");
1449         builder.install(&src.join("README.md"), &doc, 0o644);
1450         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1451         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1452
1453         // Prepare the overlay
1454         let overlay = tmp.join("rustfmt-overlay");
1455         drop(fs::remove_dir_all(&overlay));
1456         builder.create_dir(&overlay);
1457         builder.install(&src.join("README.md"), &overlay, 0o644);
1458         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1459         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1460         builder.create(&overlay.join("version"), &version);
1461
1462         // Generate the installer tarball
1463         let mut cmd = rust_installer(builder);
1464         cmd.arg("generate")
1465            .arg("--product-name=Rust")
1466            .arg("--rel-manifest-dir=rustlib")
1467            .arg("--success-message=rustfmt-ready-to-fmt.")
1468            .arg("--image-dir").arg(&image)
1469            .arg("--work-dir").arg(&tmpdir(builder))
1470            .arg("--output-dir").arg(&distdir(builder))
1471            .arg("--non-installed-overlay").arg(&overlay)
1472            .arg(format!("--package-name={}-{}", name, target))
1473            .arg("--legacy-manifest-dirs=rustlib,cargo")
1474            .arg("--component-name=rustfmt-preview");
1475
1476         builder.run(&mut cmd);
1477         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1478     }
1479 }
1480
1481 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1482 pub struct Extended {
1483     stage: u32,
1484     host: Interned<String>,
1485     target: Interned<String>,
1486 }
1487
1488 impl Step for Extended {
1489     type Output = ();
1490     const DEFAULT: bool = true;
1491     const ONLY_HOSTS: bool = true;
1492
1493     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1494         let builder = run.builder;
1495         run.path("extended").default_condition(builder.config.extended)
1496     }
1497
1498     fn make_run(run: RunConfig<'_>) {
1499         run.builder.ensure(Extended {
1500             stage: run.builder.top_stage,
1501             host: run.builder.config.build,
1502             target: run.target,
1503         });
1504     }
1505
1506     /// Creates a combined installer for the specified target in the provided stage.
1507     fn run(self, builder: &Builder<'_>) {
1508         let stage = self.stage;
1509         let target = self.target;
1510
1511         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1512
1513         let rustc_installer = builder.ensure(Rustc {
1514             compiler: builder.compiler(stage, target),
1515         });
1516         let cargo_installer = builder.ensure(Cargo { stage, target });
1517         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1518         let rls_installer = builder.ensure(Rls { stage, target });
1519         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1520         let clippy_installer = builder.ensure(Clippy { stage, target });
1521         let miri_installer = builder.ensure(Miri { stage, target });
1522         let lldb_installer = builder.ensure(Lldb { target });
1523         let mingw_installer = builder.ensure(Mingw { host: target });
1524         let analysis_installer = builder.ensure(Analysis {
1525             compiler: builder.compiler(stage, self.host),
1526             target
1527         });
1528
1529         let docs_installer = builder.ensure(Docs { stage, host: target, });
1530         let std_installer = builder.ensure(Std {
1531             compiler: builder.compiler(stage, self.host),
1532             target,
1533         });
1534
1535         let tmp = tmpdir(builder);
1536         let overlay = tmp.join("extended-overlay");
1537         let etc = builder.src.join("src/etc/installer");
1538         let work = tmp.join("work");
1539
1540         let _ = fs::remove_dir_all(&overlay);
1541         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1542         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1543         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1544         let version = builder.rust_version();
1545         builder.create(&overlay.join("version"), &version);
1546         if let Some(sha) = builder.rust_sha() {
1547             builder.create(&overlay.join("git-commit-hash"), &sha);
1548         }
1549         builder.install(&etc.join("README.md"), &overlay, 0o644);
1550
1551         // When rust-std package split from rustc, we needed to ensure that during
1552         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1553         // the std files during uninstall. To do this ensure that rustc comes
1554         // before rust-std in the list below.
1555         let mut tarballs = Vec::new();
1556         tarballs.push(rustc_installer);
1557         tarballs.push(cargo_installer);
1558         tarballs.extend(rls_installer.clone());
1559         tarballs.extend(clippy_installer.clone());
1560         tarballs.extend(miri_installer.clone());
1561         tarballs.extend(rustfmt_installer.clone());
1562         tarballs.extend(llvm_tools_installer);
1563         tarballs.extend(lldb_installer);
1564         tarballs.push(analysis_installer);
1565         tarballs.push(std_installer);
1566         if builder.config.docs {
1567             tarballs.push(docs_installer);
1568         }
1569         if target.contains("pc-windows-gnu") {
1570             tarballs.push(mingw_installer.unwrap());
1571         }
1572         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1573         for tarball in &tarballs[1..] {
1574             input_tarballs.push(",");
1575             input_tarballs.push(tarball);
1576         }
1577
1578         let mut cmd = rust_installer(builder);
1579         cmd.arg("combine")
1580             .arg("--product-name=Rust")
1581             .arg("--rel-manifest-dir=rustlib")
1582             .arg("--success-message=Rust-is-ready-to-roll.")
1583             .arg("--work-dir").arg(&work)
1584             .arg("--output-dir").arg(&distdir(builder))
1585             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1586             .arg("--legacy-manifest-dirs=rustlib,cargo")
1587             .arg("--input-tarballs").arg(input_tarballs)
1588             .arg("--non-installed-overlay").arg(&overlay);
1589         builder.run(&mut cmd);
1590
1591         let mut license = String::new();
1592         license += &builder.read(&builder.src.join("COPYRIGHT"));
1593         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1594         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1595         license.push_str("\n");
1596         license.push_str("\n");
1597
1598         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1599         let mut rtf = rtf.to_string();
1600         rtf.push_str("\n");
1601         for line in license.lines() {
1602             rtf.push_str(line);
1603             rtf.push_str("\\line ");
1604         }
1605         rtf.push_str("}");
1606
1607         fn filter(contents: &str, marker: &str) -> String {
1608             let start = format!("tool-{}-start", marker);
1609             let end = format!("tool-{}-end", marker);
1610             let mut lines = Vec::new();
1611             let mut omitted = false;
1612             for line in contents.lines() {
1613                 if line.contains(&start) {
1614                     omitted = true;
1615                 } else if line.contains(&end) {
1616                     omitted = false;
1617                 } else if !omitted {
1618                     lines.push(line);
1619                 }
1620             }
1621
1622             lines.join("\n")
1623         }
1624
1625         let xform = |p: &Path| {
1626             let mut contents = t!(fs::read_to_string(p));
1627             if rls_installer.is_none() {
1628                 contents = filter(&contents, "rls");
1629             }
1630             if clippy_installer.is_none() {
1631                 contents = filter(&contents, "clippy");
1632             }
1633             if miri_installer.is_none() {
1634                 contents = filter(&contents, "miri");
1635             }
1636             if rustfmt_installer.is_none() {
1637                 contents = filter(&contents, "rustfmt");
1638             }
1639             let ret = tmp.join(p.file_name().unwrap());
1640             t!(fs::write(&ret, &contents));
1641             ret
1642         };
1643
1644         if target.contains("apple-darwin") {
1645             let pkg = tmp.join("pkg");
1646             let _ = fs::remove_dir_all(&pkg);
1647
1648             let pkgbuild = |component: &str| {
1649                 let mut cmd = Command::new("pkgbuild");
1650                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1651                     .arg("--scripts").arg(pkg.join(component))
1652                     .arg("--nopayload")
1653                     .arg(pkg.join(component).with_extension("pkg"));
1654                 builder.run(&mut cmd);
1655             };
1656
1657             let prepare = |name: &str| {
1658                 builder.create_dir(&pkg.join(name));
1659                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1660                         &pkg.join(name));
1661                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1662                 pkgbuild(name);
1663             };
1664             prepare("rustc");
1665             prepare("cargo");
1666             prepare("rust-docs");
1667             prepare("rust-std");
1668             prepare("rust-analysis");
1669
1670             if rls_installer.is_some() {
1671                 prepare("rls");
1672             }
1673             if clippy_installer.is_some() {
1674                 prepare("clippy");
1675             }
1676             if miri_installer.is_some() {
1677                 prepare("miri");
1678             }
1679
1680             // create an 'uninstall' package
1681             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1682             pkgbuild("uninstall");
1683
1684             builder.create_dir(&pkg.join("res"));
1685             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1686             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1687             let mut cmd = Command::new("productbuild");
1688             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1689                 .arg("--resources").arg(pkg.join("res"))
1690                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1691                                                     pkgname(builder, "rust"),
1692                                                     target)))
1693                 .arg("--package-path").arg(&pkg);
1694             builder.run(&mut cmd);
1695         }
1696
1697         if target.contains("windows") {
1698             let exe = tmp.join("exe");
1699             let _ = fs::remove_dir_all(&exe);
1700
1701             let prepare = |name: &str| {
1702                 builder.create_dir(&exe.join(name));
1703                 let dir = if name == "rust-std" || name == "rust-analysis" {
1704                     format!("{}-{}", name, target)
1705                 } else if name == "rls" {
1706                     "rls-preview".to_string()
1707                 } else if name == "clippy" {
1708                     "clippy-preview".to_string()
1709                 } else if name == "miri" {
1710                     "miri-preview".to_string()
1711                 } else {
1712                     name.to_string()
1713                 };
1714                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1715                             .join(dir),
1716                         &exe.join(name));
1717                 builder.remove(&exe.join(name).join("manifest.in"));
1718             };
1719             prepare("rustc");
1720             prepare("cargo");
1721             prepare("rust-analysis");
1722             prepare("rust-docs");
1723             prepare("rust-std");
1724             if rls_installer.is_some() {
1725                 prepare("rls");
1726             }
1727             if clippy_installer.is_some() {
1728                 prepare("clippy");
1729             }
1730             if miri_installer.is_some() {
1731                 prepare("miri");
1732             }
1733             if target.contains("windows-gnu") {
1734                 prepare("rust-mingw");
1735             }
1736
1737             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1738             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1739             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1740             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1741             builder.create(&exe.join("LICENSE.txt"), &license);
1742
1743             // Generate exe installer
1744             let mut cmd = Command::new("iscc");
1745             cmd.arg("rust.iss")
1746                 .current_dir(&exe);
1747             if target.contains("windows-gnu") {
1748                 cmd.arg("/dMINGW");
1749             }
1750             add_env(builder, &mut cmd, target);
1751             builder.run(&mut cmd);
1752             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1753                     &distdir(builder),
1754                     0o755);
1755
1756             // Generate msi installer
1757             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1758             let heat = wix.join("bin/heat.exe");
1759             let candle = wix.join("bin/candle.exe");
1760             let light = wix.join("bin/light.exe");
1761
1762             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1763             builder.run(Command::new(&heat)
1764                             .current_dir(&exe)
1765                             .arg("dir")
1766                             .arg("rustc")
1767                             .args(&heat_flags)
1768                             .arg("-cg").arg("RustcGroup")
1769                             .arg("-dr").arg("Rustc")
1770                             .arg("-var").arg("var.RustcDir")
1771                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1772             builder.run(Command::new(&heat)
1773                             .current_dir(&exe)
1774                             .arg("dir")
1775                             .arg("rust-docs")
1776                             .args(&heat_flags)
1777                             .arg("-cg").arg("DocsGroup")
1778                             .arg("-dr").arg("Docs")
1779                             .arg("-var").arg("var.DocsDir")
1780                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1781                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1782             builder.run(Command::new(&heat)
1783                             .current_dir(&exe)
1784                             .arg("dir")
1785                             .arg("cargo")
1786                             .args(&heat_flags)
1787                             .arg("-cg").arg("CargoGroup")
1788                             .arg("-dr").arg("Cargo")
1789                             .arg("-var").arg("var.CargoDir")
1790                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1791                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1792             builder.run(Command::new(&heat)
1793                             .current_dir(&exe)
1794                             .arg("dir")
1795                             .arg("rust-std")
1796                             .args(&heat_flags)
1797                             .arg("-cg").arg("StdGroup")
1798                             .arg("-dr").arg("Std")
1799                             .arg("-var").arg("var.StdDir")
1800                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1801             if rls_installer.is_some() {
1802                 builder.run(Command::new(&heat)
1803                                 .current_dir(&exe)
1804                                 .arg("dir")
1805                                 .arg("rls")
1806                                 .args(&heat_flags)
1807                                 .arg("-cg").arg("RlsGroup")
1808                                 .arg("-dr").arg("Rls")
1809                                 .arg("-var").arg("var.RlsDir")
1810                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1811                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1812             }
1813             if clippy_installer.is_some() {
1814                 builder.run(Command::new(&heat)
1815                                 .current_dir(&exe)
1816                                 .arg("dir")
1817                                 .arg("clippy")
1818                                 .args(&heat_flags)
1819                                 .arg("-cg").arg("ClippyGroup")
1820                                 .arg("-dr").arg("Clippy")
1821                                 .arg("-var").arg("var.ClippyDir")
1822                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1823                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1824             }
1825             if miri_installer.is_some() {
1826                 builder.run(Command::new(&heat)
1827                                 .current_dir(&exe)
1828                                 .arg("dir")
1829                                 .arg("miri")
1830                                 .args(&heat_flags)
1831                                 .arg("-cg").arg("MiriGroup")
1832                                 .arg("-dr").arg("Miri")
1833                                 .arg("-var").arg("var.MiriDir")
1834                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1835                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1836             }
1837             builder.run(Command::new(&heat)
1838                             .current_dir(&exe)
1839                             .arg("dir")
1840                             .arg("rust-analysis")
1841                             .args(&heat_flags)
1842                             .arg("-cg").arg("AnalysisGroup")
1843                             .arg("-dr").arg("Analysis")
1844                             .arg("-var").arg("var.AnalysisDir")
1845                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1846                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1847             if target.contains("windows-gnu") {
1848                 builder.run(Command::new(&heat)
1849                                 .current_dir(&exe)
1850                                 .arg("dir")
1851                                 .arg("rust-mingw")
1852                                 .args(&heat_flags)
1853                                 .arg("-cg").arg("GccGroup")
1854                                 .arg("-dr").arg("Gcc")
1855                                 .arg("-var").arg("var.GccDir")
1856                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1857             }
1858
1859             let candle = |input: &Path| {
1860                 let output = exe.join(input.file_stem().unwrap())
1861                                 .with_extension("wixobj");
1862                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1863                 let mut cmd = Command::new(&candle);
1864                 cmd.current_dir(&exe)
1865                     .arg("-nologo")
1866                     .arg("-dRustcDir=rustc")
1867                     .arg("-dDocsDir=rust-docs")
1868                     .arg("-dCargoDir=cargo")
1869                     .arg("-dStdDir=rust-std")
1870                     .arg("-dAnalysisDir=rust-analysis")
1871                     .arg("-arch").arg(&arch)
1872                     .arg("-out").arg(&output)
1873                     .arg(&input);
1874                 add_env(builder, &mut cmd, target);
1875
1876                 if rls_installer.is_some() {
1877                     cmd.arg("-dRlsDir=rls");
1878                 }
1879                 if clippy_installer.is_some() {
1880                     cmd.arg("-dClippyDir=clippy");
1881                 }
1882                 if miri_installer.is_some() {
1883                     cmd.arg("-dMiriDir=miri");
1884                 }
1885                 if target.contains("windows-gnu") {
1886                     cmd.arg("-dGccDir=rust-mingw");
1887                 }
1888                 builder.run(&mut cmd);
1889             };
1890             candle(&xform(&etc.join("msi/rust.wxs")));
1891             candle(&etc.join("msi/ui.wxs"));
1892             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1893             candle("RustcGroup.wxs".as_ref());
1894             candle("DocsGroup.wxs".as_ref());
1895             candle("CargoGroup.wxs".as_ref());
1896             candle("StdGroup.wxs".as_ref());
1897             if rls_installer.is_some() {
1898                 candle("RlsGroup.wxs".as_ref());
1899             }
1900             if clippy_installer.is_some() {
1901                 candle("ClippyGroup.wxs".as_ref());
1902             }
1903             if miri_installer.is_some() {
1904                 candle("MiriGroup.wxs".as_ref());
1905             }
1906             candle("AnalysisGroup.wxs".as_ref());
1907
1908             if target.contains("windows-gnu") {
1909                 candle("GccGroup.wxs".as_ref());
1910             }
1911
1912             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1913             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1914             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1915
1916             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1917             let mut cmd = Command::new(&light);
1918             cmd.arg("-nologo")
1919                 .arg("-ext").arg("WixUIExtension")
1920                 .arg("-ext").arg("WixUtilExtension")
1921                 .arg("-out").arg(exe.join(&filename))
1922                 .arg("rust.wixobj")
1923                 .arg("ui.wixobj")
1924                 .arg("rustwelcomedlg.wixobj")
1925                 .arg("RustcGroup.wixobj")
1926                 .arg("DocsGroup.wixobj")
1927                 .arg("CargoGroup.wixobj")
1928                 .arg("StdGroup.wixobj")
1929                 .arg("AnalysisGroup.wixobj")
1930                 .current_dir(&exe);
1931
1932             if rls_installer.is_some() {
1933                 cmd.arg("RlsGroup.wixobj");
1934             }
1935             if clippy_installer.is_some() {
1936                 cmd.arg("ClippyGroup.wixobj");
1937             }
1938             if miri_installer.is_some() {
1939                 cmd.arg("MiriGroup.wixobj");
1940             }
1941
1942             if target.contains("windows-gnu") {
1943                 cmd.arg("GccGroup.wixobj");
1944             }
1945             // ICE57 wrongly complains about the shortcuts
1946             cmd.arg("-sice:ICE57");
1947
1948             builder.run(&mut cmd);
1949
1950             if !builder.config.dry_run {
1951                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1952             }
1953         }
1954     }
1955 }
1956
1957 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1958     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1959     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1960        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1961        .env("CFG_RELEASE", builder.rust_release())
1962        .env("CFG_VER_MAJOR", parts.next().unwrap())
1963        .env("CFG_VER_MINOR", parts.next().unwrap())
1964        .env("CFG_VER_PATCH", parts.next().unwrap())
1965        .env("CFG_VER_BUILD", "0") // just needed to build
1966        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1967        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1968        .env("CFG_BUILD", target)
1969        .env("CFG_CHANNEL", &builder.config.channel);
1970
1971     if target.contains("windows-gnu") {
1972        cmd.env("CFG_MINGW", "1")
1973           .env("CFG_ABI", "GNU");
1974     } else {
1975        cmd.env("CFG_MINGW", "0")
1976           .env("CFG_ABI", "MSVC");
1977     }
1978
1979     if target.contains("x86_64") {
1980        cmd.env("CFG_PLATFORM", "x64");
1981     } else {
1982        cmd.env("CFG_PLATFORM", "x86");
1983     }
1984 }
1985
1986 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1987 pub struct HashSign;
1988
1989 impl Step for HashSign {
1990     type Output = ();
1991     const ONLY_HOSTS: bool = true;
1992
1993     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1994         run.path("hash-and-sign")
1995     }
1996
1997     fn make_run(run: RunConfig<'_>) {
1998         run.builder.ensure(HashSign);
1999     }
2000
2001     fn run(self, builder: &Builder<'_>) {
2002         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
2003         if builder.config.dry_run {
2004             return;
2005         }
2006         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2007             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2008         });
2009         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2010             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2011         });
2012         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2013             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2014         });
2015         let pass = t!(fs::read_to_string(&file));
2016
2017         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2018
2019         cmd.arg(sign);
2020         cmd.arg(distdir(builder));
2021         cmd.arg(today.trim());
2022         cmd.arg(builder.rust_package_vers());
2023         cmd.arg(addr);
2024         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2025         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2026         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2027         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2028         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2029         cmd.arg(builder.llvm_tools_package_vers());
2030         cmd.arg(builder.lldb_package_vers());
2031
2032         builder.create_dir(&distdir(builder));
2033
2034         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2035         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2036         let status = t!(child.wait());
2037         assert!(status.success());
2038     }
2039 }
2040
2041 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2042 // LLVM tools are linked dynamically.
2043 // Note: This function does no yet support Windows but we also don't support
2044 //       linking LLVM tools dynamically on Windows yet.
2045 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2046                                 target: Interned<String>,
2047                                 sysroot: &Path) {
2048     let src_libdir = builder
2049         .llvm_out(target)
2050         .join("lib");
2051     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2052     t!(fs::create_dir_all(&dst_libdir));
2053
2054     if target.contains("apple-darwin") {
2055         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2056         if llvm_dylib_path.exists() {
2057             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2058         }
2059         return
2060     }
2061
2062     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2063     // Since tools link to the latter rather than the former, we have to
2064     // follow the symlink to find out what to distribute.
2065     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2066     if llvm_dylib_path.exists() {
2067         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2068             panic!("dist: Error calling canonicalize path `{}`: {}",
2069                    llvm_dylib_path.display(), e);
2070         });
2071
2072
2073         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2074     }
2075 }
2076
2077 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2078 pub struct LlvmTools {
2079     pub stage: u32,
2080     pub target: Interned<String>,
2081 }
2082
2083 impl Step for LlvmTools {
2084     type Output = Option<PathBuf>;
2085     const ONLY_HOSTS: bool = true;
2086
2087     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2088         run.path("llvm-tools")
2089     }
2090
2091     fn make_run(run: RunConfig<'_>) {
2092         run.builder.ensure(LlvmTools {
2093             stage: run.builder.top_stage,
2094             target: run.target,
2095         });
2096     }
2097
2098     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2099         let stage = self.stage;
2100         let target = self.target;
2101         assert!(builder.config.extended);
2102
2103         /* run only if llvm-config isn't used */
2104         if let Some(config) = builder.config.target_config.get(&target) {
2105             if let Some(ref _s) = config.llvm_config {
2106                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2107                     stage, target));
2108                 return None;
2109             }
2110         }
2111
2112         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2113         let src = builder.src.join("src/llvm-project/llvm");
2114         let name = pkgname(builder, "llvm-tools");
2115
2116         let tmp = tmpdir(builder);
2117         let image = tmp.join("llvm-tools-image");
2118         drop(fs::remove_dir_all(&image));
2119
2120         // Prepare the image directory
2121         let src_bindir = builder
2122             .llvm_out(target)
2123             .join("bin");
2124         let dst_bindir = image.join("lib/rustlib")
2125             .join(&*target)
2126             .join("bin");
2127         t!(fs::create_dir_all(&dst_bindir));
2128         for tool in LLVM_TOOLS {
2129             let exe = src_bindir.join(exe(tool, &target));
2130             builder.install(&exe, &dst_bindir, 0o755);
2131         }
2132
2133         // Prepare the overlay
2134         let overlay = tmp.join("llvm-tools-overlay");
2135         drop(fs::remove_dir_all(&overlay));
2136         builder.create_dir(&overlay);
2137         builder.install(&src.join("README.txt"), &overlay, 0o644);
2138         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2139         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2140
2141         // Generate the installer tarball
2142         let mut cmd = rust_installer(builder);
2143         cmd.arg("generate")
2144             .arg("--product-name=Rust")
2145             .arg("--rel-manifest-dir=rustlib")
2146             .arg("--success-message=llvm-tools-installed.")
2147             .arg("--image-dir").arg(&image)
2148             .arg("--work-dir").arg(&tmpdir(builder))
2149             .arg("--output-dir").arg(&distdir(builder))
2150             .arg("--non-installed-overlay").arg(&overlay)
2151             .arg(format!("--package-name={}-{}", name, target))
2152             .arg("--legacy-manifest-dirs=rustlib,cargo")
2153             .arg("--component-name=llvm-tools-preview");
2154
2155
2156         builder.run(&mut cmd);
2157         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2158     }
2159 }
2160
2161 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2162 pub struct Lldb {
2163     pub target: Interned<String>,
2164 }
2165
2166 impl Step for Lldb {
2167     type Output = Option<PathBuf>;
2168     const ONLY_HOSTS: bool = true;
2169     const DEFAULT: bool = true;
2170
2171     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2172         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2173     }
2174
2175     fn make_run(run: RunConfig<'_>) {
2176         run.builder.ensure(Lldb {
2177             target: run.target,
2178         });
2179     }
2180
2181     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2182         let target = self.target;
2183
2184         if builder.config.dry_run {
2185             return None;
2186         }
2187
2188         let bindir = builder
2189             .llvm_out(target)
2190             .join("bin");
2191         let lldb_exe = bindir.join(exe("lldb", &target));
2192         if !lldb_exe.exists() {
2193             return None;
2194         }
2195
2196         builder.info(&format!("Dist Lldb ({})", target));
2197         let src = builder.src.join("src/llvm-project/lldb");
2198         let name = pkgname(builder, "lldb");
2199
2200         let tmp = tmpdir(builder);
2201         let image = tmp.join("lldb-image");
2202         drop(fs::remove_dir_all(&image));
2203
2204         // Prepare the image directory
2205         let root = image.join("lib/rustlib").join(&*target);
2206         let dst = root.join("bin");
2207         t!(fs::create_dir_all(&dst));
2208         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2209             let exe = bindir.join(exe(program, &target));
2210             builder.install(&exe, &dst, 0o755);
2211         }
2212
2213         // The libraries.
2214         let libdir = builder.llvm_out(target).join("lib");
2215         let dst = root.join("lib");
2216         t!(fs::create_dir_all(&dst));
2217         for entry in t!(fs::read_dir(&libdir)) {
2218             let entry = entry.unwrap();
2219             if let Ok(name) = entry.file_name().into_string() {
2220                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2221                     if t!(entry.file_type()).is_symlink() {
2222                         builder.copy_to_folder(&entry.path(), &dst);
2223                     } else {
2224                        builder.install(&entry.path(), &dst, 0o755);
2225                     }
2226                 }
2227             }
2228         }
2229
2230         // The lldb scripts might be installed in lib/python$version
2231         // or in lib64/python$version.  If lib64 exists, use it;
2232         // otherwise lib.
2233         let libdir = builder.llvm_out(target).join("lib64");
2234         let (libdir, libdir_name) = if libdir.exists() {
2235             (libdir, "lib64")
2236         } else {
2237             (builder.llvm_out(target).join("lib"), "lib")
2238         };
2239         for entry in t!(fs::read_dir(&libdir)) {
2240             let entry = t!(entry);
2241             if let Ok(name) = entry.file_name().into_string() {
2242                 if name.starts_with("python") {
2243                     let dst = root.join(libdir_name)
2244                         .join(entry.file_name());
2245                     t!(fs::create_dir_all(&dst));
2246                     builder.cp_r(&entry.path(), &dst);
2247                     break;
2248                 }
2249             }
2250         }
2251
2252         // Prepare the overlay
2253         let overlay = tmp.join("lldb-overlay");
2254         drop(fs::remove_dir_all(&overlay));
2255         builder.create_dir(&overlay);
2256         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2257         builder.create(&overlay.join("version"), &builder.lldb_vers());
2258
2259         // Generate the installer tarball
2260         let mut cmd = rust_installer(builder);
2261         cmd.arg("generate")
2262             .arg("--product-name=Rust")
2263             .arg("--rel-manifest-dir=rustlib")
2264             .arg("--success-message=lldb-installed.")
2265             .arg("--image-dir").arg(&image)
2266             .arg("--work-dir").arg(&tmpdir(builder))
2267             .arg("--output-dir").arg(&distdir(builder))
2268             .arg("--non-installed-overlay").arg(&overlay)
2269             .arg(format!("--package-name={}-{}", name, target))
2270             .arg("--legacy-manifest-dirs=rustlib,cargo")
2271             .arg("--component-name=lldb-preview");
2272
2273
2274         builder.run(&mut cmd);
2275         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2276     }
2277 }