src/bootstrap/dist.rs

   1 //! Implementation of the various distribution aspects of the compiler.
   2 //!
   3 //! This module is responsible for creating tarballs of the standard library,
   4 //! compiler, and documentation. This ends up being what we distribute to
   5 //! everyone as well.
   6 //!
   7 //! No tarball is actually created literally in this file, but rather we shell
   8 //! out to `rust-installer` still. This may one day be replaced with bits and
   9 //! pieces of `rustup.rs`!
  10
  11 use std::env;
  12 use std::fs;
  13 use std::io::Write;
  14 use std::path::{PathBuf, Path};
  15 use std::process::{Command, Stdio};
  16
  17 use build_helper::output;
  18
  19 use crate::{Compiler, Mode, LLVM_TOOLS};
  20 use crate::channel;
  21 use crate::util::{libdir, is_dylib, exe};
  22 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
  23 use crate::compile;
  24 use crate::tool::{self, Tool};
  25 use crate::cache::{INTERNER, Interned};
  26 use time::{self, Timespec};
  27
  28 pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
  29     if component == "cargo" {
  30         format!("{}-{}", component, builder.cargo_package_vers())
  31     } else if component == "rls" {
  32         format!("{}-{}", component, builder.rls_package_vers())
  33     } else if component == "clippy" {
  34         format!("{}-{}", component, builder.clippy_package_vers())
  35     } else if component == "miri" {
  36         format!("{}-{}", component, builder.miri_package_vers())
  37     } else if component == "rustfmt" {
  38         format!("{}-{}", component, builder.rustfmt_package_vers())
  39     } else if component == "llvm-tools" {
  40         format!("{}-{}", component, builder.llvm_tools_package_vers())
  41     } else if component == "lldb" {
  42         format!("{}-{}", component, builder.lldb_package_vers())
  43     } else {
  44         assert!(component.starts_with("rust"));
  45         format!("{}-{}", component, builder.rust_package_vers())
  46     }
  47 }
  48
  49 fn distdir(builder: &Builder<'_>) -> PathBuf {
  50     builder.out.join("dist")
  51 }
  52
  53 pub fn tmpdir(builder: &Builder<'_>) -> PathBuf {
  54     builder.out.join("tmp/dist")
  55 }
  56
  57 fn rust_installer(builder: &Builder<'_>) -> Command {
  58     builder.tool_cmd(Tool::RustInstaller)
  59 }
  60
  61 fn missing_tool(tool_name: &str, skip: bool) {
  62     if skip {
  63         println!("Unable to build {}, skipping dist", tool_name)
  64     } else {
  65         panic!("Unable to build {}", tool_name)
  66     }
  67 }
  68
  69 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
  70 pub struct Docs {
  71     pub stage: u32,
  72     pub host: Interned<String>,
  73 }
  74
  75 impl Step for Docs {
  76     type Output = PathBuf;
  77     const DEFAULT: bool = true;
  78
  79     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
  80         run.path("src/doc")
  81     }
  82
  83     fn make_run(run: RunConfig<'_>) {
  84         run.builder.ensure(Docs {
  85             stage: run.builder.top_stage,
  86             host: run.target,
  87         });
  88     }
  89
  90     /// Builds the `rust-docs` installer component.
  91     fn run(self, builder: &Builder<'_>) -> PathBuf {
  92         let host = self.host;
  93
  94         let name = pkgname(builder, "rust-docs");
  95
  96         builder.info(&format!("Dist docs ({})", host));
  97         if !builder.config.docs {
  98             builder.info("\tskipping - docs disabled");
  99             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 100         }
 101
 102         builder.default_doc(None);
 103
 104         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 105         let _ = fs::remove_dir_all(&image);
 106
 107         let dst = image.join("share/doc/rust/html");
 108         t!(fs::create_dir_all(&dst));
 109         let src = builder.doc_out(host);
 110         builder.cp_r(&src, &dst);
 111
 112         let mut cmd = rust_installer(builder);
 113         cmd.arg("generate")
 114            .arg("--product-name=Rust-Documentation")
 115            .arg("--rel-manifest-dir=rustlib")
 116            .arg("--success-message=Rust-documentation-is-installed.")
 117            .arg("--image-dir").arg(&image)
 118            .arg("--work-dir").arg(&tmpdir(builder))
 119            .arg("--output-dir").arg(&distdir(builder))
 120            .arg(format!("--package-name={}-{}", name, host))
 121            .arg("--component-name=rust-docs")
 122            .arg("--legacy-manifest-dirs=rustlib,cargo")
 123            .arg("--bulk-dirs=share/doc/rust/html");
 124         builder.run(&mut cmd);
 125         builder.remove_dir(&image);
 126
 127         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 128     }
 129 }
 130
 131 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 132 pub struct RustcDocs {
 133     pub stage: u32,
 134     pub host: Interned<String>,
 135 }
 136
 137 impl Step for RustcDocs {
 138     type Output = PathBuf;
 139     const DEFAULT: bool = true;
 140
 141     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 142         run.path("src/librustc")
 143     }
 144
 145     fn make_run(run: RunConfig<'_>) {
 146         run.builder.ensure(RustcDocs {
 147             stage: run.builder.top_stage,
 148             host: run.target,
 149         });
 150     }
 151
 152     /// Builds the `rustc-docs` installer component.
 153     fn run(self, builder: &Builder<'_>) -> PathBuf {
 154         let host = self.host;
 155
 156         let name = pkgname(builder, "rustc-docs");
 157
 158         builder.info(&format!("Dist compiler docs ({})", host));
 159         if !builder.config.compiler_docs {
 160             builder.info("\tskipping - compiler docs disabled");
 161             return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 162         }
 163
 164         builder.default_doc(None);
 165
 166         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 167         let _ = fs::remove_dir_all(&image);
 168
 169         let dst = image.join("share/doc/rust/html");
 170         t!(fs::create_dir_all(&dst));
 171         let src = builder.compiler_doc_out(host);
 172         builder.cp_r(&src, &dst);
 173
 174         let mut cmd = rust_installer(builder);
 175         cmd.arg("generate")
 176            .arg("--product-name=Rustc-Documentation")
 177            .arg("--rel-manifest-dir=rustlib")
 178            .arg("--success-message=Rustc-documentation-is-installed.")
 179            .arg("--image-dir").arg(&image)
 180            .arg("--work-dir").arg(&tmpdir(builder))
 181            .arg("--output-dir").arg(&distdir(builder))
 182            .arg(format!("--package-name={}-{}", name, host))
 183            .arg("--component-name=rustc-docs")
 184            .arg("--legacy-manifest-dirs=rustlib,cargo")
 185            .arg("--bulk-dirs=share/doc/rust/html");
 186         builder.run(&mut cmd);
 187         builder.remove_dir(&image);
 188
 189         distdir(builder).join(format!("{}-{}.tar.gz", name, host))
 190     }
 191 }
 192
 193 fn find_files(files: &[&str], path: &[PathBuf]) -> Vec<PathBuf> {
 194     let mut found = Vec::with_capacity(files.len());
 195
 196     for file in files {
 197         let file_path =
 198             path.iter()
 199                 .map(|dir| dir.join(file))
 200                 .find(|p| p.exists());
 201
 202         if let Some(file_path) = file_path {
 203             found.push(file_path);
 204         } else {
 205             panic!("Could not find '{}' in {:?}", file, path);
 206         }
 207     }
 208
 209     found
 210 }
 211
 212 fn make_win_dist(
 213     rust_root: &Path, plat_root: &Path, target_triple: Interned<String>, builder: &Builder<'_>
 214 ) {
 215     //Ask gcc where it keeps its stuff
 216     let mut cmd = Command::new(builder.cc(target_triple));
 217     cmd.arg("-print-search-dirs");
 218     let gcc_out = output(&mut cmd);
 219
 220     let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect();
 221     let mut lib_path = Vec::new();
 222
 223     for line in gcc_out.lines() {
 224         let idx = line.find(':').unwrap();
 225         let key = &line[..idx];
 226         let trim_chars: &[_] = &[' ', '='];
 227         let value =
 228             line[(idx + 1)..]
 229                 .trim_start_matches(trim_chars)
 230                 .split(';')
 231                 .map(PathBuf::from);
 232
 233         if key == "programs" {
 234             bin_path.extend(value);
 235         } else if key == "libraries" {
 236             lib_path.extend(value);
 237         }
 238     }
 239
 240     let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
 241     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
 242     if target_triple.starts_with("i686-") {
 243         rustc_dlls.push("libgcc_s_dw2-1.dll");
 244     } else {
 245         rustc_dlls.push("libgcc_s_seh-1.dll");
 246     }
 247
 248     let target_libs = [ //MinGW libs
 249         "libgcc.a",
 250         "libgcc_eh.a",
 251         "libgcc_s.a",
 252         "libm.a",
 253         "libmingw32.a",
 254         "libmingwex.a",
 255         "libstdc++.a",
 256         "libiconv.a",
 257         "libmoldname.a",
 258         "libpthread.a",
 259         //Windows import libs
 260         "libadvapi32.a",
 261         "libbcrypt.a",
 262         "libcomctl32.a",
 263         "libcomdlg32.a",
 264         "libcredui.a",
 265         "libcrypt32.a",
 266         "libdbghelp.a",
 267         "libgdi32.a",
 268         "libimagehlp.a",
 269         "libiphlpapi.a",
 270         "libkernel32.a",
 271         "libmsimg32.a",
 272         "libmsvcrt.a",
 273         "libodbc32.a",
 274         "libole32.a",
 275         "liboleaut32.a",
 276         "libopengl32.a",
 277         "libpsapi.a",
 278         "librpcrt4.a",
 279         "libsecur32.a",
 280         "libsetupapi.a",
 281         "libshell32.a",
 282         "libsynchronization.a",
 283         "libuser32.a",
 284         "libuserenv.a",
 285         "libuuid.a",
 286         "libwinhttp.a",
 287         "libwinmm.a",
 288         "libwinspool.a",
 289         "libws2_32.a",
 290         "libwsock32.a",
 291     ];
 292
 293     //Find mingw artifacts we want to bundle
 294     let target_tools = find_files(&target_tools, &bin_path);
 295     let rustc_dlls = find_files(&rustc_dlls, &bin_path);
 296     let target_libs = find_files(&target_libs, &lib_path);
 297
 298     // Copy runtime dlls next to rustc.exe
 299     let dist_bin_dir = rust_root.join("bin/");
 300     fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed");
 301     for src in rustc_dlls {
 302         builder.copy_to_folder(&src, &dist_bin_dir);
 303     }
 304
 305     //Copy platform tools to platform-specific bin directory
 306     let target_bin_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("bin");
 307     fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed");
 308     for src in target_tools {
 309         builder.copy_to_folder(&src, &target_bin_dir);
 310     }
 311
 312     // Warn windows-gnu users that the bundled GCC cannot compile C files
 313     builder.create(
 314         &target_bin_dir.join("GCC-WARNING.txt"),
 315         "gcc.exe contained in this folder cannot be used for compiling C files - it is only\
 316          used as a linker. In order to be able to compile projects containing C code use\
 317          the GCC provided by MinGW or Cygwin."
 318     );
 319
 320     //Copy platform libs to platform-specific lib directory
 321     let target_lib_dir = plat_root.join("lib").join("rustlib").join(target_triple).join("lib");
 322     fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed");
 323     for src in target_libs {
 324         builder.copy_to_folder(&src, &target_lib_dir);
 325     }
 326 }
 327
 328 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 329 pub struct Mingw {
 330     pub host: Interned<String>,
 331 }
 332
 333 impl Step for Mingw {
 334     type Output = Option<PathBuf>;
 335     const DEFAULT: bool = true;
 336
 337     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 338         run.never()
 339     }
 340
 341     fn make_run(run: RunConfig<'_>) {
 342         run.builder.ensure(Mingw { host: run.target });
 343     }
 344
 345     /// Builds the `rust-mingw` installer component.
 346     ///
 347     /// This contains all the bits and pieces to run the MinGW Windows targets
 348     /// without any extra installed software (e.g., we bundle gcc, libraries, etc).
 349     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
 350         let host = self.host;
 351
 352         if !host.contains("pc-windows-gnu") {
 353             return None;
 354         }
 355
 356         builder.info(&format!("Dist mingw ({})", host));
 357         let name = pkgname(builder, "rust-mingw");
 358         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 359         let _ = fs::remove_dir_all(&image);
 360         t!(fs::create_dir_all(&image));
 361
 362         // The first argument is a "temporary directory" which is just
 363         // thrown away (this contains the runtime DLLs included in the rustc package
 364         // above) and the second argument is where to place all the MinGW components
 365         // (which is what we want).
 366         make_win_dist(&tmpdir(builder), &image, host, &builder);
 367
 368         let mut cmd = rust_installer(builder);
 369         cmd.arg("generate")
 370            .arg("--product-name=Rust-MinGW")
 371            .arg("--rel-manifest-dir=rustlib")
 372            .arg("--success-message=Rust-MinGW-is-installed.")
 373            .arg("--image-dir").arg(&image)
 374            .arg("--work-dir").arg(&tmpdir(builder))
 375            .arg("--output-dir").arg(&distdir(builder))
 376            .arg(format!("--package-name={}-{}", name, host))
 377            .arg("--component-name=rust-mingw")
 378            .arg("--legacy-manifest-dirs=rustlib,cargo");
 379         builder.run(&mut cmd);
 380         t!(fs::remove_dir_all(&image));
 381         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host)))
 382     }
 383 }
 384
 385 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 386 pub struct Rustc {
 387     pub compiler: Compiler,
 388 }
 389
 390 impl Step for Rustc {
 391     type Output = PathBuf;
 392     const DEFAULT: bool = true;
 393     const ONLY_HOSTS: bool = true;
 394
 395     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 396         run.path("src/librustc")
 397     }
 398
 399     fn make_run(run: RunConfig<'_>) {
 400         run.builder.ensure(Rustc {
 401             compiler: run.builder.compiler(run.builder.top_stage, run.target),
 402         });
 403     }
 404
 405     /// Creates the `rustc` installer component.
 406     fn run(self, builder: &Builder<'_>) -> PathBuf {
 407         let compiler = self.compiler;
 408         let host = self.compiler.host;
 409
 410         builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host));
 411         let name = pkgname(builder, "rustc");
 412         let image = tmpdir(builder).join(format!("{}-{}-image", name, host));
 413         let _ = fs::remove_dir_all(&image);
 414         let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host));
 415         let _ = fs::remove_dir_all(&overlay);
 416
 417         // Prepare the rustc "image", what will actually end up getting installed
 418         prepare_image(builder, compiler, &image);
 419
 420         // Prepare the overlay which is part of the tarball but won't actually be
 421         // installed
 422         let cp = |file: &str| {
 423             builder.install(&builder.src.join(file), &overlay, 0o644);
 424         };
 425         cp("COPYRIGHT");
 426         cp("LICENSE-APACHE");
 427         cp("LICENSE-MIT");
 428         cp("README.md");
 429         // tiny morsel of metadata is used by rust-packaging
 430         let version = builder.rust_version();
 431         builder.create(&overlay.join("version"), &version);
 432         if let Some(sha) = builder.rust_sha() {
 433             builder.create(&overlay.join("git-commit-hash"), &sha);
 434         }
 435
 436         // On MinGW we've got a few runtime DLL dependencies that we need to
 437         // include. The first argument to this script is where to put these DLLs
 438         // (the image we're creating), and the second argument is a junk directory
 439         // to ignore all other MinGW stuff the script creates.
 440         //
 441         // On 32-bit MinGW we're always including a DLL which needs some extra
 442         // licenses to distribute. On 64-bit MinGW we don't actually distribute
 443         // anything requiring us to distribute a license, but it's likely the
 444         // install will *also* include the rust-mingw package, which also needs
 445         // licenses, so to be safe we just include it here in all MinGW packages.
 446         if host.contains("pc-windows-gnu") {
 447             make_win_dist(&image, &tmpdir(builder), host, builder);
 448
 449             let dst = image.join("share/doc");
 450             t!(fs::create_dir_all(&dst));
 451             builder.cp_r(&builder.src.join("src/etc/third-party"), &dst);
 452         }
 453
 454         // Finally, wrap everything up in a nice tarball!
 455         let mut cmd = rust_installer(builder);
 456         cmd.arg("generate")
 457            .arg("--product-name=Rust")
 458            .arg("--rel-manifest-dir=rustlib")
 459            .arg("--success-message=Rust-is-ready-to-roll.")
 460            .arg("--image-dir").arg(&image)
 461            .arg("--work-dir").arg(&tmpdir(builder))
 462            .arg("--output-dir").arg(&distdir(builder))
 463            .arg("--non-installed-overlay").arg(&overlay)
 464            .arg(format!("--package-name={}-{}", name, host))
 465            .arg("--component-name=rustc")
 466            .arg("--legacy-manifest-dirs=rustlib,cargo");
 467         builder.run(&mut cmd);
 468         builder.remove_dir(&image);
 469         builder.remove_dir(&overlay);
 470
 471         return distdir(builder).join(format!("{}-{}.tar.gz", name, host));
 472
 473         fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) {
 474             let host = compiler.host;
 475             let src = builder.sysroot(compiler);
 476             let libdir = libdir(&host);
 477
 478             // Copy rustc/rustdoc binaries
 479             t!(fs::create_dir_all(image.join("bin")));
 480             builder.cp_r(&src.join("bin"), &image.join("bin"));
 481
 482             builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
 483
 484             // Copy runtime DLLs needed by the compiler
 485             if libdir != "bin" {
 486                 for entry in builder.read_dir(&src.join(libdir)) {
 487                     let name = entry.file_name();
 488                     if let Some(s) = name.to_str() {
 489                         if is_dylib(s) {
 490                             builder.install(&entry.path(), &image.join(libdir), 0o644);
 491                         }
 492                     }
 493                 }
 494             }
 495
 496             // Copy over the codegen backends
 497             let backends_src = builder.sysroot_codegen_backends(compiler);
 498             let backends_rel = backends_src.strip_prefix(&src).unwrap();
 499             let backends_dst = image.join(&backends_rel);
 500             t!(fs::create_dir_all(&backends_dst));
 501             builder.cp_r(&backends_src, &backends_dst);
 502
 503             // Copy libLLVM.so to the lib dir as well, if needed. While not
 504             // technically needed by rustc itself it's needed by lots of other
 505             // components like the llvm tools and LLD. LLD is included below and
 506             // tools/LLDB come later, so let's just throw it in the rustc
 507             // component for now.
 508             maybe_install_llvm_dylib(builder, host, image);
 509
 510             // Copy over lld if it's there
 511             if builder.config.lld_enabled {
 512                 let exe = exe("rust-lld", &compiler.host);
 513                 let src = builder.sysroot_libdir(compiler, host)
 514                     .parent()
 515                     .unwrap()
 516                     .join("bin")
 517                     .join(&exe);
 518                 // for the rationale about this rename check `compile::copy_lld_to_sysroot`
 519                 let dst = image.join("lib/rustlib")
 520                     .join(&*host)
 521                     .join("bin")
 522                     .join(&exe);
 523                 t!(fs::create_dir_all(&dst.parent().unwrap()));
 524                 builder.copy(&src, &dst);
 525             }
 526
 527             // Man pages
 528             t!(fs::create_dir_all(image.join("share/man/man1")));
 529             let man_src = builder.src.join("src/doc/man");
 530             let man_dst = image.join("share/man/man1");
 531
 532             // Reproducible builds: If SOURCE_DATE_EPOCH is set, use that as the time.
 533             let time = env::var("SOURCE_DATE_EPOCH")
 534                 .map(|timestamp| {
 535                     let epoch = timestamp.parse().map_err(|err| {
 536                         format!("could not parse SOURCE_DATE_EPOCH: {}", err)
 537                     }).unwrap();
 538
 539                     time::at(Timespec::new(epoch, 0))
 540                 })
 541                 .unwrap_or_else(|_| time::now());
 542
 543             let month_year = t!(time::strftime("%B %Y", &time));
 544             // don't use our `bootstrap::util::{copy, cp_r}`, because those try
 545             // to hardlink, and we don't want to edit the source templates
 546             for file_entry in builder.read_dir(&man_src) {
 547                 let page_src = file_entry.path();
 548                 let page_dst = man_dst.join(file_entry.file_name());
 549                 t!(fs::copy(&page_src, &page_dst));
 550                 // template in month/year and version number
 551                 builder.replace_in_file(&page_dst,
 552                                 &[("<INSERT DATE HERE>", &month_year),
 553                                   ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM)]);
 554             }
 555
 556             // Debugger scripts
 557             builder.ensure(DebuggerScripts {
 558                 sysroot: INTERNER.intern_path(image.to_owned()),
 559                 host,
 560             });
 561
 562             // Misc license info
 563             let cp = |file: &str| {
 564                 builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644);
 565             };
 566             cp("COPYRIGHT");
 567             cp("LICENSE-APACHE");
 568             cp("LICENSE-MIT");
 569             cp("README.md");
 570         }
 571     }
 572 }
 573
 574 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 575 pub struct DebuggerScripts {
 576     pub sysroot: Interned<PathBuf>,
 577     pub host: Interned<String>,
 578 }
 579
 580 impl Step for DebuggerScripts {
 581     type Output = ();
 582
 583     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 584         run.path("src/lldb_batchmode.py")
 585     }
 586
 587     fn make_run(run: RunConfig<'_>) {
 588         run.builder.ensure(DebuggerScripts {
 589             sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)),
 590             host: run.target,
 591         });
 592     }
 593
 594     /// Copies debugger scripts for `target` into the `sysroot` specified.
 595     fn run(self, builder: &Builder<'_>) {
 596         let host = self.host;
 597         let sysroot = self.sysroot;
 598         let dst = sysroot.join("lib/rustlib/etc");
 599         t!(fs::create_dir_all(&dst));
 600         let cp_debugger_script = |file: &str| {
 601             builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644);
 602         };
 603         if host.contains("windows-msvc") {
 604             // windbg debugger scripts
 605             builder.install(&builder.src.join("src/etc/rust-windbg.cmd"), &sysroot.join("bin"),
 606                 0o755);
 607
 608             cp_debugger_script("natvis/intrinsic.natvis");
 609             cp_debugger_script("natvis/liballoc.natvis");
 610             cp_debugger_script("natvis/libcore.natvis");
 611         } else {
 612             cp_debugger_script("debugger_pretty_printers_common.py");
 613
 614             // gdb debugger scripts
 615             builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"),
 616                     0o755);
 617             builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"),
 618                     0o755);
 619
 620             cp_debugger_script("gdb_load_rust_pretty_printers.py");
 621             cp_debugger_script("gdb_rust_pretty_printing.py");
 622
 623             // lldb debugger scripts
 624             builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"),
 625                     0o755);
 626
 627             cp_debugger_script("lldb_rust_formatters.py");
 628         }
 629     }
 630 }
 631
 632 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 633 pub struct Std {
 634     pub compiler: Compiler,
 635     pub target: Interned<String>,
 636 }
 637
 638 impl Step for Std {
 639     type Output = PathBuf;
 640     const DEFAULT: bool = true;
 641
 642     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 643         run.path("src/libstd")
 644     }
 645
 646     fn make_run(run: RunConfig<'_>) {
 647         run.builder.ensure(Std {
 648             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 649             target: run.target,
 650         });
 651     }
 652
 653     fn run(self, builder: &Builder<'_>) -> PathBuf {
 654         let compiler = self.compiler;
 655         let target = self.target;
 656
 657         let name = pkgname(builder, "rust-std");
 658         builder.info(&format!("Dist std stage{} ({} -> {})",
 659             compiler.stage, &compiler.host, target));
 660
 661         // The only true set of target libraries came from the build triple, so
 662         // let's reduce redundant work by only producing archives from that host.
 663         if compiler.host != builder.config.build {
 664             builder.info("\tskipping, not a build host");
 665             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 666         }
 667
 668         // We want to package up as many target libraries as possible
 669         // for the `rust-std` package, so if this is a host target we
 670         // depend on librustc and otherwise we just depend on libtest.
 671         if builder.hosts.iter().any(|t| t == target) {
 672             builder.ensure(compile::Rustc { compiler, target });
 673         } else {
 674             if builder.no_std(target) == Some(true) {
 675                 // the `test` doesn't compile for no-std targets
 676                 builder.ensure(compile::Std { compiler, target });
 677             } else {
 678                 builder.ensure(compile::Test { compiler, target });
 679             }
 680         }
 681
 682         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 683         let _ = fs::remove_dir_all(&image);
 684
 685         let dst = image.join("lib/rustlib").join(target);
 686         t!(fs::create_dir_all(&dst));
 687         let mut src = builder.sysroot_libdir(compiler, target).to_path_buf();
 688         src.pop(); // Remove the trailing /lib folder from the sysroot_libdir
 689         builder.cp_filtered(&src, &dst, &|path| {
 690             if let Some(name) = path.file_name().and_then(|s| s.to_str()) {
 691                 if name == builder.config.rust_codegen_backends_dir.as_str() {
 692                     return false
 693                 }
 694                 if name == "bin" {
 695                     return false
 696                 }
 697                 if name.contains("LLVM") {
 698                     return false
 699                 }
 700             }
 701             true
 702         });
 703
 704         let mut cmd = rust_installer(builder);
 705         cmd.arg("generate")
 706            .arg("--product-name=Rust")
 707            .arg("--rel-manifest-dir=rustlib")
 708            .arg("--success-message=std-is-standing-at-the-ready.")
 709            .arg("--image-dir").arg(&image)
 710            .arg("--work-dir").arg(&tmpdir(builder))
 711            .arg("--output-dir").arg(&distdir(builder))
 712            .arg(format!("--package-name={}-{}", name, target))
 713            .arg(format!("--component-name=rust-std-{}", target))
 714            .arg("--legacy-manifest-dirs=rustlib,cargo");
 715         builder.run(&mut cmd);
 716         builder.remove_dir(&image);
 717         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 718     }
 719 }
 720
 721 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 722 pub struct Analysis {
 723     pub compiler: Compiler,
 724     pub target: Interned<String>,
 725 }
 726
 727 impl Step for Analysis {
 728     type Output = PathBuf;
 729     const DEFAULT: bool = true;
 730
 731     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 732         let builder = run.builder;
 733         run.path("analysis").default_condition(builder.config.extended)
 734     }
 735
 736     fn make_run(run: RunConfig<'_>) {
 737         run.builder.ensure(Analysis {
 738             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
 739             target: run.target,
 740         });
 741     }
 742
 743     /// Creates a tarball of save-analysis metadata, if available.
 744     fn run(self, builder: &Builder<'_>) -> PathBuf {
 745         let compiler = self.compiler;
 746         let target = self.target;
 747         assert!(builder.config.extended);
 748         builder.info("Dist analysis");
 749         let name = pkgname(builder, "rust-analysis");
 750
 751         if &compiler.host != builder.config.build {
 752             builder.info("\tskipping, not a build host");
 753             return distdir(builder).join(format!("{}-{}.tar.gz", name, target));
 754         }
 755
 756         builder.ensure(Std { compiler, target });
 757
 758         // Package save-analysis from stage1 if not doing a full bootstrap, as the
 759         // stage2 artifacts is simply copied from stage1 in that case.
 760         let compiler = if builder.force_use_stage1(compiler, target) {
 761             builder.compiler(1, compiler.host)
 762         } else {
 763             compiler.clone()
 764         };
 765
 766         let image = tmpdir(builder).join(format!("{}-{}-image", name, target));
 767
 768         let src = builder.stage_out(compiler, Mode::Std)
 769             .join(target).join(builder.cargo_dir()).join("deps");
 770
 771         let image_src = src.join("save-analysis");
 772         let dst = image.join("lib/rustlib").join(target).join("analysis");
 773         t!(fs::create_dir_all(&dst));
 774         builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst));
 775         builder.cp_r(&image_src, &dst);
 776
 777         let mut cmd = rust_installer(builder);
 778         cmd.arg("generate")
 779            .arg("--product-name=Rust")
 780            .arg("--rel-manifest-dir=rustlib")
 781            .arg("--success-message=save-analysis-saved.")
 782            .arg("--image-dir").arg(&image)
 783            .arg("--work-dir").arg(&tmpdir(builder))
 784            .arg("--output-dir").arg(&distdir(builder))
 785            .arg(format!("--package-name={}-{}", name, target))
 786            .arg(format!("--component-name=rust-analysis-{}", target))
 787            .arg("--legacy-manifest-dirs=rustlib,cargo");
 788         builder.run(&mut cmd);
 789         builder.remove_dir(&image);
 790         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
 791     }
 792 }
 793
 794 fn copy_src_dirs(builder: &Builder<'_>, src_dirs: &[&str], exclude_dirs: &[&str], dst_dir: &Path) {
 795     fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool {
 796         let spath = match path.to_str() {
 797             Some(path) => path,
 798             None => return false,
 799         };
 800         if spath.ends_with("~") || spath.ends_with(".pyc") {
 801             return false
 802         }
 803
 804         const LLVM_PROJECTS: &[&str] = &[
 805             "llvm-project/clang", "llvm-project\\clang",
 806             "llvm-project/lld", "llvm-project\\lld",
 807             "llvm-project/lldb", "llvm-project\\lldb",
 808             "llvm-project/llvm", "llvm-project\\llvm",
 809         ];
 810         if spath.contains("llvm-project") && !spath.ends_with("llvm-project")
 811             && !LLVM_PROJECTS.iter().any(|path| spath.contains(path))
 812         {
 813             return false;
 814         }
 815
 816         const LLVM_TEST: &[&str] = &[
 817             "llvm-project/llvm/test", "llvm-project\\llvm\\test",
 818             "llvm-emscripten/test", "llvm-emscripten\\test",
 819         ];
 820         if LLVM_TEST.iter().any(|path| spath.contains(path)) &&
 821             (spath.ends_with(".ll") ||
 822              spath.ends_with(".td") ||
 823              spath.ends_with(".s")) {
 824             return false
 825         }
 826         if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
 827             return false
 828         }
 829
 830         let full_path = Path::new(dir).join(path);
 831         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
 832             return false;
 833         }
 834
 835         let excludes = [
 836             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
 837             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
 838             "=RELEASE-ID", "=meta-update", "=update", ".bzr", ".bzrignore",
 839             ".bzrtags", ".hg", ".hgignore", ".hgrags", "_darcs",
 840         ];
 841         !path.iter()
 842              .map(|s| s.to_str().unwrap())
 843              .any(|s| excludes.contains(&s))
 844     }
 845
 846     // Copy the directories using our filter
 847     for item in src_dirs {
 848         let dst = &dst_dir.join(item);
 849         t!(fs::create_dir_all(dst));
 850         builder.cp_filtered(
 851             &builder.src.join(item), dst, &|path| filter_fn(exclude_dirs, item, path));
 852     }
 853 }
 854
 855 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 856 pub struct Src;
 857
 858 impl Step for Src {
 859     /// The output path of the src installer tarball
 860     type Output = PathBuf;
 861     const DEFAULT: bool = true;
 862     const ONLY_HOSTS: bool = true;
 863
 864     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 865         run.path("src")
 866     }
 867
 868     fn make_run(run: RunConfig<'_>) {
 869         run.builder.ensure(Src);
 870     }
 871
 872     /// Creates the `rust-src` installer component
 873     fn run(self, builder: &Builder<'_>) -> PathBuf {
 874         builder.info("Dist src");
 875
 876         let name = pkgname(builder, "rust-src");
 877         let image = tmpdir(builder).join(format!("{}-image", name));
 878         let _ = fs::remove_dir_all(&image);
 879
 880         let dst = image.join("lib/rustlib/src");
 881         let dst_src = dst.join("rust");
 882         t!(fs::create_dir_all(&dst_src));
 883
 884         let src_files = [
 885             "Cargo.lock",
 886         ];
 887         // This is the reduced set of paths which will become the rust-src component
 888         // (essentially libstd and all of its path dependencies)
 889         let std_src_dirs = [
 890             "src/build_helper",
 891             "src/liballoc",
 892             "src/libcore",
 893             "src/libpanic_abort",
 894             "src/libpanic_unwind",
 895             "src/librustc_asan",
 896             "src/librustc_lsan",
 897             "src/librustc_msan",
 898             "src/librustc_tsan",
 899             "src/libstd",
 900             "src/libunwind",
 901             "src/libtest",
 902             "src/libterm",
 903             "src/libprofiler_builtins",
 904             "src/stdsimd",
 905             "src/libproc_macro",
 906             "src/tools/rustc-std-workspace-core",
 907         ];
 908
 909         copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src);
 910         for file in src_files.iter() {
 911             builder.copy(&builder.src.join(file), &dst_src.join(file));
 912         }
 913
 914         // Create source tarball in rust-installer format
 915         let mut cmd = rust_installer(builder);
 916         cmd.arg("generate")
 917            .arg("--product-name=Rust")
 918            .arg("--rel-manifest-dir=rustlib")
 919            .arg("--success-message=Awesome-Source.")
 920            .arg("--image-dir").arg(&image)
 921            .arg("--work-dir").arg(&tmpdir(builder))
 922            .arg("--output-dir").arg(&distdir(builder))
 923            .arg(format!("--package-name={}", name))
 924            .arg("--component-name=rust-src")
 925            .arg("--legacy-manifest-dirs=rustlib,cargo");
 926         builder.run(&mut cmd);
 927
 928         builder.remove_dir(&image);
 929         distdir(builder).join(&format!("{}.tar.gz", name))
 930     }
 931 }
 932
 933 const CARGO_VENDOR_VERSION: &str = "0.1.22";
 934
 935 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 936 pub struct PlainSourceTarball;
 937
 938 impl Step for PlainSourceTarball {
 939     /// Produces the location of the tarball generated
 940     type Output = PathBuf;
 941     const DEFAULT: bool = true;
 942     const ONLY_HOSTS: bool = true;
 943
 944     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 945         let builder = run.builder;
 946         run.path("src").default_condition(builder.config.rust_dist_src)
 947     }
 948
 949     fn make_run(run: RunConfig<'_>) {
 950         run.builder.ensure(PlainSourceTarball);
 951     }
 952
 953     /// Creates the plain source tarball
 954     fn run(self, builder: &Builder<'_>) -> PathBuf {
 955         builder.info("Create plain source tarball");
 956
 957         // Make sure that the root folder of tarball has the correct name
 958         let plain_name = format!("{}-src", pkgname(builder, "rustc"));
 959         let plain_dst_src = tmpdir(builder).join(&plain_name);
 960         let _ = fs::remove_dir_all(&plain_dst_src);
 961         t!(fs::create_dir_all(&plain_dst_src));
 962
 963         // This is the set of root paths which will become part of the source package
 964         let src_files = [
 965             "COPYRIGHT",
 966             "LICENSE-APACHE",
 967             "LICENSE-MIT",
 968             "CONTRIBUTING.md",
 969             "README.md",
 970             "RELEASES.md",
 971             "configure",
 972             "x.py",
 973             "config.toml.example",
 974             "Cargo.toml",
 975             "Cargo.lock",
 976         ];
 977         let src_dirs = [
 978             "src",
 979         ];
 980
 981         copy_src_dirs(builder, &src_dirs[..], &[], &plain_dst_src);
 982
 983         // Copy the files normally
 984         for item in &src_files {
 985             builder.copy(&builder.src.join(item), &plain_dst_src.join(item));
 986         }
 987
 988         // Create the version file
 989         builder.create(&plain_dst_src.join("version"), &builder.rust_version());
 990         if let Some(sha) = builder.rust_sha() {
 991             builder.create(&plain_dst_src.join("git-commit-hash"), &sha);
 992         }
 993
 994         // If we're building from git sources, we need to vendor a complete distribution.
 995         if builder.rust_info.is_git() {
 996             // Get cargo-vendor installed, if it isn't already.
 997             let mut has_cargo_vendor = false;
 998             let mut cmd = Command::new(&builder.initial_cargo);
 999             for line in output(cmd.arg("install").arg("--list")).lines() {
1000                 has_cargo_vendor |= line.starts_with("cargo-vendor ");
1001             }
1002             if !has_cargo_vendor {
1003                 let mut cmd = builder.cargo(
1004                     builder.compiler(0, builder.config.build),
1005                     Mode::ToolBootstrap,
1006                     builder.config.build,
1007                     "install"
1008                 );
1009                 cmd.arg("--force")
1010                    .arg("--debug")
1011                    .arg("--vers").arg(CARGO_VENDOR_VERSION)
1012                    .arg("cargo-vendor");
1013                 builder.run(&mut cmd);
1014             }
1015
1016             // Vendor all Cargo dependencies
1017             let mut cmd = Command::new(&builder.initial_cargo);
1018             cmd.arg("vendor")
1019                .current_dir(&plain_dst_src);
1020             builder.run(&mut cmd);
1021         }
1022
1023         // Create plain source tarball
1024         let plain_name = format!("rustc-{}-src", builder.rust_package_vers());
1025         let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name));
1026         tarball.set_extension(""); // strip .gz
1027         tarball.set_extension(""); // strip .tar
1028         if let Some(dir) = tarball.parent() {
1029             builder.create_dir(&dir);
1030         }
1031         builder.info("running installer");
1032         let mut cmd = rust_installer(builder);
1033         cmd.arg("tarball")
1034            .arg("--input").arg(&plain_name)
1035            .arg("--output").arg(&tarball)
1036            .arg("--work-dir=.")
1037            .current_dir(tmpdir(builder));
1038         builder.run(&mut cmd);
1039         distdir(builder).join(&format!("{}.tar.gz", plain_name))
1040     }
1041 }
1042
1043 // We have to run a few shell scripts, which choke quite a bit on both `\`
1044 // characters and on `C:\` paths, so normalize both of them away.
1045 pub fn sanitize_sh(path: &Path) -> String {
1046     let path = path.to_str().unwrap().replace("\\", "/");
1047     return change_drive(&path).unwrap_or(path);
1048
1049     fn change_drive(s: &str) -> Option<String> {
1050         let mut ch = s.chars();
1051         let drive = ch.next().unwrap_or('C');
1052         if ch.next() != Some(':') {
1053             return None
1054         }
1055         if ch.next() != Some('/') {
1056             return None
1057         }
1058         Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..]))
1059     }
1060 }
1061
1062 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1063 pub struct Cargo {
1064     pub stage: u32,
1065     pub target: Interned<String>,
1066 }
1067
1068 impl Step for Cargo {
1069     type Output = PathBuf;
1070     const ONLY_HOSTS: bool = true;
1071
1072     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1073         run.path("cargo")
1074     }
1075
1076     fn make_run(run: RunConfig<'_>) {
1077         run.builder.ensure(Cargo {
1078             stage: run.builder.top_stage,
1079             target: run.target,
1080         });
1081     }
1082
1083     fn run(self, builder: &Builder<'_>) -> PathBuf {
1084         let stage = self.stage;
1085         let target = self.target;
1086
1087         builder.info(&format!("Dist cargo stage{} ({})", stage, target));
1088         let src = builder.src.join("src/tools/cargo");
1089         let etc = src.join("src/etc");
1090         let release_num = builder.release_num("cargo");
1091         let name = pkgname(builder, "cargo");
1092         let version = builder.cargo_info.version(builder, &release_num);
1093
1094         let tmp = tmpdir(builder);
1095         let image = tmp.join("cargo-image");
1096         drop(fs::remove_dir_all(&image));
1097         builder.create_dir(&image);
1098
1099         // Prepare the image directory
1100         builder.create_dir(&image.join("share/zsh/site-functions"));
1101         builder.create_dir(&image.join("etc/bash_completion.d"));
1102         let cargo = builder.ensure(tool::Cargo {
1103             compiler: builder.compiler(stage, builder.config.build),
1104             target
1105         });
1106         builder.install(&cargo, &image.join("bin"), 0o755);
1107         for man in t!(etc.join("man").read_dir()) {
1108             let man = t!(man);
1109             builder.install(&man.path(), &image.join("share/man/man1"), 0o644);
1110         }
1111         builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644);
1112         builder.copy(&etc.join("cargo.bashcomp.sh"),
1113              &image.join("etc/bash_completion.d/cargo"));
1114         let doc = image.join("share/doc/cargo");
1115         builder.install(&src.join("README.md"), &doc, 0o644);
1116         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1117         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1118         builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644);
1119
1120         // Prepare the overlay
1121         let overlay = tmp.join("cargo-overlay");
1122         drop(fs::remove_dir_all(&overlay));
1123         builder.create_dir(&overlay);
1124         builder.install(&src.join("README.md"), &overlay, 0o644);
1125         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1126         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1127         builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644);
1128         builder.create(&overlay.join("version"), &version);
1129
1130         // Generate the installer tarball
1131         let mut cmd = rust_installer(builder);
1132         cmd.arg("generate")
1133            .arg("--product-name=Rust")
1134            .arg("--rel-manifest-dir=rustlib")
1135            .arg("--success-message=Rust-is-ready-to-roll.")
1136            .arg("--image-dir").arg(&image)
1137            .arg("--work-dir").arg(&tmpdir(builder))
1138            .arg("--output-dir").arg(&distdir(builder))
1139            .arg("--non-installed-overlay").arg(&overlay)
1140            .arg(format!("--package-name={}-{}", name, target))
1141            .arg("--component-name=cargo")
1142            .arg("--legacy-manifest-dirs=rustlib,cargo");
1143         builder.run(&mut cmd);
1144         distdir(builder).join(format!("{}-{}.tar.gz", name, target))
1145     }
1146 }
1147
1148 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1149 pub struct Rls {
1150     pub stage: u32,
1151     pub target: Interned<String>,
1152 }
1153
1154 impl Step for Rls {
1155     type Output = Option<PathBuf>;
1156     const ONLY_HOSTS: bool = true;
1157
1158     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1159         run.path("rls")
1160     }
1161
1162     fn make_run(run: RunConfig<'_>) {
1163         run.builder.ensure(Rls {
1164             stage: run.builder.top_stage,
1165             target: run.target,
1166         });
1167     }
1168
1169     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1170         let stage = self.stage;
1171         let target = self.target;
1172         assert!(builder.config.extended);
1173
1174         builder.info(&format!("Dist RLS stage{} ({})", stage, target));
1175         let src = builder.src.join("src/tools/rls");
1176         let release_num = builder.release_num("rls");
1177         let name = pkgname(builder, "rls");
1178         let version = builder.rls_info.version(builder, &release_num);
1179
1180         let tmp = tmpdir(builder);
1181         let image = tmp.join("rls-image");
1182         drop(fs::remove_dir_all(&image));
1183         t!(fs::create_dir_all(&image));
1184
1185         // Prepare the image directory
1186         // We expect RLS to build, because we've exited this step above if tool
1187         // state for RLS isn't testing.
1188         let rls = builder.ensure(tool::Rls {
1189             compiler: builder.compiler(stage, builder.config.build),
1190             target, extra_features: Vec::new()
1191         }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
1192
1193         builder.install(&rls, &image.join("bin"), 0o755);
1194         let doc = image.join("share/doc/rls");
1195         builder.install(&src.join("README.md"), &doc, 0o644);
1196         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1197         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1198
1199         // Prepare the overlay
1200         let overlay = tmp.join("rls-overlay");
1201         drop(fs::remove_dir_all(&overlay));
1202         t!(fs::create_dir_all(&overlay));
1203         builder.install(&src.join("README.md"), &overlay, 0o644);
1204         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1205         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1206         builder.create(&overlay.join("version"), &version);
1207
1208         // Generate the installer tarball
1209         let mut cmd = rust_installer(builder);
1210         cmd.arg("generate")
1211            .arg("--product-name=Rust")
1212            .arg("--rel-manifest-dir=rustlib")
1213            .arg("--success-message=RLS-ready-to-serve.")
1214            .arg("--image-dir").arg(&image)
1215            .arg("--work-dir").arg(&tmpdir(builder))
1216            .arg("--output-dir").arg(&distdir(builder))
1217            .arg("--non-installed-overlay").arg(&overlay)
1218            .arg(format!("--package-name={}-{}", name, target))
1219            .arg("--legacy-manifest-dirs=rustlib,cargo")
1220            .arg("--component-name=rls-preview");
1221
1222         builder.run(&mut cmd);
1223         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1224     }
1225 }
1226
1227 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1228 pub struct Clippy {
1229     pub stage: u32,
1230     pub target: Interned<String>,
1231 }
1232
1233 impl Step for Clippy {
1234     type Output = Option<PathBuf>;
1235     const ONLY_HOSTS: bool = true;
1236
1237     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1238         run.path("clippy")
1239     }
1240
1241     fn make_run(run: RunConfig<'_>) {
1242         run.builder.ensure(Clippy {
1243             stage: run.builder.top_stage,
1244             target: run.target,
1245         });
1246     }
1247
1248     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1249         let stage = self.stage;
1250         let target = self.target;
1251         assert!(builder.config.extended);
1252
1253         builder.info(&format!("Dist clippy stage{} ({})", stage, target));
1254         let src = builder.src.join("src/tools/clippy");
1255         let release_num = builder.release_num("clippy");
1256         let name = pkgname(builder, "clippy");
1257         let version = builder.clippy_info.version(builder, &release_num);
1258
1259         let tmp = tmpdir(builder);
1260         let image = tmp.join("clippy-image");
1261         drop(fs::remove_dir_all(&image));
1262         builder.create_dir(&image);
1263
1264         // Prepare the image directory
1265         // We expect clippy to build, because we've exited this step above if tool
1266         // state for clippy isn't testing.
1267         let clippy = builder.ensure(tool::Clippy {
1268             compiler: builder.compiler(stage, builder.config.build),
1269             target, extra_features: Vec::new()
1270         }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
1271         let cargoclippy = builder.ensure(tool::CargoClippy {
1272             compiler: builder.compiler(stage, builder.config.build),
1273             target, extra_features: Vec::new()
1274         }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
1275
1276         builder.install(&clippy, &image.join("bin"), 0o755);
1277         builder.install(&cargoclippy, &image.join("bin"), 0o755);
1278         let doc = image.join("share/doc/clippy");
1279         builder.install(&src.join("README.md"), &doc, 0o644);
1280         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1281         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1282
1283         // Prepare the overlay
1284         let overlay = tmp.join("clippy-overlay");
1285         drop(fs::remove_dir_all(&overlay));
1286         t!(fs::create_dir_all(&overlay));
1287         builder.install(&src.join("README.md"), &overlay, 0o644);
1288         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1289         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1290         builder.create(&overlay.join("version"), &version);
1291
1292         // Generate the installer tarball
1293         let mut cmd = rust_installer(builder);
1294         cmd.arg("generate")
1295            .arg("--product-name=Rust")
1296            .arg("--rel-manifest-dir=rustlib")
1297            .arg("--success-message=clippy-ready-to-serve.")
1298            .arg("--image-dir").arg(&image)
1299            .arg("--work-dir").arg(&tmpdir(builder))
1300            .arg("--output-dir").arg(&distdir(builder))
1301            .arg("--non-installed-overlay").arg(&overlay)
1302            .arg(format!("--package-name={}-{}", name, target))
1303            .arg("--legacy-manifest-dirs=rustlib,cargo")
1304            .arg("--component-name=clippy-preview");
1305
1306         builder.run(&mut cmd);
1307         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1308     }
1309 }
1310
1311 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1312 pub struct Miri {
1313     pub stage: u32,
1314     pub target: Interned<String>,
1315 }
1316
1317 impl Step for Miri {
1318     type Output = Option<PathBuf>;
1319     const ONLY_HOSTS: bool = true;
1320
1321     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1322         run.path("miri")
1323     }
1324
1325     fn make_run(run: RunConfig<'_>) {
1326         run.builder.ensure(Miri {
1327             stage: run.builder.top_stage,
1328             target: run.target,
1329         });
1330     }
1331
1332     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1333         let stage = self.stage;
1334         let target = self.target;
1335         assert!(builder.config.extended);
1336
1337         builder.info(&format!("Dist miri stage{} ({})", stage, target));
1338         let src = builder.src.join("src/tools/miri");
1339         let release_num = builder.release_num("miri");
1340         let name = pkgname(builder, "miri");
1341         let version = builder.miri_info.version(builder, &release_num);
1342
1343         let tmp = tmpdir(builder);
1344         let image = tmp.join("miri-image");
1345         drop(fs::remove_dir_all(&image));
1346         builder.create_dir(&image);
1347
1348         // Prepare the image directory
1349         // We expect miri to build, because we've exited this step above if tool
1350         // state for miri isn't testing.
1351         let miri = builder.ensure(tool::Miri {
1352             compiler: builder.compiler(stage, builder.config.build),
1353             target, extra_features: Vec::new()
1354         }).or_else(|| { missing_tool("miri", builder.build.config.missing_tools); None })?;
1355         let cargomiri = builder.ensure(tool::CargoMiri {
1356             compiler: builder.compiler(stage, builder.config.build),
1357             target, extra_features: Vec::new()
1358         }).or_else(|| { missing_tool("cargo miri", builder.build.config.missing_tools); None })?;
1359
1360         builder.install(&miri, &image.join("bin"), 0o755);
1361         builder.install(&cargomiri, &image.join("bin"), 0o755);
1362         let doc = image.join("share/doc/miri");
1363         builder.install(&src.join("README.md"), &doc, 0o644);
1364         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1365         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1366
1367         // Prepare the overlay
1368         let overlay = tmp.join("miri-overlay");
1369         drop(fs::remove_dir_all(&overlay));
1370         t!(fs::create_dir_all(&overlay));
1371         builder.install(&src.join("README.md"), &overlay, 0o644);
1372         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1373         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1374         builder.create(&overlay.join("version"), &version);
1375
1376         // Generate the installer tarball
1377         let mut cmd = rust_installer(builder);
1378         cmd.arg("generate")
1379            .arg("--product-name=Rust")
1380            .arg("--rel-manifest-dir=rustlib")
1381            .arg("--success-message=miri-ready-to-serve.")
1382            .arg("--image-dir").arg(&image)
1383            .arg("--work-dir").arg(&tmpdir(builder))
1384            .arg("--output-dir").arg(&distdir(builder))
1385            .arg("--non-installed-overlay").arg(&overlay)
1386            .arg(format!("--package-name={}-{}", name, target))
1387            .arg("--legacy-manifest-dirs=rustlib,cargo")
1388            .arg("--component-name=miri-preview");
1389
1390         builder.run(&mut cmd);
1391         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1392     }
1393 }
1394
1395 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1396 pub struct Rustfmt {
1397     pub stage: u32,
1398     pub target: Interned<String>,
1399 }
1400
1401 impl Step for Rustfmt {
1402     type Output = Option<PathBuf>;
1403     const ONLY_HOSTS: bool = true;
1404
1405     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1406         run.path("rustfmt")
1407     }
1408
1409     fn make_run(run: RunConfig<'_>) {
1410         run.builder.ensure(Rustfmt {
1411             stage: run.builder.top_stage,
1412             target: run.target,
1413         });
1414     }
1415
1416     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
1417         let stage = self.stage;
1418         let target = self.target;
1419
1420         builder.info(&format!("Dist Rustfmt stage{} ({})", stage, target));
1421         let src = builder.src.join("src/tools/rustfmt");
1422         let release_num = builder.release_num("rustfmt");
1423         let name = pkgname(builder, "rustfmt");
1424         let version = builder.rustfmt_info.version(builder, &release_num);
1425
1426         let tmp = tmpdir(builder);
1427         let image = tmp.join("rustfmt-image");
1428         drop(fs::remove_dir_all(&image));
1429         builder.create_dir(&image);
1430
1431         // Prepare the image directory
1432         let rustfmt = builder.ensure(tool::Rustfmt {
1433             compiler: builder.compiler(stage, builder.config.build),
1434             target, extra_features: Vec::new()
1435         }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
1436         let cargofmt = builder.ensure(tool::Cargofmt {
1437             compiler: builder.compiler(stage, builder.config.build),
1438             target, extra_features: Vec::new()
1439         }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
1440
1441         builder.install(&rustfmt, &image.join("bin"), 0o755);
1442         builder.install(&cargofmt, &image.join("bin"), 0o755);
1443         let doc = image.join("share/doc/rustfmt");
1444         builder.install(&src.join("README.md"), &doc, 0o644);
1445         builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
1446         builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
1447
1448         // Prepare the overlay
1449         let overlay = tmp.join("rustfmt-overlay");
1450         drop(fs::remove_dir_all(&overlay));
1451         builder.create_dir(&overlay);
1452         builder.install(&src.join("README.md"), &overlay, 0o644);
1453         builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644);
1454         builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
1455         builder.create(&overlay.join("version"), &version);
1456
1457         // Generate the installer tarball
1458         let mut cmd = rust_installer(builder);
1459         cmd.arg("generate")
1460            .arg("--product-name=Rust")
1461            .arg("--rel-manifest-dir=rustlib")
1462            .arg("--success-message=rustfmt-ready-to-fmt.")
1463            .arg("--image-dir").arg(&image)
1464            .arg("--work-dir").arg(&tmpdir(builder))
1465            .arg("--output-dir").arg(&distdir(builder))
1466            .arg("--non-installed-overlay").arg(&overlay)
1467            .arg(format!("--package-name={}-{}", name, target))
1468            .arg("--legacy-manifest-dirs=rustlib,cargo")
1469            .arg("--component-name=rustfmt-preview");
1470
1471         builder.run(&mut cmd);
1472         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
1473     }
1474 }
1475
1476 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1477 pub struct Extended {
1478     stage: u32,
1479     host: Interned<String>,
1480     target: Interned<String>,
1481 }
1482
1483 impl Step for Extended {
1484     type Output = ();
1485     const DEFAULT: bool = true;
1486     const ONLY_HOSTS: bool = true;
1487
1488     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1489         let builder = run.builder;
1490         run.path("extended").default_condition(builder.config.extended)
1491     }
1492
1493     fn make_run(run: RunConfig<'_>) {
1494         run.builder.ensure(Extended {
1495             stage: run.builder.top_stage,
1496             host: run.builder.config.build,
1497             target: run.target,
1498         });
1499     }
1500
1501     /// Creates a combined installer for the specified target in the provided stage.
1502     fn run(self, builder: &Builder<'_>) {
1503         let stage = self.stage;
1504         let target = self.target;
1505
1506         builder.info(&format!("Dist extended stage{} ({})", stage, target));
1507
1508         let rustc_installer = builder.ensure(Rustc {
1509             compiler: builder.compiler(stage, target),
1510         });
1511         let cargo_installer = builder.ensure(Cargo { stage, target });
1512         let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
1513         let rls_installer = builder.ensure(Rls { stage, target });
1514         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
1515         let clippy_installer = builder.ensure(Clippy { stage, target });
1516         let miri_installer = builder.ensure(Miri { stage, target });
1517         let lldb_installer = builder.ensure(Lldb { target });
1518         let mingw_installer = builder.ensure(Mingw { host: target });
1519         let analysis_installer = builder.ensure(Analysis {
1520             compiler: builder.compiler(stage, self.host),
1521             target
1522         });
1523
1524         let docs_installer = builder.ensure(Docs { stage, host: target, });
1525         let std_installer = builder.ensure(Std {
1526             compiler: builder.compiler(stage, self.host),
1527             target,
1528         });
1529
1530         let tmp = tmpdir(builder);
1531         let overlay = tmp.join("extended-overlay");
1532         let etc = builder.src.join("src/etc/installer");
1533         let work = tmp.join("work");
1534
1535         let _ = fs::remove_dir_all(&overlay);
1536         builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644);
1537         builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644);
1538         builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644);
1539         let version = builder.rust_version();
1540         builder.create(&overlay.join("version"), &version);
1541         if let Some(sha) = builder.rust_sha() {
1542             builder.create(&overlay.join("git-commit-hash"), &sha);
1543         }
1544         builder.install(&etc.join("README.md"), &overlay, 0o644);
1545
1546         // When rust-std package split from rustc, we needed to ensure that during
1547         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
1548         // the std files during uninstall. To do this ensure that rustc comes
1549         // before rust-std in the list below.
1550         let mut tarballs = Vec::new();
1551         tarballs.push(rustc_installer);
1552         tarballs.push(cargo_installer);
1553         tarballs.extend(rls_installer.clone());
1554         tarballs.extend(clippy_installer.clone());
1555         tarballs.extend(miri_installer.clone());
1556         tarballs.extend(rustfmt_installer.clone());
1557         tarballs.extend(llvm_tools_installer);
1558         tarballs.extend(lldb_installer);
1559         tarballs.push(analysis_installer);
1560         tarballs.push(std_installer);
1561         if builder.config.docs {
1562             tarballs.push(docs_installer);
1563         }
1564         if target.contains("pc-windows-gnu") {
1565             tarballs.push(mingw_installer.unwrap());
1566         }
1567         let mut input_tarballs = tarballs[0].as_os_str().to_owned();
1568         for tarball in &tarballs[1..] {
1569             input_tarballs.push(",");
1570             input_tarballs.push(tarball);
1571         }
1572
1573         let mut cmd = rust_installer(builder);
1574         cmd.arg("combine")
1575             .arg("--product-name=Rust")
1576             .arg("--rel-manifest-dir=rustlib")
1577             .arg("--success-message=Rust-is-ready-to-roll.")
1578             .arg("--work-dir").arg(&work)
1579             .arg("--output-dir").arg(&distdir(builder))
1580             .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target))
1581             .arg("--legacy-manifest-dirs=rustlib,cargo")
1582             .arg("--input-tarballs").arg(input_tarballs)
1583             .arg("--non-installed-overlay").arg(&overlay);
1584         builder.run(&mut cmd);
1585
1586         let mut license = String::new();
1587         license += &builder.read(&builder.src.join("COPYRIGHT"));
1588         license += &builder.read(&builder.src.join("LICENSE-APACHE"));
1589         license += &builder.read(&builder.src.join("LICENSE-MIT"));
1590         license.push_str("\n");
1591         license.push_str("\n");
1592
1593         let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18";
1594         let mut rtf = rtf.to_string();
1595         rtf.push_str("\n");
1596         for line in license.lines() {
1597             rtf.push_str(line);
1598             rtf.push_str("\\line ");
1599         }
1600         rtf.push_str("}");
1601
1602         fn filter(contents: &str, marker: &str) -> String {
1603             let start = format!("tool-{}-start", marker);
1604             let end = format!("tool-{}-end", marker);
1605             let mut lines = Vec::new();
1606             let mut omitted = false;
1607             for line in contents.lines() {
1608                 if line.contains(&start) {
1609                     omitted = true;
1610                 } else if line.contains(&end) {
1611                     omitted = false;
1612                 } else if !omitted {
1613                     lines.push(line);
1614                 }
1615             }
1616
1617             lines.join("\n")
1618         }
1619
1620         let xform = |p: &Path| {
1621             let mut contents = t!(fs::read_to_string(p));
1622             if rls_installer.is_none() {
1623                 contents = filter(&contents, "rls");
1624             }
1625             if clippy_installer.is_none() {
1626                 contents = filter(&contents, "clippy");
1627             }
1628             if miri_installer.is_none() {
1629                 contents = filter(&contents, "miri");
1630             }
1631             if rustfmt_installer.is_none() {
1632                 contents = filter(&contents, "rustfmt");
1633             }
1634             let ret = tmp.join(p.file_name().unwrap());
1635             t!(fs::write(&ret, &contents));
1636             ret
1637         };
1638
1639         if target.contains("apple-darwin") {
1640             let pkg = tmp.join("pkg");
1641             let _ = fs::remove_dir_all(&pkg);
1642
1643             let pkgbuild = |component: &str| {
1644                 let mut cmd = Command::new("pkgbuild");
1645                 cmd.arg("--identifier").arg(format!("org.rust-lang.{}", component))
1646                     .arg("--scripts").arg(pkg.join(component))
1647                     .arg("--nopayload")
1648                     .arg(pkg.join(component).with_extension("pkg"));
1649                 builder.run(&mut cmd);
1650             };
1651
1652             let prepare = |name: &str| {
1653                 builder.create_dir(&pkg.join(name));
1654                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target)),
1655                         &pkg.join(name));
1656                 builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
1657                 pkgbuild(name);
1658             };
1659             prepare("rustc");
1660             prepare("cargo");
1661             prepare("rust-docs");
1662             prepare("rust-std");
1663             prepare("rust-analysis");
1664
1665             if rls_installer.is_some() {
1666                 prepare("rls");
1667             }
1668             if clippy_installer.is_some() {
1669                 prepare("clippy");
1670             }
1671             if miri_installer.is_some() {
1672                 prepare("miri");
1673             }
1674
1675             // create an 'uninstall' package
1676             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
1677             pkgbuild("uninstall");
1678
1679             builder.create_dir(&pkg.join("res"));
1680             builder.create(&pkg.join("res/LICENSE.txt"), &license);
1681             builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
1682             let mut cmd = Command::new("productbuild");
1683             cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
1684                 .arg("--resources").arg(pkg.join("res"))
1685                 .arg(distdir(builder).join(format!("{}-{}.pkg",
1686                                                     pkgname(builder, "rust"),
1687                                                     target)))
1688                 .arg("--package-path").arg(&pkg);
1689             builder.run(&mut cmd);
1690         }
1691
1692         if target.contains("windows") {
1693             let exe = tmp.join("exe");
1694             let _ = fs::remove_dir_all(&exe);
1695
1696             let prepare = |name: &str| {
1697                 builder.create_dir(&exe.join(name));
1698                 let dir = if name == "rust-std" || name == "rust-analysis" {
1699                     format!("{}-{}", name, target)
1700                 } else if name == "rls" {
1701                     "rls-preview".to_string()
1702                 } else if name == "clippy" {
1703                     "clippy-preview".to_string()
1704                 } else if name == "miri" {
1705                     "miri-preview".to_string()
1706                 } else {
1707                     name.to_string()
1708                 };
1709                 builder.cp_r(&work.join(&format!("{}-{}", pkgname(builder, name), target))
1710                             .join(dir),
1711                         &exe.join(name));
1712                 builder.remove(&exe.join(name).join("manifest.in"));
1713             };
1714             prepare("rustc");
1715             prepare("cargo");
1716             prepare("rust-analysis");
1717             prepare("rust-docs");
1718             prepare("rust-std");
1719             if rls_installer.is_some() {
1720                 prepare("rls");
1721             }
1722             if clippy_installer.is_some() {
1723                 prepare("clippy");
1724             }
1725             if miri_installer.is_some() {
1726                 prepare("miri");
1727             }
1728             if target.contains("windows-gnu") {
1729                 prepare("rust-mingw");
1730             }
1731
1732             builder.install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
1733             builder.install(&etc.join("exe/modpath.iss"), &exe, 0o644);
1734             builder.install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
1735             builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
1736             builder.create(&exe.join("LICENSE.txt"), &license);
1737
1738             // Generate exe installer
1739             let mut cmd = Command::new("iscc");
1740             cmd.arg("rust.iss")
1741                 .current_dir(&exe);
1742             if target.contains("windows-gnu") {
1743                 cmd.arg("/dMINGW");
1744             }
1745             add_env(builder, &mut cmd, target);
1746             builder.run(&mut cmd);
1747             builder.install(&exe.join(format!("{}-{}.exe", pkgname(builder, "rust"), target)),
1748                     &distdir(builder),
1749                     0o755);
1750
1751             // Generate msi installer
1752             let wix = PathBuf::from(env::var_os("WIX").unwrap());
1753             let heat = wix.join("bin/heat.exe");
1754             let candle = wix.join("bin/candle.exe");
1755             let light = wix.join("bin/light.exe");
1756
1757             let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"];
1758             builder.run(Command::new(&heat)
1759                             .current_dir(&exe)
1760                             .arg("dir")
1761                             .arg("rustc")
1762                             .args(&heat_flags)
1763                             .arg("-cg").arg("RustcGroup")
1764                             .arg("-dr").arg("Rustc")
1765                             .arg("-var").arg("var.RustcDir")
1766                             .arg("-out").arg(exe.join("RustcGroup.wxs")));
1767             builder.run(Command::new(&heat)
1768                             .current_dir(&exe)
1769                             .arg("dir")
1770                             .arg("rust-docs")
1771                             .args(&heat_flags)
1772                             .arg("-cg").arg("DocsGroup")
1773                             .arg("-dr").arg("Docs")
1774                             .arg("-var").arg("var.DocsDir")
1775                             .arg("-out").arg(exe.join("DocsGroup.wxs"))
1776                             .arg("-t").arg(etc.join("msi/squash-components.xsl")));
1777             builder.run(Command::new(&heat)
1778                             .current_dir(&exe)
1779                             .arg("dir")
1780                             .arg("cargo")
1781                             .args(&heat_flags)
1782                             .arg("-cg").arg("CargoGroup")
1783                             .arg("-dr").arg("Cargo")
1784                             .arg("-var").arg("var.CargoDir")
1785                             .arg("-out").arg(exe.join("CargoGroup.wxs"))
1786                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1787             builder.run(Command::new(&heat)
1788                             .current_dir(&exe)
1789                             .arg("dir")
1790                             .arg("rust-std")
1791                             .args(&heat_flags)
1792                             .arg("-cg").arg("StdGroup")
1793                             .arg("-dr").arg("Std")
1794                             .arg("-var").arg("var.StdDir")
1795                             .arg("-out").arg(exe.join("StdGroup.wxs")));
1796             if rls_installer.is_some() {
1797                 builder.run(Command::new(&heat)
1798                                 .current_dir(&exe)
1799                                 .arg("dir")
1800                                 .arg("rls")
1801                                 .args(&heat_flags)
1802                                 .arg("-cg").arg("RlsGroup")
1803                                 .arg("-dr").arg("Rls")
1804                                 .arg("-var").arg("var.RlsDir")
1805                                 .arg("-out").arg(exe.join("RlsGroup.wxs"))
1806                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1807             }
1808             if clippy_installer.is_some() {
1809                 builder.run(Command::new(&heat)
1810                                 .current_dir(&exe)
1811                                 .arg("dir")
1812                                 .arg("clippy")
1813                                 .args(&heat_flags)
1814                                 .arg("-cg").arg("ClippyGroup")
1815                                 .arg("-dr").arg("Clippy")
1816                                 .arg("-var").arg("var.ClippyDir")
1817                                 .arg("-out").arg(exe.join("ClippyGroup.wxs"))
1818                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1819             }
1820             if miri_installer.is_some() {
1821                 builder.run(Command::new(&heat)
1822                                 .current_dir(&exe)
1823                                 .arg("dir")
1824                                 .arg("miri")
1825                                 .args(&heat_flags)
1826                                 .arg("-cg").arg("MiriGroup")
1827                                 .arg("-dr").arg("Miri")
1828                                 .arg("-var").arg("var.MiriDir")
1829                                 .arg("-out").arg(exe.join("MiriGroup.wxs"))
1830                                 .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1831             }
1832             builder.run(Command::new(&heat)
1833                             .current_dir(&exe)
1834                             .arg("dir")
1835                             .arg("rust-analysis")
1836                             .args(&heat_flags)
1837                             .arg("-cg").arg("AnalysisGroup")
1838                             .arg("-dr").arg("Analysis")
1839                             .arg("-var").arg("var.AnalysisDir")
1840                             .arg("-out").arg(exe.join("AnalysisGroup.wxs"))
1841                             .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
1842             if target.contains("windows-gnu") {
1843                 builder.run(Command::new(&heat)
1844                                 .current_dir(&exe)
1845                                 .arg("dir")
1846                                 .arg("rust-mingw")
1847                                 .args(&heat_flags)
1848                                 .arg("-cg").arg("GccGroup")
1849                                 .arg("-dr").arg("Gcc")
1850                                 .arg("-var").arg("var.GccDir")
1851                                 .arg("-out").arg(exe.join("GccGroup.wxs")));
1852             }
1853
1854             let candle = |input: &Path| {
1855                 let output = exe.join(input.file_stem().unwrap())
1856                                 .with_extension("wixobj");
1857                 let arch = if target.contains("x86_64") {"x64"} else {"x86"};
1858                 let mut cmd = Command::new(&candle);
1859                 cmd.current_dir(&exe)
1860                     .arg("-nologo")
1861                     .arg("-dRustcDir=rustc")
1862                     .arg("-dDocsDir=rust-docs")
1863                     .arg("-dCargoDir=cargo")
1864                     .arg("-dStdDir=rust-std")
1865                     .arg("-dAnalysisDir=rust-analysis")
1866                     .arg("-arch").arg(&arch)
1867                     .arg("-out").arg(&output)
1868                     .arg(&input);
1869                 add_env(builder, &mut cmd, target);
1870
1871                 if rls_installer.is_some() {
1872                     cmd.arg("-dRlsDir=rls");
1873                 }
1874                 if clippy_installer.is_some() {
1875                     cmd.arg("-dClippyDir=clippy");
1876                 }
1877                 if miri_installer.is_some() {
1878                     cmd.arg("-dMiriDir=miri");
1879                 }
1880                 if target.contains("windows-gnu") {
1881                     cmd.arg("-dGccDir=rust-mingw");
1882                 }
1883                 builder.run(&mut cmd);
1884             };
1885             candle(&xform(&etc.join("msi/rust.wxs")));
1886             candle(&etc.join("msi/ui.wxs"));
1887             candle(&etc.join("msi/rustwelcomedlg.wxs"));
1888             candle("RustcGroup.wxs".as_ref());
1889             candle("DocsGroup.wxs".as_ref());
1890             candle("CargoGroup.wxs".as_ref());
1891             candle("StdGroup.wxs".as_ref());
1892             if rls_installer.is_some() {
1893                 candle("RlsGroup.wxs".as_ref());
1894             }
1895             if clippy_installer.is_some() {
1896                 candle("ClippyGroup.wxs".as_ref());
1897             }
1898             if miri_installer.is_some() {
1899                 candle("MiriGroup.wxs".as_ref());
1900             }
1901             candle("AnalysisGroup.wxs".as_ref());
1902
1903             if target.contains("windows-gnu") {
1904                 candle("GccGroup.wxs".as_ref());
1905             }
1906
1907             builder.create(&exe.join("LICENSE.rtf"), &rtf);
1908             builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644);
1909             builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644);
1910
1911             let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target);
1912             let mut cmd = Command::new(&light);
1913             cmd.arg("-nologo")
1914                 .arg("-ext").arg("WixUIExtension")
1915                 .arg("-ext").arg("WixUtilExtension")
1916                 .arg("-out").arg(exe.join(&filename))
1917                 .arg("rust.wixobj")
1918                 .arg("ui.wixobj")
1919                 .arg("rustwelcomedlg.wixobj")
1920                 .arg("RustcGroup.wixobj")
1921                 .arg("DocsGroup.wixobj")
1922                 .arg("CargoGroup.wixobj")
1923                 .arg("StdGroup.wixobj")
1924                 .arg("AnalysisGroup.wixobj")
1925                 .current_dir(&exe);
1926
1927             if rls_installer.is_some() {
1928                 cmd.arg("RlsGroup.wixobj");
1929             }
1930             if clippy_installer.is_some() {
1931                 cmd.arg("ClippyGroup.wixobj");
1932             }
1933             if miri_installer.is_some() {
1934                 cmd.arg("MiriGroup.wixobj");
1935             }
1936
1937             if target.contains("windows-gnu") {
1938                 cmd.arg("GccGroup.wixobj");
1939             }
1940             // ICE57 wrongly complains about the shortcuts
1941             cmd.arg("-sice:ICE57");
1942
1943             builder.run(&mut cmd);
1944
1945             if !builder.config.dry_run {
1946                 t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename)));
1947             }
1948         }
1949     }
1950 }
1951
1952 fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: Interned<String>) {
1953     let mut parts = channel::CFG_RELEASE_NUM.split('.');
1954     cmd.env("CFG_RELEASE_INFO", builder.rust_version())
1955        .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM)
1956        .env("CFG_RELEASE", builder.rust_release())
1957        .env("CFG_VER_MAJOR", parts.next().unwrap())
1958        .env("CFG_VER_MINOR", parts.next().unwrap())
1959        .env("CFG_VER_PATCH", parts.next().unwrap())
1960        .env("CFG_VER_BUILD", "0") // just needed to build
1961        .env("CFG_PACKAGE_VERS", builder.rust_package_vers())
1962        .env("CFG_PACKAGE_NAME", pkgname(builder, "rust"))
1963        .env("CFG_BUILD", target)
1964        .env("CFG_CHANNEL", &builder.config.channel);
1965
1966     if target.contains("windows-gnu") {
1967        cmd.env("CFG_MINGW", "1")
1968           .env("CFG_ABI", "GNU");
1969     } else {
1970        cmd.env("CFG_MINGW", "0")
1971           .env("CFG_ABI", "MSVC");
1972     }
1973
1974     if target.contains("x86_64") {
1975        cmd.env("CFG_PLATFORM", "x64");
1976     } else {
1977        cmd.env("CFG_PLATFORM", "x86");
1978     }
1979 }
1980
1981 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
1982 pub struct HashSign;
1983
1984 impl Step for HashSign {
1985     type Output = ();
1986     const ONLY_HOSTS: bool = true;
1987
1988     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1989         run.path("hash-and-sign")
1990     }
1991
1992     fn make_run(run: RunConfig<'_>) {
1993         run.builder.ensure(HashSign);
1994     }
1995
1996     fn run(self, builder: &Builder<'_>) {
1997         let mut cmd = builder.tool_cmd(Tool::BuildManifest);
1998         if builder.config.dry_run {
1999             return;
2000         }
2001         let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
2002             panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
2003         });
2004         let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
2005             panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
2006         });
2007         let file = builder.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
2008             panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
2009         });
2010         let pass = t!(fs::read_to_string(&file));
2011
2012         let today = output(Command::new("date").arg("+%Y-%m-%d"));
2013
2014         cmd.arg(sign);
2015         cmd.arg(distdir(builder));
2016         cmd.arg(today.trim());
2017         cmd.arg(builder.rust_package_vers());
2018         cmd.arg(addr);
2019         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
2020         cmd.arg(builder.package_vers(&builder.release_num("rls")));
2021         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
2022         cmd.arg(builder.package_vers(&builder.release_num("miri")));
2023         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
2024         cmd.arg(builder.llvm_tools_package_vers());
2025         cmd.arg(builder.lldb_package_vers());
2026
2027         builder.create_dir(&distdir(builder));
2028
2029         let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
2030         t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
2031         let status = t!(child.wait());
2032         assert!(status.success());
2033     }
2034 }
2035
2036 // Maybe add libLLVM.so to the lib-dir. It will only have been built if
2037 // LLVM tools are linked dynamically.
2038 // Note: This function does no yet support Windows but we also don't support
2039 //       linking LLVM tools dynamically on Windows yet.
2040 pub fn maybe_install_llvm_dylib(builder: &Builder<'_>,
2041                                 target: Interned<String>,
2042                                 sysroot: &Path) {
2043     let src_libdir = builder
2044         .llvm_out(target)
2045         .join("lib");
2046     let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib");
2047     t!(fs::create_dir_all(&dst_libdir));
2048
2049     if target.contains("apple-darwin") {
2050         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
2051         if llvm_dylib_path.exists() {
2052             builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2053         }
2054         return
2055     }
2056
2057     // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
2058     // Since tools link to the latter rather than the former, we have to
2059     // follow the symlink to find out what to distribute.
2060     let llvm_dylib_path = src_libdir.join("libLLVM.so");
2061     if llvm_dylib_path.exists() {
2062         let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
2063             panic!("dist: Error calling canonicalize path `{}`: {}",
2064                    llvm_dylib_path.display(), e);
2065         });
2066
2067
2068         builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
2069     }
2070 }
2071
2072 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2073 pub struct LlvmTools {
2074     pub stage: u32,
2075     pub target: Interned<String>,
2076 }
2077
2078 impl Step for LlvmTools {
2079     type Output = Option<PathBuf>;
2080     const ONLY_HOSTS: bool = true;
2081
2082     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2083         run.path("llvm-tools")
2084     }
2085
2086     fn make_run(run: RunConfig<'_>) {
2087         run.builder.ensure(LlvmTools {
2088             stage: run.builder.top_stage,
2089             target: run.target,
2090         });
2091     }
2092
2093     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2094         let stage = self.stage;
2095         let target = self.target;
2096         assert!(builder.config.extended);
2097
2098         /* run only if llvm-config isn't used */
2099         if let Some(config) = builder.config.target_config.get(&target) {
2100             if let Some(ref _s) = config.llvm_config {
2101                 builder.info(&format!("Skipping LlvmTools stage{} ({}): external LLVM",
2102                     stage, target));
2103                 return None;
2104             }
2105         }
2106
2107         builder.info(&format!("Dist LlvmTools stage{} ({})", stage, target));
2108         let src = builder.src.join("src/llvm-project/llvm");
2109         let name = pkgname(builder, "llvm-tools");
2110
2111         let tmp = tmpdir(builder);
2112         let image = tmp.join("llvm-tools-image");
2113         drop(fs::remove_dir_all(&image));
2114
2115         // Prepare the image directory
2116         let src_bindir = builder
2117             .llvm_out(target)
2118             .join("bin");
2119         let dst_bindir = image.join("lib/rustlib")
2120             .join(&*target)
2121             .join("bin");
2122         t!(fs::create_dir_all(&dst_bindir));
2123         for tool in LLVM_TOOLS {
2124             let exe = src_bindir.join(exe(tool, &target));
2125             builder.install(&exe, &dst_bindir, 0o755);
2126         }
2127
2128         // Prepare the overlay
2129         let overlay = tmp.join("llvm-tools-overlay");
2130         drop(fs::remove_dir_all(&overlay));
2131         builder.create_dir(&overlay);
2132         builder.install(&src.join("README.txt"), &overlay, 0o644);
2133         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2134         builder.create(&overlay.join("version"), &builder.llvm_tools_vers());
2135
2136         // Generate the installer tarball
2137         let mut cmd = rust_installer(builder);
2138         cmd.arg("generate")
2139             .arg("--product-name=Rust")
2140             .arg("--rel-manifest-dir=rustlib")
2141             .arg("--success-message=llvm-tools-installed.")
2142             .arg("--image-dir").arg(&image)
2143             .arg("--work-dir").arg(&tmpdir(builder))
2144             .arg("--output-dir").arg(&distdir(builder))
2145             .arg("--non-installed-overlay").arg(&overlay)
2146             .arg(format!("--package-name={}-{}", name, target))
2147             .arg("--legacy-manifest-dirs=rustlib,cargo")
2148             .arg("--component-name=llvm-tools-preview");
2149
2150
2151         builder.run(&mut cmd);
2152         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2153     }
2154 }
2155
2156 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
2157 pub struct Lldb {
2158     pub target: Interned<String>,
2159 }
2160
2161 impl Step for Lldb {
2162     type Output = Option<PathBuf>;
2163     const ONLY_HOSTS: bool = true;
2164     const DEFAULT: bool = true;
2165
2166     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2167         run.path("src/llvm-project/lldb").path("src/tools/lldb")
2168     }
2169
2170     fn make_run(run: RunConfig<'_>) {
2171         run.builder.ensure(Lldb {
2172             target: run.target,
2173         });
2174     }
2175
2176     fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
2177         let target = self.target;
2178
2179         if builder.config.dry_run {
2180             return None;
2181         }
2182
2183         let bindir = builder
2184             .llvm_out(target)
2185             .join("bin");
2186         let lldb_exe = bindir.join(exe("lldb", &target));
2187         if !lldb_exe.exists() {
2188             return None;
2189         }
2190
2191         builder.info(&format!("Dist Lldb ({})", target));
2192         let src = builder.src.join("src/llvm-project/lldb");
2193         let name = pkgname(builder, "lldb");
2194
2195         let tmp = tmpdir(builder);
2196         let image = tmp.join("lldb-image");
2197         drop(fs::remove_dir_all(&image));
2198
2199         // Prepare the image directory
2200         let root = image.join("lib/rustlib").join(&*target);
2201         let dst = root.join("bin");
2202         t!(fs::create_dir_all(&dst));
2203         for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
2204             let exe = bindir.join(exe(program, &target));
2205             builder.install(&exe, &dst, 0o755);
2206         }
2207
2208         // The libraries.
2209         let libdir = builder.llvm_out(target).join("lib");
2210         let dst = root.join("lib");
2211         t!(fs::create_dir_all(&dst));
2212         for entry in t!(fs::read_dir(&libdir)) {
2213             let entry = entry.unwrap();
2214             if let Ok(name) = entry.file_name().into_string() {
2215                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
2216                     if t!(entry.file_type()).is_symlink() {
2217                         builder.copy_to_folder(&entry.path(), &dst);
2218                     } else {
2219                        builder.install(&entry.path(), &dst, 0o755);
2220                     }
2221                 }
2222             }
2223         }
2224
2225         // The lldb scripts might be installed in lib/python$version
2226         // or in lib64/python$version.  If lib64 exists, use it;
2227         // otherwise lib.
2228         let libdir = builder.llvm_out(target).join("lib64");
2229         let (libdir, libdir_name) = if libdir.exists() {
2230             (libdir, "lib64")
2231         } else {
2232             (builder.llvm_out(target).join("lib"), "lib")
2233         };
2234         for entry in t!(fs::read_dir(&libdir)) {
2235             let entry = t!(entry);
2236             if let Ok(name) = entry.file_name().into_string() {
2237                 if name.starts_with("python") {
2238                     let dst = root.join(libdir_name)
2239                         .join(entry.file_name());
2240                     t!(fs::create_dir_all(&dst));
2241                     builder.cp_r(&entry.path(), &dst);
2242                     break;
2243                 }
2244             }
2245         }
2246
2247         // Prepare the overlay
2248         let overlay = tmp.join("lldb-overlay");
2249         drop(fs::remove_dir_all(&overlay));
2250         builder.create_dir(&overlay);
2251         builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
2252         builder.create(&overlay.join("version"), &builder.lldb_vers());
2253
2254         // Generate the installer tarball
2255         let mut cmd = rust_installer(builder);
2256         cmd.arg("generate")
2257             .arg("--product-name=Rust")
2258             .arg("--rel-manifest-dir=rustlib")
2259             .arg("--success-message=lldb-installed.")
2260             .arg("--image-dir").arg(&image)
2261             .arg("--work-dir").arg(&tmpdir(builder))
2262             .arg("--output-dir").arg(&distdir(builder))
2263             .arg("--non-installed-overlay").arg(&overlay)
2264             .arg(format!("--package-name={}-{}", name, target))
2265             .arg("--legacy-manifest-dirs=rustlib,cargo")
2266             .arg("--component-name=lldb-preview");
2267
2268
2269         builder.run(&mut cmd);
2270         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
2271     }
2272 }