src/bootstrap/download.rs

   1 use std::{
   2     env,
   3     ffi::{OsStr, OsString},
   4     fs::{self, File},
   5     io::{self, BufRead, BufReader, ErrorKind},
   6     path::{Path, PathBuf},
   7     process::{Command, Stdio},
   8 };
   9
  10 use once_cell::sync::OnceCell;
  11 use xz2::bufread::XzDecoder;
  12
  13 use crate::{
  14     config::RustfmtMetadata,
  15     native::detect_llvm_sha,
  16     t,
  17     util::{check_run, exe, program_out_of_date, try_run},
  18     Config,
  19 };
  20
  21 static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell<bool> = OnceCell::new();
  22
  23 /// Generic helpers that are useful anywhere in bootstrap.
  24 impl Config {
  25     pub fn is_verbose(&self) -> bool {
  26         self.verbose > 0
  27     }
  28
  29     pub fn symlink_file<P: AsRef<Path>, Q: AsRef<Path>>(&self, src: P, link: Q) -> io::Result<()> {
  30         #[cfg(unix)]
  31         use std::os::unix::fs::symlink as symlink_file;
  32         #[cfg(windows)]
  33         use std::os::windows::fs::symlink_file;
  34         if !self.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) }
  35     }
  36
  37     pub(crate) fn create(&self, path: &Path, s: &str) {
  38         if self.dry_run() {
  39             return;
  40         }
  41         t!(fs::write(path, s));
  42     }
  43
  44     pub(crate) fn remove(&self, f: &Path) {
  45         if self.dry_run() {
  46             return;
  47         }
  48         fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f));
  49     }
  50
  51     /// Create a temporary directory in `out` and return its path.
  52     ///
  53     /// NOTE: this temporary directory is shared between all steps;
  54     /// if you need an empty directory, create a new subdirectory inside it.
  55     pub(crate) fn tempdir(&self) -> PathBuf {
  56         let tmp = self.out.join("tmp");
  57         t!(fs::create_dir_all(&tmp));
  58         tmp
  59     }
  60
  61     /// Runs a command, printing out nice contextual information if it fails.
  62     /// Exits if the command failed to execute at all, otherwise returns its
  63     /// `status.success()`.
  64     pub(crate) fn try_run(&self, cmd: &mut Command) -> bool {
  65         if self.dry_run() {
  66             return true;
  67         }
  68         self.verbose(&format!("running: {:?}", cmd));
  69         try_run(cmd, self.is_verbose())
  70     }
  71
  72     /// Runs a command, printing out nice contextual information if it fails.
  73     /// Returns false if do not execute at all, otherwise returns its
  74     /// `status.success()`.
  75     pub(crate) fn check_run(&self, cmd: &mut Command) -> bool {
  76         if self.dry_run() {
  77             return true;
  78         }
  79         self.verbose(&format!("running: {:?}", cmd));
  80         check_run(cmd, self.is_verbose())
  81     }
  82
  83     /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true
  84     /// on NixOS
  85     fn should_fix_bins_and_dylibs(&self) -> bool {
  86         let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| {
  87             match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
  88                 Err(_) => return false,
  89                 Ok(output) if !output.status.success() => return false,
  90                 Ok(output) => {
  91                     let mut os_name = output.stdout;
  92                     if os_name.last() == Some(&b'\n') {
  93                         os_name.pop();
  94                     }
  95                     if os_name != b"Linux" {
  96                         return false;
  97                     }
  98                 }
  99             }
 100
 101             // If the user has asked binaries to be patched for Nix, then
 102             // don't check for NixOS or `/lib`.
 103             // NOTE: this intentionally comes after the Linux check:
 104             // - patchelf only works with ELF files, so no need to run it on Mac or Windows
 105             // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
 106             if self.patch_binaries_for_nix {
 107                 return true;
 108             }
 109
 110             // Use `/etc/os-release` instead of `/etc/NIXOS`.
 111             // The latter one does not exist on NixOS when using tmpfs as root.
 112             let is_nixos = match File::open("/etc/os-release") {
 113                 Err(e) if e.kind() == ErrorKind::NotFound => false,
 114                 Err(e) => panic!("failed to access /etc/os-release: {}", e),
 115                 Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
 116                     let l = l.expect("reading /etc/os-release");
 117                     matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
 118                 }),
 119             };
 120             is_nixos && !Path::new("/lib").exists()
 121         });
 122         if val {
 123             println!("info: You seem to be using Nix.");
 124         }
 125         val
 126     }
 127
 128     /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
 129     /// or the RPATH section, to fix the dynamic library search path
 130     ///
 131     /// This is only required on NixOS and uses the PatchELF utility to
 132     /// change the interpreter/RPATH of ELF executables.
 133     ///
 134     /// Please see https://nixos.org/patchelf.html for more information
 135     fn fix_bin_or_dylib(&self, fname: &Path) {
 136         assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true));
 137         println!("attempting to patch {}", fname.display());
 138
 139         // Only build `.nix-deps` once.
 140         static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
 141         let mut nix_build_succeeded = true;
 142         let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
 143             // Run `nix-build` to "build" each dependency (which will likely reuse
 144             // the existing `/nix/store` copy, or at most download a pre-built copy).
 145             //
 146             // Importantly, we create a gc-root called `.nix-deps` in the `build/`
 147             // directory, but still reference the actual `/nix/store` path in the rpath
 148             // as it makes it significantly more robust against changes to the location of
 149             // the `.nix-deps` location.
 150             //
 151             // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 152             // zlib: Needed as a system dependency of `libLLVM-*.so`.
 153             // patchelf: Needed for patching ELF binaries (see doc comment above).
 154             let nix_deps_dir = self.out.join(".nix-deps");
 155             const NIX_EXPR: &str = "
 156             with (import <nixpkgs> {});
 157             symlinkJoin {
 158                 name = \"rust-stage0-dependencies\";
 159                 paths = [
 160                     zlib
 161                     patchelf
 162                     stdenv.cc.bintools
 163                 ];
 164             }
 165             ";
 166             nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
 167                 Path::new("-E"),
 168                 Path::new(NIX_EXPR),
 169                 Path::new("-o"),
 170                 &nix_deps_dir,
 171             ]));
 172             nix_deps_dir
 173         });
 174         if !nix_build_succeeded {
 175             return;
 176         }
 177
 178         let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
 179         let rpath_entries = {
 180             // ORIGIN is a relative default, all binary and dynamic libraries we ship
 181             // appear to have this (even when `../lib` is redundant).
 182             // NOTE: there are only two paths here, delimited by a `:`
 183             let mut entries = OsString::from("$ORIGIN/../lib:");
 184             entries.push(t!(fs::canonicalize(nix_deps_dir)));
 185             entries.push("/lib");
 186             entries
 187         };
 188         patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
 189         if !fname.extension().map_or(false, |ext| ext == "so") {
 190             // Finally, set the correct .interp for binaries
 191             let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
 192             // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
 193             let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
 194             patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
 195         }
 196
 197         self.try_run(patchelf.arg(fname));
 198     }
 199
 200     fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
 201         self.verbose(&format!("download {url}"));
 202         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
 203         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
 204         // While bootstrap itself only supports http and https downloads, downstream forks might
 205         // need to download components from other protocols. The match allows them adding more
 206         // protocols without worrying about merge conflicts if we change the HTTP implementation.
 207         match url.split_once("://").map(|(proto, _)| proto) {
 208             Some("http") | Some("https") => {
 209                 self.download_http_with_retries(&tempfile, url, help_on_error)
 210             }
 211             Some(other) => panic!("unsupported protocol {other} in {url}"),
 212             None => panic!("no protocol in {url}"),
 213         }
 214         t!(std::fs::rename(&tempfile, dest_path));
 215     }
 216
 217     fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
 218         println!("downloading {}", url);
 219         // Try curl. If that fails and we are on windows, fallback to PowerShell.
 220         let mut curl = Command::new("curl");
 221         curl.args(&[
 222             "-#",
 223             "-y",
 224             "30",
 225             "-Y",
 226             "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
 227             "--connect-timeout",
 228             "30", // timeout if cannot connect within 30 seconds
 229             "--retry",
 230             "3",
 231             "-Sf",
 232             "-o",
 233         ]);
 234         curl.arg(tempfile);
 235         curl.arg(url);
 236         if !self.check_run(&mut curl) {
 237             if self.build.contains("windows-msvc") {
 238                 println!("Fallback to PowerShell");
 239                 for _ in 0..3 {
 240                     if self.try_run(Command::new("PowerShell.exe").args(&[
 241                         "/nologo",
 242                         "-Command",
 243                         "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 244                         &format!(
 245                             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
 246                             url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
 247                         ),
 248                     ])) {
 249                         return;
 250                     }
 251                     println!("\nspurious failure, trying again");
 252                 }
 253             }
 254             if !help_on_error.is_empty() {
 255                 eprintln!("{}", help_on_error);
 256             }
 257             crate::detail_exit(1);
 258         }
 259     }
 260
 261     fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
 262         println!("extracting {} to {}", tarball.display(), dst.display());
 263         if !dst.exists() {
 264             t!(fs::create_dir_all(dst));
 265         }
 266
 267         // `tarball` ends with `.tar.xz`; strip that suffix
 268         // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
 269         let uncompressed_filename =
 270             Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
 271         let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
 272
 273         // decompress the file
 274         let data = t!(File::open(tarball));
 275         let decompressor = XzDecoder::new(BufReader::new(data));
 276
 277         let mut tar = tar::Archive::new(decompressor);
 278         for member in t!(tar.entries()) {
 279             let mut member = t!(member);
 280             let original_path = t!(member.path()).into_owned();
 281             // skip the top-level directory
 282             if original_path == directory_prefix {
 283                 continue;
 284             }
 285             let mut short_path = t!(original_path.strip_prefix(directory_prefix));
 286             if !short_path.starts_with(pattern) {
 287                 continue;
 288             }
 289             short_path = t!(short_path.strip_prefix(pattern));
 290             let dst_path = dst.join(short_path);
 291             self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
 292             if !t!(member.unpack_in(dst)) {
 293                 panic!("path traversal attack ??");
 294             }
 295             let src_path = dst.join(original_path);
 296             if src_path.is_dir() && dst_path.exists() {
 297                 continue;
 298             }
 299             t!(fs::rename(src_path, dst_path));
 300         }
 301         t!(fs::remove_dir_all(dst.join(directory_prefix)));
 302     }
 303
 304     /// Returns whether the SHA256 checksum of `path` matches `expected`.
 305     fn verify(&self, path: &Path, expected: &str) -> bool {
 306         use sha2::Digest;
 307
 308         self.verbose(&format!("verifying {}", path.display()));
 309         let mut hasher = sha2::Sha256::new();
 310         // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
 311         // Consider using streaming IO instead?
 312         let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) };
 313         hasher.update(&contents);
 314         let found = hex::encode(hasher.finalize().as_slice());
 315         let verified = found == expected;
 316         if !verified && !self.dry_run() {
 317             println!(
 318                 "invalid checksum: \n\
 319                 found:    {found}\n\
 320                 expected: {expected}",
 321             );
 322         }
 323         return verified;
 324     }
 325 }
 326
 327 enum DownloadSource {
 328     CI,
 329     Dist,
 330 }
 331
 332 /// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions.
 333 impl Config {
 334     pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
 335         let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
 336         let channel = format!("{version}-{date}");
 337
 338         let host = self.build;
 339         let bin_root = self.out.join(host.triple).join("rustfmt");
 340         let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host));
 341         let rustfmt_stamp = bin_root.join(".rustfmt-stamp");
 342
 343         let legacy_rustfmt = self.initial_rustc.with_file_name(exe("rustfmt", host));
 344         if !legacy_rustfmt.exists() {
 345             t!(self.symlink_file(&rustfmt_path, &legacy_rustfmt));
 346         }
 347
 348         if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) {
 349             return Some(rustfmt_path);
 350         }
 351
 352         self.download_component(
 353             DownloadSource::Dist,
 354             format!("rustfmt-{version}-{build}.tar.xz", build = host.triple),
 355             "rustfmt-preview",
 356             &date,
 357             "rustfmt",
 358         );
 359         self.download_component(
 360             DownloadSource::Dist,
 361             format!("rustc-{version}-{build}.tar.xz", build = host.triple),
 362             "rustc",
 363             &date,
 364             "rustfmt",
 365         );
 366
 367         if self.should_fix_bins_and_dylibs() {
 368             self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
 369             self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
 370         }
 371
 372         self.create(&rustfmt_stamp, &channel);
 373         Some(rustfmt_path)
 374     }
 375
 376     pub(crate) fn download_ci_rustc(&self, commit: &str) {
 377         self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
 378         let version = self.artifact_version_part(commit);
 379         let host = self.build.triple;
 380         let bin_root = self.out.join(host).join("ci-rustc");
 381         let rustc_stamp = bin_root.join(".rustc-stamp");
 382
 383         if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit)
 384         {
 385             if bin_root.exists() {
 386                 t!(fs::remove_dir_all(&bin_root));
 387             }
 388             let filename = format!("rust-std-{version}-{host}.tar.xz");
 389             let pattern = format!("rust-std-{host}");
 390             self.download_ci_component(filename, &pattern, commit);
 391             let filename = format!("rustc-{version}-{host}.tar.xz");
 392             self.download_ci_component(filename, "rustc", commit);
 393             // download-rustc doesn't need its own cargo, it can just use beta's.
 394             let filename = format!("rustc-dev-{version}-{host}.tar.xz");
 395             self.download_ci_component(filename, "rustc-dev", commit);
 396             let filename = format!("rust-src-{version}.tar.xz");
 397             self.download_ci_component(filename, "rust-src", commit);
 398
 399             if self.should_fix_bins_and_dylibs() {
 400                 self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
 401                 self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
 402                 self.fix_bin_or_dylib(
 403                     &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
 404                 );
 405                 let lib_dir = bin_root.join("lib");
 406                 for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
 407                     let lib = t!(lib);
 408                     if lib.path().extension() == Some(OsStr::new("so")) {
 409                         self.fix_bin_or_dylib(&lib.path());
 410                     }
 411                 }
 412             }
 413
 414             t!(fs::write(rustc_stamp, commit));
 415         }
 416     }
 417
 418     /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
 419     // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
 420     fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) {
 421         Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
 422     }
 423
 424     fn download_component(
 425         &self,
 426         mode: DownloadSource,
 427         filename: String,
 428         prefix: &str,
 429         key: &str,
 430         destination: &str,
 431     ) {
 432         let cache_dst = self.out.join("cache");
 433         let cache_dir = cache_dst.join(key);
 434         if !cache_dir.exists() {
 435             t!(fs::create_dir_all(&cache_dir));
 436         }
 437
 438         let bin_root = self.out.join(self.build.triple).join(destination);
 439         let tarball = cache_dir.join(&filename);
 440         let (base_url, url, should_verify) = match mode {
 441             DownloadSource::CI => (
 442                 self.stage0_metadata.config.artifacts_server.clone(),
 443                 format!("{key}/{filename}"),
 444                 false,
 445             ),
 446             DownloadSource::Dist => {
 447                 let dist_server = env::var("RUSTUP_DIST_SERVER")
 448                     .unwrap_or(self.stage0_metadata.config.dist_server.to_string());
 449                 // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
 450                 (dist_server, format!("dist/{key}/{filename}"), true)
 451             }
 452         };
 453
 454         // For the beta compiler, put special effort into ensuring the checksums are valid.
 455         // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
 456         // this on each and every nightly ...
 457         let checksum = if should_verify {
 458             let error = format!(
 459                 "src/stage0.json doesn't contain a checksum for {url}. \
 460                 Pre-built artifacts might not be available for this \
 461                 target at this time, see https://doc.rust-lang.org/nightly\
 462                 /rustc/platform-support.html for more information."
 463             );
 464             let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error);
 465             if tarball.exists() {
 466                 if self.verify(&tarball, sha256) {
 467                     self.unpack(&tarball, &bin_root, prefix);
 468                     return;
 469                 } else {
 470                     self.verbose(&format!(
 471                         "ignoring cached file {} due to failed verification",
 472                         tarball.display()
 473                     ));
 474                     self.remove(&tarball);
 475                 }
 476             }
 477             Some(sha256)
 478         } else if tarball.exists() {
 479             self.unpack(&tarball, &bin_root, prefix);
 480             return;
 481         } else {
 482             None
 483         };
 484
 485         self.download_file(&format!("{base_url}/{url}"), &tarball, "");
 486         if let Some(sha256) = checksum {
 487             if !self.verify(&tarball, sha256) {
 488                 panic!("failed to verify {}", tarball.display());
 489             }
 490         }
 491
 492         self.unpack(&tarball, &bin_root, prefix);
 493     }
 494
 495     pub(crate) fn maybe_download_ci_llvm(&self) {
 496         if !self.llvm_from_ci {
 497             return;
 498         }
 499         let llvm_root = self.ci_llvm_root();
 500         let llvm_stamp = llvm_root.join(".llvm-stamp");
 501         let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository());
 502         let key = format!("{}{}", llvm_sha, self.llvm_assertions);
 503         if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() {
 504             self.download_ci_llvm(&llvm_sha);
 505             if self.should_fix_bins_and_dylibs() {
 506                 for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
 507                     self.fix_bin_or_dylib(&t!(entry).path());
 508                 }
 509             }
 510
 511             // Update the timestamp of llvm-config to force rustc_llvm to be
 512             // rebuilt. This is a hacky workaround for a deficiency in Cargo where
 513             // the rerun-if-changed directive doesn't handle changes very well.
 514             // https://github.com/rust-lang/cargo/issues/10791
 515             // Cargo only compares the timestamp of the file relative to the last
 516             // time `rustc_llvm` build script ran. However, the timestamps of the
 517             // files in the tarball are in the past, so it doesn't trigger a
 518             // rebuild.
 519             let now = filetime::FileTime::from_system_time(std::time::SystemTime::now());
 520             let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build));
 521             t!(filetime::set_file_times(&llvm_config, now, now));
 522
 523             if self.should_fix_bins_and_dylibs() {
 524                 let llvm_lib = llvm_root.join("lib");
 525                 for entry in t!(fs::read_dir(&llvm_lib)) {
 526                     let lib = t!(entry).path();
 527                     if lib.extension().map_or(false, |ext| ext == "so") {
 528                         self.fix_bin_or_dylib(&lib);
 529                     }
 530                 }
 531             }
 532
 533             t!(fs::write(llvm_stamp, key));
 534         }
 535     }
 536
 537     fn download_ci_llvm(&self, llvm_sha: &str) {
 538         let llvm_assertions = self.llvm_assertions;
 539
 540         let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
 541         let cache_dst = self.out.join("cache");
 542         let rustc_cache = cache_dst.join(cache_prefix);
 543         if !rustc_cache.exists() {
 544             t!(fs::create_dir_all(&rustc_cache));
 545         }
 546         let base = if llvm_assertions {
 547             &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
 548         } else {
 549             &self.stage0_metadata.config.artifacts_server
 550         };
 551         let version = self.artifact_version_part(llvm_sha);
 552         let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple);
 553         let tarball = rustc_cache.join(&filename);
 554         if !tarball.exists() {
 555             let help_on_error = "error: failed to download llvm from ci
 556
 557     help: old builds get deleted after a certain time
 558     help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml:
 559
 560     [llvm]
 561     download-ci-llvm = false
 562     ";
 563             self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
 564         }
 565         let llvm_root = self.ci_llvm_root();
 566         self.unpack(&tarball, &llvm_root, "rust-dev");
 567     }
 568 }