src/bootstrap/download.rs

   1 use std::{
   2     env,
   3     ffi::{OsStr, OsString},
   4     fs::{self, File},
   5     io::{BufRead, BufReader, ErrorKind},
   6     path::{Path, PathBuf},
   7     process::{Command, Stdio},
   8 };
   9
  10 use once_cell::sync::OnceCell;
  11 use xz2::bufread::XzDecoder;
  12
  13 use crate::{
  14     config::RustfmtMetadata,
  15     native::detect_llvm_sha,
  16     t,
  17     util::{check_run, exe, program_out_of_date, try_run},
  18     Config,
  19 };
  20
  21 /// Generic helpers that are useful anywhere in bootstrap.
  22 impl Config {
  23     pub fn is_verbose(&self) -> bool {
  24         self.verbose > 0
  25     }
  26
  27     pub(crate) fn create(&self, path: &Path, s: &str) {
  28         if self.dry_run() {
  29             return;
  30         }
  31         t!(fs::write(path, s));
  32     }
  33
  34     pub(crate) fn remove(&self, f: &Path) {
  35         if self.dry_run() {
  36             return;
  37         }
  38         fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f));
  39     }
  40
  41     /// Create a temporary directory in `out` and return its path.
  42     ///
  43     /// NOTE: this temporary directory is shared between all steps;
  44     /// if you need an empty directory, create a new subdirectory inside it.
  45     pub(crate) fn tempdir(&self) -> PathBuf {
  46         let tmp = self.out.join("tmp");
  47         t!(fs::create_dir_all(&tmp));
  48         tmp
  49     }
  50
  51     /// Runs a command, printing out nice contextual information if it fails.
  52     /// Exits if the command failed to execute at all, otherwise returns its
  53     /// `status.success()`.
  54     pub(crate) fn try_run(&self, cmd: &mut Command) -> bool {
  55         if self.dry_run() {
  56             return true;
  57         }
  58         self.verbose(&format!("running: {:?}", cmd));
  59         try_run(cmd, self.is_verbose())
  60     }
  61
  62     /// Runs a command, printing out nice contextual information if it fails.
  63     /// Returns false if do not execute at all, otherwise returns its
  64     /// `status.success()`.
  65     pub(crate) fn check_run(&self, cmd: &mut Command) -> bool {
  66         if self.dry_run() {
  67             return true;
  68         }
  69         self.verbose(&format!("running: {:?}", cmd));
  70         check_run(cmd, self.is_verbose())
  71     }
  72
  73     /// Modifies the interpreter section of 'fname' to fix the dynamic linker,
  74     /// or the RPATH section, to fix the dynamic library search path
  75     ///
  76     /// This is only required on NixOS and uses the PatchELF utility to
  77     /// change the interpreter/RPATH of ELF executables.
  78     ///
  79     /// Please see https://nixos.org/patchelf.html for more information
  80     fn fix_bin_or_dylib(&self, fname: &Path) {
  81         // FIXME: cache NixOS detection?
  82         match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
  83             Err(_) => return,
  84             Ok(output) if !output.status.success() => return,
  85             Ok(output) => {
  86                 let mut s = output.stdout;
  87                 if s.last() == Some(&b'\n') {
  88                     s.pop();
  89                 }
  90                 if s != b"Linux" {
  91                     return;
  92                 }
  93             }
  94         }
  95
  96         // If the user has asked binaries to be patched for Nix, then
  97         // don't check for NixOS or `/lib`, just continue to the patching.
  98         // NOTE: this intentionally comes after the Linux check:
  99         // - patchelf only works with ELF files, so no need to run it on Mac or Windows
 100         // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
 101         if !self.patch_binaries_for_nix {
 102             // Use `/etc/os-release` instead of `/etc/NIXOS`.
 103             // The latter one does not exist on NixOS when using tmpfs as root.
 104             const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
 105             let os_release = match File::open("/etc/os-release") {
 106                 Err(e) if e.kind() == ErrorKind::NotFound => return,
 107                 Err(e) => panic!("failed to access /etc/os-release: {}", e),
 108                 Ok(f) => f,
 109             };
 110             if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
 111                 return;
 112             }
 113             if Path::new("/lib").exists() {
 114                 return;
 115             }
 116         }
 117
 118         // At this point we're pretty sure the user is running NixOS or using Nix
 119         println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
 120
 121         // Only build `.nix-deps` once.
 122         static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
 123         let mut nix_build_succeeded = true;
 124         let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
 125             // Run `nix-build` to "build" each dependency (which will likely reuse
 126             // the existing `/nix/store` copy, or at most download a pre-built copy).
 127             //
 128             // Importantly, we create a gc-root called `.nix-deps` in the `build/`
 129             // directory, but still reference the actual `/nix/store` path in the rpath
 130             // as it makes it significantly more robust against changes to the location of
 131             // the `.nix-deps` location.
 132             //
 133             // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 134             // zlib: Needed as a system dependency of `libLLVM-*.so`.
 135             // patchelf: Needed for patching ELF binaries (see doc comment above).
 136             let nix_deps_dir = self.out.join(".nix-deps");
 137             const NIX_EXPR: &str = "
 138             with (import <nixpkgs> {});
 139             symlinkJoin {
 140                 name = \"rust-stage0-dependencies\";
 141                 paths = [
 142                     zlib
 143                     patchelf
 144                     stdenv.cc.bintools
 145                 ];
 146             }
 147             ";
 148             nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
 149                 Path::new("-E"),
 150                 Path::new(NIX_EXPR),
 151                 Path::new("-o"),
 152                 &nix_deps_dir,
 153             ]));
 154             nix_deps_dir
 155         });
 156         if !nix_build_succeeded {
 157             return;
 158         }
 159
 160         let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
 161         let rpath_entries = {
 162             // ORIGIN is a relative default, all binary and dynamic libraries we ship
 163             // appear to have this (even when `../lib` is redundant).
 164             // NOTE: there are only two paths here, delimited by a `:`
 165             let mut entries = OsString::from("$ORIGIN/../lib:");
 166             entries.push(t!(fs::canonicalize(nix_deps_dir)));
 167             entries.push("/lib");
 168             entries
 169         };
 170         patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
 171         if !fname.extension().map_or(false, |ext| ext == "so") {
 172             // Finally, set the correct .interp for binaries
 173             let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
 174             // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
 175             let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
 176             patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
 177         }
 178
 179         self.try_run(patchelf.arg(fname));
 180     }
 181
 182     fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
 183         self.verbose(&format!("download {url}"));
 184         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
 185         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
 186         // While bootstrap itself only supports http and https downloads, downstream forks might
 187         // need to download components from other protocols. The match allows them adding more
 188         // protocols without worrying about merge conflicts if we change the HTTP implementation.
 189         match url.split_once("://").map(|(proto, _)| proto) {
 190             Some("http") | Some("https") => {
 191                 self.download_http_with_retries(&tempfile, url, help_on_error)
 192             }
 193             Some(other) => panic!("unsupported protocol {other} in {url}"),
 194             None => panic!("no protocol in {url}"),
 195         }
 196         t!(std::fs::rename(&tempfile, dest_path));
 197     }
 198
 199     fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
 200         println!("downloading {}", url);
 201         // Try curl. If that fails and we are on windows, fallback to PowerShell.
 202         let mut curl = Command::new("curl");
 203         curl.args(&[
 204             "-#",
 205             "-y",
 206             "30",
 207             "-Y",
 208             "10", // timeout if speed is < 10 bytes/sec for > 30 seconds
 209             "--connect-timeout",
 210             "30", // timeout if cannot connect within 30 seconds
 211             "--retry",
 212             "3",
 213             "-Sf",
 214             "-o",
 215         ]);
 216         curl.arg(tempfile);
 217         curl.arg(url);
 218         if !self.check_run(&mut curl) {
 219             if self.build.contains("windows-msvc") {
 220                 println!("Fallback to PowerShell");
 221                 for _ in 0..3 {
 222                     if self.try_run(Command::new("PowerShell.exe").args(&[
 223                         "/nologo",
 224                         "-Command",
 225                         "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 226                         &format!(
 227                             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
 228                             url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
 229                         ),
 230                     ])) {
 231                         return;
 232                     }
 233                     println!("\nspurious failure, trying again");
 234                 }
 235             }
 236             if !help_on_error.is_empty() {
 237                 eprintln!("{}", help_on_error);
 238             }
 239             crate::detail_exit(1);
 240         }
 241     }
 242
 243     fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
 244         println!("extracting {} to {}", tarball.display(), dst.display());
 245         if !dst.exists() {
 246             t!(fs::create_dir_all(dst));
 247         }
 248
 249         // `tarball` ends with `.tar.xz`; strip that suffix
 250         // example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
 251         let uncompressed_filename =
 252             Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
 253         let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
 254
 255         // decompress the file
 256         let data = t!(File::open(tarball));
 257         let decompressor = XzDecoder::new(BufReader::new(data));
 258
 259         let mut tar = tar::Archive::new(decompressor);
 260         for member in t!(tar.entries()) {
 261             let mut member = t!(member);
 262             let original_path = t!(member.path()).into_owned();
 263             // skip the top-level directory
 264             if original_path == directory_prefix {
 265                 continue;
 266             }
 267             let mut short_path = t!(original_path.strip_prefix(directory_prefix));
 268             if !short_path.starts_with(pattern) {
 269                 continue;
 270             }
 271             short_path = t!(short_path.strip_prefix(pattern));
 272             let dst_path = dst.join(short_path);
 273             self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
 274             if !t!(member.unpack_in(dst)) {
 275                 panic!("path traversal attack ??");
 276             }
 277             let src_path = dst.join(original_path);
 278             if src_path.is_dir() && dst_path.exists() {
 279                 continue;
 280             }
 281             t!(fs::rename(src_path, dst_path));
 282         }
 283         t!(fs::remove_dir_all(dst.join(directory_prefix)));
 284     }
 285
 286     /// Returns whether the SHA256 checksum of `path` matches `expected`.
 287     fn verify(&self, path: &Path, expected: &str) -> bool {
 288         use sha2::Digest;
 289
 290         self.verbose(&format!("verifying {}", path.display()));
 291         let mut hasher = sha2::Sha256::new();
 292         // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
 293         // Consider using streaming IO instead?
 294         let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) };
 295         hasher.update(&contents);
 296         let found = hex::encode(hasher.finalize().as_slice());
 297         let verified = found == expected;
 298         if !verified && !self.dry_run() {
 299             println!(
 300                 "invalid checksum: \n\
 301                 found:    {found}\n\
 302                 expected: {expected}",
 303             );
 304         }
 305         return verified;
 306     }
 307 }
 308
 309 enum DownloadSource {
 310     CI,
 311     Dist,
 312 }
 313
 314 /// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions.
 315 impl Config {
 316     pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> {
 317         let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?;
 318         let channel = format!("{version}-{date}");
 319
 320         let host = self.build;
 321         let bin_root = self.out.join(host.triple).join("rustfmt");
 322         let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host));
 323         let rustfmt_stamp = bin_root.join(".rustfmt-stamp");
 324         if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) {
 325             return Some(rustfmt_path);
 326         }
 327
 328         self.download_component(
 329             DownloadSource::Dist,
 330             format!("rustfmt-{version}-{build}.tar.xz", build = host.triple),
 331             "rustfmt-preview",
 332             &date,
 333             "rustfmt",
 334         );
 335         self.download_component(
 336             DownloadSource::Dist,
 337             format!("rustc-{version}-{build}.tar.xz", build = host.triple),
 338             "rustc",
 339             &date,
 340             "rustfmt",
 341         );
 342
 343         self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
 344         self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
 345
 346         self.create(&rustfmt_stamp, &channel);
 347         Some(rustfmt_path)
 348     }
 349
 350     pub(crate) fn download_ci_rustc(&self, commit: &str) {
 351         self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
 352         let version = self.artifact_version_part(commit);
 353         let host = self.build.triple;
 354         let bin_root = self.out.join(host).join("ci-rustc");
 355         let rustc_stamp = bin_root.join(".rustc-stamp");
 356
 357         if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit)
 358         {
 359             if bin_root.exists() {
 360                 t!(fs::remove_dir_all(&bin_root));
 361             }
 362             let filename = format!("rust-std-{version}-{host}.tar.xz");
 363             let pattern = format!("rust-std-{host}");
 364             self.download_ci_component(filename, &pattern, commit);
 365             let filename = format!("rustc-{version}-{host}.tar.xz");
 366             self.download_ci_component(filename, "rustc", commit);
 367             // download-rustc doesn't need its own cargo, it can just use beta's.
 368             let filename = format!("rustc-dev-{version}-{host}.tar.xz");
 369             self.download_ci_component(filename, "rustc-dev", commit);
 370             let filename = format!("rust-src-{version}.tar.xz");
 371             self.download_ci_component(filename, "rust-src", commit);
 372
 373             self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
 374             self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
 375             self.fix_bin_or_dylib(&bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"));
 376             let lib_dir = bin_root.join("lib");
 377             for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
 378                 let lib = t!(lib);
 379                 if lib.path().extension() == Some(OsStr::new("so")) {
 380                     self.fix_bin_or_dylib(&lib.path());
 381                 }
 382             }
 383             t!(fs::write(rustc_stamp, commit));
 384         }
 385     }
 386
 387     /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
 388     // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
 389     fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) {
 390         Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
 391     }
 392
 393     fn download_component(
 394         &self,
 395         mode: DownloadSource,
 396         filename: String,
 397         prefix: &str,
 398         key: &str,
 399         destination: &str,
 400     ) {
 401         let cache_dst = self.out.join("cache");
 402         let cache_dir = cache_dst.join(key);
 403         if !cache_dir.exists() {
 404             t!(fs::create_dir_all(&cache_dir));
 405         }
 406
 407         let bin_root = self.out.join(self.build.triple).join(destination);
 408         let tarball = cache_dir.join(&filename);
 409         let (base_url, url, should_verify) = match mode {
 410             DownloadSource::CI => (
 411                 self.stage0_metadata.config.artifacts_server.clone(),
 412                 format!("{key}/{filename}"),
 413                 false,
 414             ),
 415             DownloadSource::Dist => {
 416                 let dist_server = env::var("RUSTUP_DIST_SERVER")
 417                     .unwrap_or(self.stage0_metadata.config.dist_server.to_string());
 418                 // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
 419                 (dist_server, format!("dist/{key}/{filename}"), true)
 420             }
 421         };
 422
 423         // For the beta compiler, put special effort into ensuring the checksums are valid.
 424         // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
 425         // this on each and every nightly ...
 426         let checksum = if should_verify {
 427             let error = format!(
 428                 "src/stage0.json doesn't contain a checksum for {url}. \
 429                 Pre-built artifacts might not be available for this \
 430                 target at this time, see https://doc.rust-lang.org/nightly\
 431                 /rustc/platform-support.html for more information."
 432             );
 433             let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error);
 434             if tarball.exists() {
 435                 if self.verify(&tarball, sha256) {
 436                     self.unpack(&tarball, &bin_root, prefix);
 437                     return;
 438                 } else {
 439                     self.verbose(&format!(
 440                         "ignoring cached file {} due to failed verification",
 441                         tarball.display()
 442                     ));
 443                     self.remove(&tarball);
 444                 }
 445             }
 446             Some(sha256)
 447         } else if tarball.exists() {
 448             self.unpack(&tarball, &bin_root, prefix);
 449             return;
 450         } else {
 451             None
 452         };
 453
 454         self.download_file(&format!("{base_url}/{url}"), &tarball, "");
 455         if let Some(sha256) = checksum {
 456             if !self.verify(&tarball, sha256) {
 457                 panic!("failed to verify {}", tarball.display());
 458             }
 459         }
 460
 461         self.unpack(&tarball, &bin_root, prefix);
 462     }
 463
 464     pub(crate) fn maybe_download_ci_llvm(&self) {
 465         if !self.llvm_from_ci {
 466             return;
 467         }
 468         let llvm_root = self.ci_llvm_root();
 469         let llvm_stamp = llvm_root.join(".llvm-stamp");
 470         let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository());
 471         let key = format!("{}{}", llvm_sha, self.llvm_assertions);
 472         if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() {
 473             self.download_ci_llvm(&llvm_sha);
 474             for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
 475                 self.fix_bin_or_dylib(&t!(entry).path());
 476             }
 477
 478             // Update the timestamp of llvm-config to force rustc_llvm to be
 479             // rebuilt. This is a hacky workaround for a deficiency in Cargo where
 480             // the rerun-if-changed directive doesn't handle changes very well.
 481             // https://github.com/rust-lang/cargo/issues/10791
 482             // Cargo only compares the timestamp of the file relative to the last
 483             // time `rustc_llvm` build script ran. However, the timestamps of the
 484             // files in the tarball are in the past, so it doesn't trigger a
 485             // rebuild.
 486             let now = filetime::FileTime::from_system_time(std::time::SystemTime::now());
 487             let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build));
 488             t!(filetime::set_file_times(&llvm_config, now, now));
 489
 490             let llvm_lib = llvm_root.join("lib");
 491             for entry in t!(fs::read_dir(&llvm_lib)) {
 492                 let lib = t!(entry).path();
 493                 if lib.extension().map_or(false, |ext| ext == "so") {
 494                     self.fix_bin_or_dylib(&lib);
 495                 }
 496             }
 497             t!(fs::write(llvm_stamp, key));
 498         }
 499     }
 500
 501     fn download_ci_llvm(&self, llvm_sha: &str) {
 502         let llvm_assertions = self.llvm_assertions;
 503
 504         let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
 505         let cache_dst = self.out.join("cache");
 506         let rustc_cache = cache_dst.join(cache_prefix);
 507         if !rustc_cache.exists() {
 508             t!(fs::create_dir_all(&rustc_cache));
 509         }
 510         let base = if llvm_assertions {
 511             &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
 512         } else {
 513             &self.stage0_metadata.config.artifacts_server
 514         };
 515         let version = self.artifact_version_part(llvm_sha);
 516         let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple);
 517         let tarball = rustc_cache.join(&filename);
 518         if !tarball.exists() {
 519             let help_on_error = "error: failed to download llvm from ci
 520
 521     help: old builds get deleted after a certain time
 522     help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml:
 523
 524     [llvm]
 525     download-ci-llvm = false
 526     ";
 527             self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
 528         }
 529         let llvm_root = self.ci_llvm_root();
 530         self.unpack(&tarball, &llvm_root, "rust-dev");
 531     }
 532 }