src/tools/build-manifest/src/main.rs

   1 #![doc = include_str!("../README.md")]
   2
   3 mod checksum;
   4 mod manifest;
   5 mod versions;
   6
   7 use crate::checksum::Checksums;
   8 use crate::manifest::{Component, Manifest, Package, Rename, Target};
   9 use crate::versions::{PkgType, Versions};
  10 use std::collections::{BTreeMap, HashSet};
  11 use std::env;
  12 use std::fs;
  13 use std::path::{Path, PathBuf};
  14
  15 static HOSTS: &[&str] = &[
  16     "aarch64-apple-darwin",
  17     "aarch64-pc-windows-msvc",
  18     "aarch64-unknown-linux-gnu",
  19     "aarch64-unknown-linux-musl",
  20     "arm-unknown-linux-gnueabi",
  21     "arm-unknown-linux-gnueabihf",
  22     "armv7-unknown-linux-gnueabihf",
  23     "i686-apple-darwin",
  24     "i686-pc-windows-gnu",
  25     "i686-pc-windows-msvc",
  26     "i686-unknown-linux-gnu",
  27     "mips-unknown-linux-gnu",
  28     "mips64-unknown-linux-gnuabi64",
  29     "mips64el-unknown-linux-gnuabi64",
  30     "mipsel-unknown-linux-gnu",
  31     "mipsisa32r6-unknown-linux-gnu",
  32     "mipsisa32r6el-unknown-linux-gnu",
  33     "mipsisa64r6-unknown-linux-gnuabi64",
  34     "mipsisa64r6el-unknown-linux-gnuabi64",
  35     "powerpc-unknown-linux-gnu",
  36     "powerpc64-unknown-linux-gnu",
  37     "powerpc64le-unknown-linux-gnu",
  38     "riscv64gc-unknown-linux-gnu",
  39     "s390x-unknown-linux-gnu",
  40     "x86_64-apple-darwin",
  41     "x86_64-pc-windows-gnu",
  42     "x86_64-pc-windows-msvc",
  43     "x86_64-unknown-freebsd",
  44     "x86_64-unknown-illumos",
  45     "x86_64-unknown-linux-gnu",
  46     "x86_64-unknown-linux-musl",
  47     "x86_64-unknown-netbsd",
  48 ];
  49
  50 static TARGETS: &[&str] = &[
  51     "aarch64-apple-darwin",
  52     "aarch64-apple-ios",
  53     "aarch64-apple-ios-sim",
  54     "aarch64-fuchsia",
  55     "aarch64-linux-android",
  56     "aarch64-pc-windows-msvc",
  57     "aarch64-unknown-hermit",
  58     "aarch64-unknown-linux-gnu",
  59     "aarch64-unknown-linux-musl",
  60     "aarch64-unknown-none",
  61     "aarch64-unknown-none-softfloat",
  62     "aarch64-unknown-redox",
  63     "aarch64-unknown-uefi",
  64     "arm-linux-androideabi",
  65     "arm-unknown-linux-gnueabi",
  66     "arm-unknown-linux-gnueabihf",
  67     "arm-unknown-linux-musleabi",
  68     "arm-unknown-linux-musleabihf",
  69     "armv5te-unknown-linux-gnueabi",
  70     "armv5te-unknown-linux-musleabi",
  71     "armv7-apple-ios",
  72     "armv7-linux-androideabi",
  73     "thumbv7neon-linux-androideabi",
  74     "armv7-unknown-linux-gnueabi",
  75     "armv7-unknown-linux-gnueabihf",
  76     "armv7a-none-eabi",
  77     "thumbv7neon-unknown-linux-gnueabihf",
  78     "armv7-unknown-linux-musleabi",
  79     "armv7-unknown-linux-musleabihf",
  80     "armebv7r-none-eabi",
  81     "armebv7r-none-eabihf",
  82     "armv7r-none-eabi",
  83     "armv7r-none-eabihf",
  84     "armv7s-apple-ios",
  85     "asmjs-unknown-emscripten",
  86     "bpfeb-unknown-none",
  87     "bpfel-unknown-none",
  88     "i386-apple-ios",
  89     "i586-pc-windows-msvc",
  90     "i586-unknown-linux-gnu",
  91     "i586-unknown-linux-musl",
  92     "i686-apple-darwin",
  93     "i686-linux-android",
  94     "i686-pc-windows-gnu",
  95     "i686-pc-windows-msvc",
  96     "i686-unknown-freebsd",
  97     "i686-unknown-linux-gnu",
  98     "i686-unknown-linux-musl",
  99     "i686-unknown-uefi",
 100     "m68k-unknown-linux-gnu",
 101     "mips-unknown-linux-gnu",
 102     "mips-unknown-linux-musl",
 103     "mips64-unknown-linux-gnuabi64",
 104     "mips64-unknown-linux-muslabi64",
 105     "mips64el-unknown-linux-gnuabi64",
 106     "mips64el-unknown-linux-muslabi64",
 107     "mipsisa32r6-unknown-linux-gnu",
 108     "mipsisa32r6el-unknown-linux-gnu",
 109     "mipsisa64r6-unknown-linux-gnuabi64",
 110     "mipsisa64r6el-unknown-linux-gnuabi64",
 111     "mipsel-unknown-linux-gnu",
 112     "mipsel-unknown-linux-musl",
 113     "nvptx64-nvidia-cuda",
 114     "powerpc-unknown-linux-gnu",
 115     "powerpc64-unknown-linux-gnu",
 116     "powerpc64le-unknown-linux-gnu",
 117     "riscv32i-unknown-none-elf",
 118     "riscv32im-unknown-none-elf",
 119     "riscv32imc-unknown-none-elf",
 120     "riscv32imac-unknown-none-elf",
 121     "riscv32gc-unknown-linux-gnu",
 122     "riscv64imac-unknown-none-elf",
 123     "riscv64gc-unknown-none-elf",
 124     "riscv64gc-unknown-linux-gnu",
 125     "s390x-unknown-linux-gnu",
 126     "sparc64-unknown-linux-gnu",
 127     "sparcv9-sun-solaris",
 128     "thumbv6m-none-eabi",
 129     "thumbv7em-none-eabi",
 130     "thumbv7em-none-eabihf",
 131     "thumbv7m-none-eabi",
 132     "thumbv8m.base-none-eabi",
 133     "thumbv8m.main-none-eabi",
 134     "thumbv8m.main-none-eabihf",
 135     "wasm32-unknown-emscripten",
 136     "wasm32-unknown-unknown",
 137     "wasm32-wasi",
 138     "x86_64-apple-darwin",
 139     "x86_64-apple-ios",
 140     "x86_64-fortanix-unknown-sgx",
 141     "x86_64-fuchsia",
 142     "x86_64-linux-android",
 143     "x86_64-pc-windows-gnu",
 144     "x86_64-pc-windows-msvc",
 145     "x86_64-sun-solaris",
 146     "x86_64-pc-solaris",
 147     "x86_64-unknown-freebsd",
 148     "x86_64-unknown-illumos",
 149     "x86_64-unknown-linux-gnu",
 150     "x86_64-unknown-linux-gnux32",
 151     "x86_64-unknown-linux-musl",
 152     "x86_64-unknown-netbsd",
 153     "x86_64-unknown-none",
 154     "x86_64-unknown-redox",
 155     "x86_64-unknown-hermit",
 156     "x86_64-unknown-uefi",
 157 ];
 158
 159 /// This allows the manifest to contain rust-docs for hosts that don't build
 160 /// docs.
 161 ///
 162 /// Tuples of `(host_partial, host_instead)`. If the host does not have the
 163 /// rust-docs component available, then if the host name contains
 164 /// `host_partial`, it will use the docs from `host_instead` instead.
 165 ///
 166 /// The order here matters, more specific entries should be first.
 167 static DOCS_FALLBACK: &[(&str, &str)] = &[
 168     ("-apple-", "x86_64-apple-darwin"),
 169     ("aarch64", "aarch64-unknown-linux-gnu"),
 170     ("arm-", "aarch64-unknown-linux-gnu"),
 171     ("", "x86_64-unknown-linux-gnu"),
 172 ];
 173
 174 static MSI_INSTALLERS: &[&str] = &[
 175     "aarch64-pc-windows-msvc",
 176     "i686-pc-windows-gnu",
 177     "i686-pc-windows-msvc",
 178     "x86_64-pc-windows-gnu",
 179     "x86_64-pc-windows-msvc",
 180 ];
 181
 182 static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
 183
 184 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 185
 186 static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
 187
 188 macro_rules! t {
 189     ($e:expr) => {
 190         match $e {
 191             Ok(e) => e,
 192             Err(e) => panic!("{} failed with {}", stringify!($e), e),
 193         }
 194     };
 195     ($e:expr, $extra:expr) => {
 196         match $e {
 197             Ok(e) => e,
 198             Err(e) => panic!("{} failed with {}: {}", stringify!($e), e, $extra),
 199         }
 200     };
 201 }
 202
 203 struct Builder {
 204     versions: Versions,
 205     checksums: Checksums,
 206     shipped_files: HashSet<String>,
 207
 208     input: PathBuf,
 209     output: PathBuf,
 210     s3_address: String,
 211     date: String,
 212 }
 213
 214 fn main() {
 215     let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
 216         num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
 217     } else {
 218         std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
 219     };
 220     rayon::ThreadPoolBuilder::new()
 221         .num_threads(num_threads)
 222         .build_global()
 223         .expect("failed to initialize Rayon");
 224
 225     let mut args = env::args().skip(1);
 226     let input = PathBuf::from(args.next().unwrap());
 227     let output = PathBuf::from(args.next().unwrap());
 228     let date = args.next().unwrap();
 229     let s3_address = args.next().unwrap();
 230     let channel = args.next().unwrap();
 231
 232     Builder {
 233         versions: Versions::new(&channel, &input).unwrap(),
 234         checksums: t!(Checksums::new()),
 235         shipped_files: HashSet::new(),
 236
 237         input,
 238         output,
 239         s3_address,
 240         date,
 241     }
 242     .build();
 243 }
 244
 245 impl Builder {
 246     fn build(&mut self) {
 247         let manifest = self.build_manifest();
 248
 249         let channel = self.versions.channel().to_string();
 250         self.write_channel_files(&channel, &manifest);
 251         if channel == "stable" {
 252             // channel-rust-1.XX.YY.toml
 253             let rust_version = self.versions.rustc_version().to_string();
 254             self.write_channel_files(&rust_version, &manifest);
 255
 256             // channel-rust-1.XX.toml
 257             let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
 258             self.write_channel_files(&major_minor, &manifest);
 259         }
 260
 261         if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
 262             self.write_shipped_files(&Path::new(&path));
 263         }
 264
 265         t!(self.checksums.store_cache());
 266     }
 267
 268     fn build_manifest(&mut self) -> Manifest {
 269         let mut manifest = Manifest {
 270             manifest_version: "2".to_string(),
 271             date: self.date.to_string(),
 272             pkg: BTreeMap::new(),
 273             artifacts: BTreeMap::new(),
 274             renames: BTreeMap::new(),
 275             profiles: BTreeMap::new(),
 276         };
 277         self.add_packages_to(&mut manifest);
 278         self.add_artifacts_to(&mut manifest);
 279         self.add_profiles_to(&mut manifest);
 280         self.add_renames_to(&mut manifest);
 281         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 282
 283         self.checksums.fill_missing_checksums(&mut manifest);
 284
 285         manifest
 286     }
 287
 288     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 289         for pkg in PkgType::all() {
 290             self.package(pkg, &mut manifest.pkg);
 291         }
 292     }
 293
 294     fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
 295         manifest.add_artifact("source-code", |artifact| {
 296             let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
 297             artifact.add_tarball(self, "*", &tarball);
 298         });
 299
 300         manifest.add_artifact("installer-msi", |artifact| {
 301             for target in MSI_INSTALLERS {
 302                 let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
 303                 artifact.add_file(self, target, &msi);
 304             }
 305         });
 306
 307         manifest.add_artifact("installer-pkg", |artifact| {
 308             for target in PKG_INSTALLERS {
 309                 let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
 310                 artifact.add_file(self, target, &pkg);
 311             }
 312         });
 313     }
 314
 315     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 316         use PkgType::*;
 317
 318         let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs);
 319
 320         // Use a Vec here to make sure we don't exclude any components in an earlier profile.
 321         let minimal = vec![Rustc, Cargo, RustStd, RustMingw];
 322         profile("minimal", &minimal);
 323
 324         let mut default = minimal;
 325         default.extend([HtmlDocs, Rustfmt, Clippy]);
 326         profile("default", &default);
 327
 328         // NOTE: this profile is effectively deprecated; do not add new components to it.
 329         let mut complete = default;
 330         complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
 331         profile("complete", &complete);
 332
 333         // The compiler libraries are not stable for end users, and they're also huge, so we only
 334         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
 335         // for users to install the additional component manually, if needed.
 336         if self.versions.channel() == "nightly" {
 337             self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]);
 338             // Do not include the rustc-docs component for now, as it causes
 339             // conflicts with the rust-docs component when installed. See
 340             // #75833.
 341             // self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
 342         }
 343     }
 344
 345     fn add_renames_to(&self, manifest: &mut Manifest) {
 346         let mut rename = |from: &str, to: &str| {
 347             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
 348         };
 349         for pkg in PkgType::all() {
 350             if pkg.is_preview() {
 351                 rename(pkg.tarball_component_name(), &pkg.manifest_component_name());
 352             }
 353         }
 354     }
 355
 356     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 357         let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
 358         let mut pkg = Package {
 359             version: version_info.version.expect("missing Rust version"),
 360             git_commit_hash: version_info.git_commit,
 361             target: BTreeMap::new(),
 362         };
 363         for host in HOSTS {
 364             if let Some(target) = self.target_host_combination(host, &manifest) {
 365                 pkg.target.insert(host.to_string(), target);
 366             } else {
 367                 pkg.target.insert(host.to_string(), Target::unavailable());
 368                 continue;
 369             }
 370         }
 371         pkg
 372     }
 373
 374     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 375         let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
 376
 377         let mut target = Target::from_compressed_tar(self, &filename);
 378         if !target.available {
 379             return None;
 380         }
 381
 382         let mut components = Vec::new();
 383         let mut extensions = Vec::new();
 384
 385         let host_component = |pkg: &_| Component::from_pkg(pkg, host);
 386
 387         for pkg in PkgType::all() {
 388             match pkg {
 389                 // rustc/rust-std/cargo/docs are all required
 390                 PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => {
 391                     components.push(host_component(pkg));
 392                 }
 393                 PkgType::RustStd => {
 394                     components.push(host_component(pkg));
 395                     extensions.extend(
 396                         TARGETS
 397                             .iter()
 398                             .filter(|&&target| target != host)
 399                             .map(|target| Component::from_pkg(pkg, target)),
 400                     );
 401                 }
 402                 // so is rust-mingw if it's available for the target
 403                 PkgType::RustMingw => {
 404                     if host.contains("pc-windows-gnu") {
 405                         components.push(host_component(pkg));
 406                     }
 407                 }
 408                 // Tools are always present in the manifest,
 409                 // but might be marked as unavailable if they weren't built.
 410                 PkgType::Clippy
 411                 | PkgType::Miri
 412                 | PkgType::Rls
 413                 | PkgType::RustAnalyzer
 414                 | PkgType::Rustfmt
 415                 | PkgType::LlvmTools
 416                 | PkgType::RustAnalysis
 417                 | PkgType::JsonDocs => {
 418                     extensions.push(host_component(pkg));
 419                 }
 420                 PkgType::RustcDev | PkgType::RustcDocs => {
 421                     extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target)));
 422                 }
 423                 PkgType::RustSrc => {
 424                     extensions.push(Component::from_pkg(pkg, "*"));
 425                 }
 426                 PkgType::Rust => {}
 427                 // NOTE: this is intentional, these artifacts aren't intended to be used with rustup
 428                 PkgType::ReproducibleArtifacts => {}
 429             }
 430         }
 431
 432         // If the components/extensions don't actually exist for this
 433         // particular host/target combination then nix it entirely from our
 434         // lists.
 435         let has_component = |c: &Component| {
 436             if c.target == "*" {
 437                 return true;
 438             }
 439             let pkg = match manifest.pkg.get(&c.pkg) {
 440                 Some(p) => p,
 441                 None => return false,
 442             };
 443             pkg.target.get(&c.target).is_some()
 444         };
 445         extensions.retain(&has_component);
 446         components.retain(&has_component);
 447
 448         target.components = Some(components);
 449         target.extensions = Some(extensions);
 450         Some(target)
 451     }
 452
 453     fn profile(
 454         &mut self,
 455         profile_name: &str,
 456         dst: &mut BTreeMap<String, Vec<String>>,
 457         pkgs: &[PkgType],
 458     ) {
 459         dst.insert(
 460             profile_name.to_owned(),
 461             pkgs.iter().map(|s| s.manifest_component_name()).collect(),
 462         );
 463     }
 464
 465     fn extend_profile(
 466         &mut self,
 467         profile_name: &str,
 468         dst: &mut BTreeMap<String, Vec<String>>,
 469         pkgs: &[PkgType],
 470     ) {
 471         dst.get_mut(profile_name)
 472             .expect("existing profile")
 473             .extend(pkgs.iter().map(|s| s.manifest_component_name()));
 474     }
 475
 476     fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap<String, Package>) {
 477         if *pkg == PkgType::Rust {
 478             // This is handled specially by `rust_package` later.
 479             // Order is important, so don't call `rust_package` here.
 480             return;
 481         }
 482
 483         let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] };
 484         let version_info = self.versions.version(&pkg).expect("failed to load package version");
 485         let mut is_present = version_info.present;
 486
 487         // Never ship nightly-only components for other trains.
 488         if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) {
 489             is_present = false; // Pretend the component is entirely missing.
 490         }
 491
 492         macro_rules! tarball_name {
 493             ($target_name:expr) => {
 494                 self.versions.tarball_name(pkg, $target_name).unwrap()
 495             };
 496         }
 497         let mut target_from_compressed_tar = |target_name| {
 498             let target = Target::from_compressed_tar(self, &tarball_name!(target_name));
 499             if target.available {
 500                 return target;
 501             }
 502             for (substr, fallback_target) in fallback {
 503                 if target_name.contains(substr) {
 504                     let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target));
 505                     // Fallbacks should typically be available on 'production' builds
 506                     // but may not be available for try builds, which only build one target by
 507                     // default. Ideally we'd gate this being a hard error on whether we're in a
 508                     // production build or not, but it's not information that's readily available
 509                     // here.
 510                     if !t.available {
 511                         eprintln!(
 512                             "{:?} not available for fallback",
 513                             tarball_name!(fallback_target)
 514                         );
 515                         continue;
 516                     }
 517                     return t;
 518                 }
 519             }
 520             Target::unavailable()
 521         };
 522
 523         let targets = pkg
 524             .targets()
 525             .iter()
 526             .map(|name| {
 527                 let target = if is_present {
 528                     target_from_compressed_tar(name)
 529                 } else {
 530                     // If the component is not present for this build add it anyway but mark it as
 531                     // unavailable -- this way rustup won't allow upgrades without --force
 532                     Target::unavailable()
 533                 };
 534                 (name.to_string(), target)
 535             })
 536             .collect();
 537
 538         dst.insert(
 539             pkg.manifest_component_name(),
 540             Package {
 541                 version: version_info.version.unwrap_or_default(),
 542                 git_commit_hash: version_info.git_commit,
 543                 target: targets,
 544             },
 545         );
 546     }
 547
 548     fn url(&self, path: &Path) -> String {
 549         let file_name = path.file_name().unwrap().to_str().unwrap();
 550         format!("{}/{}/{}", self.s3_address, self.date, file_name)
 551     }
 552
 553     fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) {
 554         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 555         self.write(&manifest.date, channel_name, "-date.txt");
 556         self.write(
 557             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 558             channel_name,
 559             "-git-commit-hash.txt",
 560         );
 561     }
 562
 563     fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) {
 564         let name = format!("channel-rust-{}{}", channel_name, suffix);
 565         self.shipped_files.insert(name.clone());
 566
 567         let dst = self.output.join(name);
 568         t!(fs::write(&dst, contents), format!("failed to create manifest {}", dst.display()));
 569     }
 570
 571     fn write_shipped_files(&self, path: &Path) {
 572         let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::<Vec<_>>();
 573         files.sort();
 574         let content = format!("{}\n", files.join("\n"));
 575
 576         t!(std::fs::write(path, content.as_bytes()));
 577     }
 578 }