src/tools/build-manifest/src/main.rs

   1 //! Build a dist manifest, hash and sign everything.
   2 //! This gets called by `promote-release`
   3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
   4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
   5 //! by rustbuild (in `src/bootstrap/dist.rs`).
   6
   7 mod checksum;
   8 mod manifest;
   9 mod versions;
  10
  11 use crate::checksum::Checksums;
  12 use crate::manifest::{Component, Manifest, Package, Rename, Target};
  13 use crate::versions::{PkgType, Versions};
  14 use std::collections::{BTreeMap, HashMap, HashSet};
  15 use std::env;
  16 use std::fs::{self, File};
  17 use std::path::{Path, PathBuf};
  18
  19 static HOSTS: &[&str] = &[
  20     "aarch64-apple-darwin",
  21     "aarch64-pc-windows-msvc",
  22     "aarch64-unknown-linux-gnu",
  23     "aarch64-unknown-linux-musl",
  24     "arm-unknown-linux-gnueabi",
  25     "arm-unknown-linux-gnueabihf",
  26     "armv7-unknown-linux-gnueabihf",
  27     "i686-apple-darwin",
  28     "i686-pc-windows-gnu",
  29     "i686-pc-windows-msvc",
  30     "i686-unknown-linux-gnu",
  31     "mips-unknown-linux-gnu",
  32     "mips64-unknown-linux-gnuabi64",
  33     "mips64el-unknown-linux-gnuabi64",
  34     "mipsel-unknown-linux-gnu",
  35     "mipsisa32r6-unknown-linux-gnu",
  36     "mipsisa32r6el-unknown-linux-gnu",
  37     "mipsisa64r6-unknown-linux-gnuabi64",
  38     "mipsisa64r6el-unknown-linux-gnuabi64",
  39     "powerpc-unknown-linux-gnu",
  40     "powerpc64-unknown-linux-gnu",
  41     "powerpc64le-unknown-linux-gnu",
  42     "riscv64gc-unknown-linux-gnu",
  43     "s390x-unknown-linux-gnu",
  44     "x86_64-apple-darwin",
  45     "x86_64-pc-windows-gnu",
  46     "x86_64-pc-windows-msvc",
  47     "x86_64-unknown-freebsd",
  48     "x86_64-unknown-illumos",
  49     "x86_64-unknown-linux-gnu",
  50     "x86_64-unknown-linux-musl",
  51     "x86_64-unknown-netbsd",
  52 ];
  53
  54 static TARGETS: &[&str] = &[
  55     "aarch64-apple-darwin",
  56     "aarch64-apple-ios",
  57     "aarch64-apple-ios-sim",
  58     "aarch64-fuchsia",
  59     "aarch64-linux-android",
  60     "aarch64-pc-windows-msvc",
  61     "aarch64-unknown-hermit",
  62     "aarch64-unknown-linux-gnu",
  63     "aarch64-unknown-linux-musl",
  64     "aarch64-unknown-none",
  65     "aarch64-unknown-none-softfloat",
  66     "aarch64-unknown-redox",
  67     "arm-linux-androideabi",
  68     "arm-unknown-linux-gnueabi",
  69     "arm-unknown-linux-gnueabihf",
  70     "arm-unknown-linux-musleabi",
  71     "arm-unknown-linux-musleabihf",
  72     "armv5te-unknown-linux-gnueabi",
  73     "armv5te-unknown-linux-musleabi",
  74     "armv7-apple-ios",
  75     "armv7-linux-androideabi",
  76     "thumbv7neon-linux-androideabi",
  77     "armv7-unknown-linux-gnueabi",
  78     "armv7-unknown-linux-gnueabihf",
  79     "armv7a-none-eabi",
  80     "thumbv7neon-unknown-linux-gnueabihf",
  81     "armv7-unknown-linux-musleabi",
  82     "armv7-unknown-linux-musleabihf",
  83     "armebv7r-none-eabi",
  84     "armebv7r-none-eabihf",
  85     "armv7r-none-eabi",
  86     "armv7r-none-eabihf",
  87     "armv7s-apple-ios",
  88     "asmjs-unknown-emscripten",
  89     "bpfeb-unknown-none",
  90     "bpfel-unknown-none",
  91     "i386-apple-ios",
  92     "i586-pc-windows-msvc",
  93     "i586-unknown-linux-gnu",
  94     "i586-unknown-linux-musl",
  95     "i686-apple-darwin",
  96     "i686-linux-android",
  97     "i686-pc-windows-gnu",
  98     "i686-pc-windows-msvc",
  99     "i686-unknown-freebsd",
 100     "i686-unknown-linux-gnu",
 101     "i686-unknown-linux-musl",
 102     "m68k-unknown-linux-gnu",
 103     "mips-unknown-linux-gnu",
 104     "mips-unknown-linux-musl",
 105     "mips64-unknown-linux-gnuabi64",
 106     "mips64-unknown-linux-muslabi64",
 107     "mips64el-unknown-linux-gnuabi64",
 108     "mips64el-unknown-linux-muslabi64",
 109     "mipsisa32r6-unknown-linux-gnu",
 110     "mipsisa32r6el-unknown-linux-gnu",
 111     "mipsisa64r6-unknown-linux-gnuabi64",
 112     "mipsisa64r6el-unknown-linux-gnuabi64",
 113     "mipsel-unknown-linux-gnu",
 114     "mipsel-unknown-linux-musl",
 115     "nvptx64-nvidia-cuda",
 116     "powerpc-unknown-linux-gnu",
 117     "powerpc64-unknown-linux-gnu",
 118     "powerpc64le-unknown-linux-gnu",
 119     "riscv32i-unknown-none-elf",
 120     "riscv32im-unknown-none-elf",
 121     "riscv32imc-unknown-none-elf",
 122     "riscv32imac-unknown-none-elf",
 123     "riscv32gc-unknown-linux-gnu",
 124     "riscv64imac-unknown-none-elf",
 125     "riscv64gc-unknown-none-elf",
 126     "riscv64gc-unknown-linux-gnu",
 127     "s390x-unknown-linux-gnu",
 128     "sparc64-unknown-linux-gnu",
 129     "sparcv9-sun-solaris",
 130     "thumbv6m-none-eabi",
 131     "thumbv7em-none-eabi",
 132     "thumbv7em-none-eabihf",
 133     "thumbv7m-none-eabi",
 134     "thumbv8m.base-none-eabi",
 135     "thumbv8m.main-none-eabi",
 136     "thumbv8m.main-none-eabihf",
 137     "wasm32-unknown-emscripten",
 138     "wasm32-unknown-unknown",
 139     "wasm32-wasi",
 140     "x86_64-apple-darwin",
 141     "x86_64-apple-ios",
 142     "x86_64-fortanix-unknown-sgx",
 143     "x86_64-fuchsia",
 144     "x86_64-linux-android",
 145     "x86_64-pc-windows-gnu",
 146     "x86_64-pc-windows-msvc",
 147     "x86_64-sun-solaris",
 148     "x86_64-pc-solaris",
 149     "x86_64-unknown-freebsd",
 150     "x86_64-unknown-illumos",
 151     "x86_64-unknown-linux-gnu",
 152     "x86_64-unknown-linux-gnux32",
 153     "x86_64-unknown-linux-musl",
 154     "x86_64-unknown-netbsd",
 155     "x86_64-unknown-none",
 156     "x86_64-unknown-redox",
 157     "x86_64-unknown-hermit",
 158 ];
 159
 160 /// This allows the manifest to contain rust-docs for hosts that don't build
 161 /// docs.
 162 ///
 163 /// Tuples of `(host_partial, host_instead)`. If the host does not have the
 164 /// rust-docs component available, then if the host name contains
 165 /// `host_partial`, it will use the docs from `host_instead` instead.
 166 ///
 167 /// The order here matters, more specific entries should be first.
 168 static DOCS_FALLBACK: &[(&str, &str)] = &[
 169     ("-apple-", "x86_64-apple-darwin"),
 170     ("aarch64", "aarch64-unknown-linux-gnu"),
 171     ("arm-", "aarch64-unknown-linux-gnu"),
 172     ("", "x86_64-unknown-linux-gnu"),
 173 ];
 174
 175 static MSI_INSTALLERS: &[&str] = &[
 176     "aarch64-pc-windows-msvc",
 177     "i686-pc-windows-gnu",
 178     "i686-pc-windows-msvc",
 179     "x86_64-pc-windows-gnu",
 180     "x86_64-pc-windows-msvc",
 181 ];
 182
 183 static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
 184
 185 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 186
 187 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview"];
 188
 189 macro_rules! t {
 190     ($e:expr) => {
 191         match $e {
 192             Ok(e) => e,
 193             Err(e) => panic!("{} failed with {}", stringify!($e), e),
 194         }
 195     };
 196 }
 197
 198 struct Builder {
 199     versions: Versions,
 200     checksums: Checksums,
 201     shipped_files: HashSet<String>,
 202
 203     input: PathBuf,
 204     output: PathBuf,
 205     s3_address: String,
 206     date: String,
 207 }
 208
 209 fn main() {
 210     let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
 211         num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
 212     } else {
 213         std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
 214     };
 215     rayon::ThreadPoolBuilder::new()
 216         .num_threads(num_threads)
 217         .build_global()
 218         .expect("failed to initialize Rayon");
 219
 220     let mut args = env::args().skip(1);
 221     let input = PathBuf::from(args.next().unwrap());
 222     let output = PathBuf::from(args.next().unwrap());
 223     let date = args.next().unwrap();
 224     let s3_address = args.next().unwrap();
 225     let channel = args.next().unwrap();
 226
 227     Builder {
 228         versions: Versions::new(&channel, &input).unwrap(),
 229         checksums: t!(Checksums::new()),
 230         shipped_files: HashSet::new(),
 231
 232         input,
 233         output,
 234         s3_address,
 235         date,
 236     }
 237     .build();
 238 }
 239
 240 impl Builder {
 241     fn build(&mut self) {
 242         self.check_toolstate();
 243         let manifest = self.build_manifest();
 244
 245         let channel = self.versions.channel().to_string();
 246         self.write_channel_files(&channel, &manifest);
 247         if channel == "stable" {
 248             // channel-rust-1.XX.YY.toml
 249             let rust_version = self.versions.rustc_version().to_string();
 250             self.write_channel_files(&rust_version, &manifest);
 251
 252             // channel-rust-1.XX.toml
 253             let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
 254             self.write_channel_files(&major_minor, &manifest);
 255         }
 256
 257         if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
 258             self.write_shipped_files(&Path::new(&path));
 259         }
 260
 261         t!(self.checksums.store_cache());
 262     }
 263
 264     /// If a tool does not pass its tests on *any* of Linux and Windows, don't ship
 265     /// it on *all* targets, because tools like Miri can "cross-run" programs for
 266     /// different targets, for example, run a program for `x86_64-pc-windows-msvc`
 267     /// on `x86_64-unknown-linux-gnu`.
 268     /// Right now, we do this only for Miri.
 269     fn check_toolstate(&mut self) {
 270         for file in &["toolstates-linux.json", "toolstates-windows.json"] {
 271             let toolstates: Option<HashMap<String, String>> = File::open(self.input.join(file))
 272                 .ok()
 273                 .and_then(|f| serde_json::from_reader(&f).ok());
 274             let toolstates = toolstates.unwrap_or_else(|| {
 275                 println!("WARNING: `{}` missing/malformed; assuming all tools failed", file);
 276                 HashMap::default() // Use empty map if anything went wrong.
 277             });
 278             // Mark some tools as missing based on toolstate.
 279             if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
 280                 println!("Miri tests are not passing, removing component");
 281                 self.versions.disable_version(&PkgType::Miri);
 282                 break;
 283             }
 284         }
 285     }
 286
 287     fn build_manifest(&mut self) -> Manifest {
 288         let mut manifest = Manifest {
 289             manifest_version: "2".to_string(),
 290             date: self.date.to_string(),
 291             pkg: BTreeMap::new(),
 292             artifacts: BTreeMap::new(),
 293             renames: BTreeMap::new(),
 294             profiles: BTreeMap::new(),
 295         };
 296         self.add_packages_to(&mut manifest);
 297         self.add_artifacts_to(&mut manifest);
 298         self.add_profiles_to(&mut manifest);
 299         self.add_renames_to(&mut manifest);
 300         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 301
 302         self.checksums.fill_missing_checksums(&mut manifest);
 303
 304         manifest
 305     }
 306
 307     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 308         macro_rules! package {
 309             ($name:expr, $targets:expr) => {
 310                 self.package($name, &mut manifest.pkg, $targets, &[])
 311             };
 312         }
 313         package!("rustc", HOSTS);
 314         package!("rustc-dev", HOSTS);
 315         package!("reproducible-artifacts", HOSTS);
 316         package!("rustc-docs", HOSTS);
 317         package!("cargo", HOSTS);
 318         package!("rust-mingw", MINGW);
 319         package!("rust-std", TARGETS);
 320         self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
 321         package!("rust-src", &["*"]);
 322         package!("rls-preview", HOSTS);
 323         package!("rust-analyzer-preview", HOSTS);
 324         package!("clippy-preview", HOSTS);
 325         package!("miri-preview", HOSTS);
 326         package!("rustfmt-preview", HOSTS);
 327         package!("rust-analysis", TARGETS);
 328         package!("llvm-tools-preview", TARGETS);
 329     }
 330
 331     fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
 332         manifest.add_artifact("source-code", |artifact| {
 333             let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
 334             artifact.add_tarball(self, "*", &tarball);
 335         });
 336
 337         manifest.add_artifact("installer-msi", |artifact| {
 338             for target in MSI_INSTALLERS {
 339                 let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
 340                 artifact.add_file(self, target, &msi);
 341             }
 342         });
 343
 344         manifest.add_artifact("installer-pkg", |artifact| {
 345             for target in PKG_INSTALLERS {
 346                 let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
 347                 artifact.add_file(self, target, &pkg);
 348             }
 349         });
 350     }
 351
 352     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 353         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
 354         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
 355         profile(
 356             "default",
 357             &[
 358                 "rustc",
 359                 "cargo",
 360                 "rust-std",
 361                 "rust-mingw",
 362                 "rust-docs",
 363                 "rustfmt-preview",
 364                 "clippy-preview",
 365             ],
 366         );
 367         profile(
 368             "complete",
 369             &[
 370                 "rustc",
 371                 "cargo",
 372                 "rust-std",
 373                 "rust-mingw",
 374                 "rust-docs",
 375                 "rustfmt-preview",
 376                 "clippy-preview",
 377                 "rls-preview",
 378                 "rust-analyzer-preview",
 379                 "rust-src",
 380                 "llvm-tools-preview",
 381                 "rust-analysis",
 382                 "miri-preview",
 383             ],
 384         );
 385
 386         // The compiler libraries are not stable for end users, and they're also huge, so we only
 387         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
 388         // for users to install the additional component manually, if needed.
 389         if self.versions.channel() == "nightly" {
 390             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
 391             // Do not include the rustc-docs component for now, as it causes
 392             // conflicts with the rust-docs component when installed. See
 393             // #75833.
 394             // self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
 395         }
 396     }
 397
 398     fn add_renames_to(&self, manifest: &mut Manifest) {
 399         let mut rename = |from: &str, to: &str| {
 400             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
 401         };
 402         rename("rls", "rls-preview");
 403         rename("rustfmt", "rustfmt-preview");
 404         rename("clippy", "clippy-preview");
 405         rename("miri", "miri-preview");
 406         rename("rust-analyzer", "rust-analyzer-preview");
 407     }
 408
 409     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 410         let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
 411         let mut pkg = Package {
 412             version: version_info.version.expect("missing Rust version"),
 413             git_commit_hash: version_info.git_commit,
 414             target: BTreeMap::new(),
 415         };
 416         for host in HOSTS {
 417             if let Some(target) = self.target_host_combination(host, &manifest) {
 418                 pkg.target.insert(host.to_string(), target);
 419             } else {
 420                 pkg.target.insert(host.to_string(), Target::unavailable());
 421                 continue;
 422             }
 423         }
 424         pkg
 425     }
 426
 427     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 428         let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
 429
 430         let mut target = Target::from_compressed_tar(self, &filename);
 431         if !target.available {
 432             return None;
 433         }
 434
 435         let mut components = Vec::new();
 436         let mut extensions = Vec::new();
 437
 438         let host_component = |pkg| Component::from_str(pkg, host);
 439
 440         // rustc/rust-std/cargo/docs are all required,
 441         // and so is rust-mingw if it's available for the target.
 442         components.extend(vec![
 443             host_component("rustc"),
 444             host_component("rust-std"),
 445             host_component("cargo"),
 446             host_component("rust-docs"),
 447         ]);
 448         if host.contains("pc-windows-gnu") {
 449             components.push(host_component("rust-mingw"));
 450         }
 451
 452         // Tools are always present in the manifest,
 453         // but might be marked as unavailable if they weren't built.
 454         extensions.extend(vec![
 455             host_component("clippy-preview"),
 456             host_component("miri-preview"),
 457             host_component("rls-preview"),
 458             host_component("rust-analyzer-preview"),
 459             host_component("rustfmt-preview"),
 460             host_component("llvm-tools-preview"),
 461             host_component("rust-analysis"),
 462         ]);
 463
 464         extensions.extend(
 465             TARGETS
 466                 .iter()
 467                 .filter(|&&target| target != host)
 468                 .map(|target| Component::from_str("rust-std", target)),
 469         );
 470         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
 471         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
 472         extensions.push(Component::from_str("rust-src", "*"));
 473
 474         // If the components/extensions don't actually exist for this
 475         // particular host/target combination then nix it entirely from our
 476         // lists.
 477         let has_component = |c: &Component| {
 478             if c.target == "*" {
 479                 return true;
 480             }
 481             let pkg = match manifest.pkg.get(&c.pkg) {
 482                 Some(p) => p,
 483                 None => return false,
 484             };
 485             pkg.target.get(&c.target).is_some()
 486         };
 487         extensions.retain(&has_component);
 488         components.retain(&has_component);
 489
 490         target.components = Some(components);
 491         target.extensions = Some(extensions);
 492         Some(target)
 493     }
 494
 495     fn profile(
 496         &mut self,
 497         profile_name: &str,
 498         dst: &mut BTreeMap<String, Vec<String>>,
 499         pkgs: &[&str],
 500     ) {
 501         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
 502     }
 503
 504     fn extend_profile(
 505         &mut self,
 506         profile_name: &str,
 507         dst: &mut BTreeMap<String, Vec<String>>,
 508         pkgs: &[&str],
 509     ) {
 510         dst.get_mut(profile_name)
 511             .expect("existing profile")
 512             .extend(pkgs.iter().map(|s| (*s).to_owned()));
 513     }
 514
 515     fn package(
 516         &mut self,
 517         pkgname: &str,
 518         dst: &mut BTreeMap<String, Package>,
 519         targets: &[&str],
 520         fallback: &[(&str, &str)],
 521     ) {
 522         let version_info = self
 523             .versions
 524             .version(&PkgType::from_component(pkgname))
 525             .expect("failed to load package version");
 526         let mut is_present = version_info.present;
 527
 528         // Never ship nightly-only components for other trains.
 529         if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
 530             is_present = false; // Pretend the component is entirely missing.
 531         }
 532
 533         macro_rules! tarball_name {
 534             ($target_name:expr) => {
 535                 self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap()
 536             };
 537         }
 538         let mut target_from_compressed_tar = |target_name| {
 539             let target = Target::from_compressed_tar(self, &tarball_name!(target_name));
 540             if target.available {
 541                 return target;
 542             }
 543             for (substr, fallback_target) in fallback {
 544                 if target_name.contains(substr) {
 545                     let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target));
 546                     // Fallbacks must always be available.
 547                     assert!(t.available);
 548                     return t;
 549                 }
 550             }
 551             Target::unavailable()
 552         };
 553
 554         let targets = targets
 555             .iter()
 556             .map(|name| {
 557                 let target = if is_present {
 558                     target_from_compressed_tar(name)
 559                 } else {
 560                     // If the component is not present for this build add it anyway but mark it as
 561                     // unavailable -- this way rustup won't allow upgrades without --force
 562                     Target::unavailable()
 563                 };
 564                 (name.to_string(), target)
 565             })
 566             .collect();
 567
 568         dst.insert(
 569             pkgname.to_string(),
 570             Package {
 571                 version: version_info.version.unwrap_or_default(),
 572                 git_commit_hash: version_info.git_commit,
 573                 target: targets,
 574             },
 575         );
 576     }
 577
 578     fn url(&self, path: &Path) -> String {
 579         let file_name = path.file_name().unwrap().to_str().unwrap();
 580         format!("{}/{}/{}", self.s3_address, self.date, file_name)
 581     }
 582
 583     fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) {
 584         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 585         self.write(&manifest.date, channel_name, "-date.txt");
 586         self.write(
 587             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 588             channel_name,
 589             "-git-commit-hash.txt",
 590         );
 591     }
 592
 593     fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) {
 594         let name = format!("channel-rust-{}{}", channel_name, suffix);
 595         self.shipped_files.insert(name.clone());
 596
 597         let dst = self.output.join(name);
 598         t!(fs::write(&dst, contents));
 599     }
 600
 601     fn write_shipped_files(&self, path: &Path) {
 602         let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::<Vec<_>>();
 603         files.sort();
 604         let content = format!("{}\n", files.join("\n"));
 605
 606         t!(std::fs::write(path, content.as_bytes()));
 607     }
 608 }