src/tools/build-manifest/src/main.rs

   1 //! Build a dist manifest, hash and sign everything.
   2 //! This gets called by `promote-release`
   3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
   4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
   5 //! by rustbuild (in `src/bootstrap/dist.rs`).
   6
   7 use serde::Serialize;
   8
   9 use std::collections::BTreeMap;
  10 use std::collections::HashMap;
  11 use std::env;
  12 use std::fs::{self, File};
  13 use std::io::{self, Read, Write};
  14 use std::path::{Path, PathBuf};
  15 use std::process::{Command, Stdio};
  16
  17 static HOSTS: &[&str] = &[
  18     "aarch64-unknown-linux-gnu",
  19     "arm-unknown-linux-gnueabi",
  20     "arm-unknown-linux-gnueabihf",
  21     "armv7-unknown-linux-gnueabihf",
  22     "i686-apple-darwin",
  23     "i686-pc-windows-gnu",
  24     "i686-pc-windows-msvc",
  25     "i686-unknown-linux-gnu",
  26     "mips-unknown-linux-gnu",
  27     "mips64-unknown-linux-gnuabi64",
  28     "mips64el-unknown-linux-gnuabi64",
  29     "mipsel-unknown-linux-gnu",
  30     "mipsisa32r6-unknown-linux-gnu",
  31     "mipsisa32r6el-unknown-linux-gnu",
  32     "mipsisa64r6-unknown-linux-gnuabi64",
  33     "mipsisa64r6el-unknown-linux-gnuabi64",
  34     "powerpc-unknown-linux-gnu",
  35     "powerpc64-unknown-linux-gnu",
  36     "powerpc64le-unknown-linux-gnu",
  37     "riscv64gc-unknown-linux-gnu",
  38     "s390x-unknown-linux-gnu",
  39     "x86_64-apple-darwin",
  40     "x86_64-pc-windows-gnu",
  41     "x86_64-pc-windows-msvc",
  42     "x86_64-unknown-freebsd",
  43     "x86_64-unknown-illumos",
  44     "x86_64-unknown-linux-gnu",
  45     "x86_64-unknown-linux-musl",
  46     "x86_64-unknown-netbsd",
  47 ];
  48
  49 static TARGETS: &[&str] = &[
  50     "aarch64-apple-ios",
  51     "aarch64-fuchsia",
  52     "aarch64-linux-android",
  53     "aarch64-pc-windows-msvc",
  54     "aarch64-unknown-cloudabi",
  55     "aarch64-unknown-hermit",
  56     "aarch64-unknown-linux-gnu",
  57     "aarch64-unknown-linux-musl",
  58     "aarch64-unknown-none",
  59     "aarch64-unknown-none-softfloat",
  60     "aarch64-unknown-redox",
  61     "arm-linux-androideabi",
  62     "arm-unknown-linux-gnueabi",
  63     "arm-unknown-linux-gnueabihf",
  64     "arm-unknown-linux-musleabi",
  65     "arm-unknown-linux-musleabihf",
  66     "armv5te-unknown-linux-gnueabi",
  67     "armv5te-unknown-linux-musleabi",
  68     "armv7-apple-ios",
  69     "armv7-linux-androideabi",
  70     "thumbv7neon-linux-androideabi",
  71     "armv7-unknown-linux-gnueabi",
  72     "armv7-unknown-linux-gnueabihf",
  73     "armv7a-none-eabi",
  74     "thumbv7neon-unknown-linux-gnueabihf",
  75     "armv7-unknown-linux-musleabi",
  76     "armv7-unknown-linux-musleabihf",
  77     "armebv7r-none-eabi",
  78     "armebv7r-none-eabihf",
  79     "armv7r-none-eabi",
  80     "armv7r-none-eabihf",
  81     "armv7s-apple-ios",
  82     "asmjs-unknown-emscripten",
  83     "i386-apple-ios",
  84     "i586-pc-windows-msvc",
  85     "i586-unknown-linux-gnu",
  86     "i586-unknown-linux-musl",
  87     "i686-apple-darwin",
  88     "i686-linux-android",
  89     "i686-pc-windows-gnu",
  90     "i686-pc-windows-msvc",
  91     "i686-unknown-freebsd",
  92     "i686-unknown-linux-gnu",
  93     "i686-unknown-linux-musl",
  94     "mips-unknown-linux-gnu",
  95     "mips-unknown-linux-musl",
  96     "mips64-unknown-linux-gnuabi64",
  97     "mips64-unknown-linux-muslabi64",
  98     "mips64el-unknown-linux-gnuabi64",
  99     "mips64el-unknown-linux-muslabi64",
 100     "mipsisa32r6-unknown-linux-gnu",
 101     "mipsisa32r6el-unknown-linux-gnu",
 102     "mipsisa64r6-unknown-linux-gnuabi64",
 103     "mipsisa64r6el-unknown-linux-gnuabi64",
 104     "mipsel-unknown-linux-gnu",
 105     "mipsel-unknown-linux-musl",
 106     "nvptx64-nvidia-cuda",
 107     "powerpc-unknown-linux-gnu",
 108     "powerpc64-unknown-linux-gnu",
 109     "powerpc64le-unknown-linux-gnu",
 110     "riscv32i-unknown-none-elf",
 111     "riscv32imc-unknown-none-elf",
 112     "riscv32imac-unknown-none-elf",
 113     "riscv64imac-unknown-none-elf",
 114     "riscv64gc-unknown-none-elf",
 115     "riscv64gc-unknown-linux-gnu",
 116     "s390x-unknown-linux-gnu",
 117     "sparc64-unknown-linux-gnu",
 118     "sparcv9-sun-solaris",
 119     "thumbv6m-none-eabi",
 120     "thumbv7em-none-eabi",
 121     "thumbv7em-none-eabihf",
 122     "thumbv7m-none-eabi",
 123     "thumbv8m.base-none-eabi",
 124     "thumbv8m.main-none-eabi",
 125     "thumbv8m.main-none-eabihf",
 126     "wasm32-unknown-emscripten",
 127     "wasm32-unknown-unknown",
 128     "wasm32-wasi",
 129     "x86_64-apple-darwin",
 130     "x86_64-apple-ios",
 131     "x86_64-fortanix-unknown-sgx",
 132     "x86_64-fuchsia",
 133     "x86_64-linux-android",
 134     "x86_64-pc-windows-gnu",
 135     "x86_64-pc-windows-msvc",
 136     "x86_64-rumprun-netbsd",
 137     "x86_64-sun-solaris",
 138     "x86_64-pc-solaris",
 139     "x86_64-unknown-cloudabi",
 140     "x86_64-unknown-freebsd",
 141     "x86_64-unknown-illumos",
 142     "x86_64-unknown-linux-gnu",
 143     "x86_64-unknown-linux-gnux32",
 144     "x86_64-unknown-linux-musl",
 145     "x86_64-unknown-netbsd",
 146     "x86_64-unknown-redox",
 147     "x86_64-unknown-hermit",
 148 ];
 149
 150 static DOCS_TARGETS: &[&str] = &[
 151     "i686-apple-darwin",
 152     "i686-pc-windows-gnu",
 153     "i686-pc-windows-msvc",
 154     "i686-unknown-linux-gnu",
 155     "x86_64-apple-darwin",
 156     "x86_64-pc-windows-gnu",
 157     "x86_64-pc-windows-msvc",
 158     "x86_64-unknown-linux-gnu",
 159     "x86_64-unknown-linux-musl",
 160 ];
 161
 162 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 163
 164 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
 165
 166 #[derive(Serialize)]
 167 #[serde(rename_all = "kebab-case")]
 168 struct Manifest {
 169     manifest_version: String,
 170     date: String,
 171     pkg: BTreeMap<String, Package>,
 172     renames: BTreeMap<String, Rename>,
 173     profiles: BTreeMap<String, Vec<String>>,
 174 }
 175
 176 #[derive(Serialize)]
 177 struct Package {
 178     version: String,
 179     git_commit_hash: Option<String>,
 180     target: BTreeMap<String, Target>,
 181 }
 182
 183 #[derive(Serialize)]
 184 struct Rename {
 185     to: String,
 186 }
 187
 188 #[derive(Serialize, Default)]
 189 struct Target {
 190     available: bool,
 191     url: Option<String>,
 192     hash: Option<String>,
 193     xz_url: Option<String>,
 194     xz_hash: Option<String>,
 195     components: Option<Vec<Component>>,
 196     extensions: Option<Vec<Component>>,
 197 }
 198
 199 impl Target {
 200     fn unavailable() -> Self {
 201         Self::default()
 202     }
 203 }
 204
 205 #[derive(Serialize)]
 206 struct Component {
 207     pkg: String,
 208     target: String,
 209 }
 210
 211 impl Component {
 212     fn from_str(pkg: &str, target: &str) -> Self {
 213         Self { pkg: pkg.to_string(), target: target.to_string() }
 214     }
 215 }
 216
 217 macro_rules! t {
 218     ($e:expr) => {
 219         match $e {
 220             Ok(e) => e,
 221             Err(e) => panic!("{} failed with {}", stringify!($e), e),
 222         }
 223     };
 224 }
 225
 226 struct Builder {
 227     rust_release: String,
 228     cargo_release: String,
 229     rls_release: String,
 230     rust_analyzer_release: String,
 231     clippy_release: String,
 232     rustfmt_release: String,
 233     llvm_tools_release: String,
 234     miri_release: String,
 235
 236     input: PathBuf,
 237     output: PathBuf,
 238     gpg_passphrase: String,
 239     digests: BTreeMap<String, String>,
 240     s3_address: String,
 241     date: String,
 242
 243     rust_version: Option<String>,
 244     cargo_version: Option<String>,
 245     rls_version: Option<String>,
 246     rust_analyzer_version: Option<String>,
 247     clippy_version: Option<String>,
 248     rustfmt_version: Option<String>,
 249     llvm_tools_version: Option<String>,
 250     miri_version: Option<String>,
 251
 252     rust_git_commit_hash: Option<String>,
 253     cargo_git_commit_hash: Option<String>,
 254     rls_git_commit_hash: Option<String>,
 255     rust_analyzer_git_commit_hash: Option<String>,
 256     clippy_git_commit_hash: Option<String>,
 257     rustfmt_git_commit_hash: Option<String>,
 258     llvm_tools_git_commit_hash: Option<String>,
 259     miri_git_commit_hash: Option<String>,
 260
 261     should_sign: bool,
 262 }
 263
 264 fn main() {
 265     // Avoid signing packages while manually testing
 266     // Do NOT set this envvar in CI
 267     let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
 268
 269     // Safety check to ensure signing is always enabled on CI
 270     // The CI environment variable is set by both Travis and AppVeyor
 271     if !should_sign && env::var("CI").is_ok() {
 272         println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
 273         println!("If you're not running this on CI, unset the 'CI' env var.");
 274         panic!();
 275     }
 276
 277     let mut args = env::args().skip(1);
 278     let input = PathBuf::from(args.next().unwrap());
 279     let output = PathBuf::from(args.next().unwrap());
 280     let date = args.next().unwrap();
 281     let rust_release = args.next().unwrap();
 282     let s3_address = args.next().unwrap();
 283     let cargo_release = args.next().unwrap();
 284     let rls_release = args.next().unwrap();
 285     let rust_analyzer_release = args.next().unwrap();
 286     let clippy_release = args.next().unwrap();
 287     let miri_release = args.next().unwrap();
 288     let rustfmt_release = args.next().unwrap();
 289     let llvm_tools_release = args.next().unwrap();
 290
 291     // Do not ask for a passphrase while manually testing
 292     let mut passphrase = String::new();
 293     if should_sign {
 294         // `x.py` passes the passphrase via stdin.
 295         t!(io::stdin().read_to_string(&mut passphrase));
 296     }
 297
 298     Builder {
 299         rust_release,
 300         cargo_release,
 301         rls_release,
 302         rust_analyzer_release,
 303         clippy_release,
 304         rustfmt_release,
 305         llvm_tools_release,
 306         miri_release,
 307
 308         input,
 309         output,
 310         gpg_passphrase: passphrase,
 311         digests: BTreeMap::new(),
 312         s3_address,
 313         date,
 314
 315         rust_version: None,
 316         cargo_version: None,
 317         rls_version: None,
 318         rust_analyzer_version: None,
 319         clippy_version: None,
 320         rustfmt_version: None,
 321         llvm_tools_version: None,
 322         miri_version: None,
 323
 324         rust_git_commit_hash: None,
 325         cargo_git_commit_hash: None,
 326         rls_git_commit_hash: None,
 327         rust_analyzer_git_commit_hash: None,
 328         clippy_git_commit_hash: None,
 329         rustfmt_git_commit_hash: None,
 330         llvm_tools_git_commit_hash: None,
 331         miri_git_commit_hash: None,
 332
 333         should_sign,
 334     }
 335     .build();
 336 }
 337
 338 enum PkgType {
 339     RustSrc,
 340     Cargo,
 341     Rls,
 342     RustAnalyzer,
 343     Clippy,
 344     Rustfmt,
 345     LlvmTools,
 346     Miri,
 347     Other,
 348 }
 349
 350 impl PkgType {
 351     fn from_component(component: &str) -> Self {
 352         use PkgType::*;
 353         match component {
 354             "rust-src" => RustSrc,
 355             "cargo" => Cargo,
 356             "rls" | "rls-preview" => Rls,
 357             "rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer,
 358             "clippy" | "clippy-preview" => Clippy,
 359             "rustfmt" | "rustfmt-preview" => Rustfmt,
 360             "llvm-tools" | "llvm-tools-preview" => LlvmTools,
 361             "miri" | "miri-preview" => Miri,
 362             _ => Other,
 363         }
 364     }
 365 }
 366
 367 impl Builder {
 368     fn build(&mut self) {
 369         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 370         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 371         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 372         self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu");
 373         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
 374         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 375         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
 376         self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu");
 377
 378         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 379         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 380         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 381         self.rust_analyzer_git_commit_hash =
 382             self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu");
 383         self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
 384         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 385         self.llvm_tools_git_commit_hash =
 386             self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu");
 387         self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu");
 388
 389         self.check_toolstate();
 390         self.digest_and_sign();
 391         let manifest = self.build_manifest();
 392         self.write_channel_files(&self.rust_release, &manifest);
 393
 394         if self.rust_release != "beta" && self.rust_release != "nightly" {
 395             self.write_channel_files("stable", &manifest);
 396         }
 397     }
 398
 399     /// If a tool does not pass its tests, don't ship it.
 400     /// Right now, we do this only for Miri.
 401     fn check_toolstate(&mut self) {
 402         let toolstates: Option<HashMap<String, String>> =
 403             File::open(self.input.join("toolstates-linux.json"))
 404                 .ok()
 405                 .and_then(|f| serde_json::from_reader(&f).ok());
 406         let toolstates = toolstates.unwrap_or_else(|| {
 407             println!(
 408                 "WARNING: `toolstates-linux.json` missing/malformed; \
 409                 assuming all tools failed"
 410             );
 411             HashMap::default() // Use empty map if anything went wrong.
 412         });
 413         // Mark some tools as missing based on toolstate.
 414         if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
 415             println!("Miri tests are not passing, removing component");
 416             self.miri_version = None;
 417             self.miri_git_commit_hash = None;
 418         }
 419     }
 420
 421     /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
 422     fn digest_and_sign(&mut self) {
 423         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 424             let filename = file.file_name().unwrap().to_str().unwrap();
 425             let digest = self.hash(&file);
 426             self.sign(&file);
 427             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 428         }
 429     }
 430
 431     fn build_manifest(&mut self) -> Manifest {
 432         let mut manifest = Manifest {
 433             manifest_version: "2".to_string(),
 434             date: self.date.to_string(),
 435             pkg: BTreeMap::new(),
 436             renames: BTreeMap::new(),
 437             profiles: BTreeMap::new(),
 438         };
 439         self.add_packages_to(&mut manifest);
 440         self.add_profiles_to(&mut manifest);
 441         self.add_renames_to(&mut manifest);
 442         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
 443         manifest
 444     }
 445
 446     fn add_packages_to(&mut self, manifest: &mut Manifest) {
 447         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
 448         package("rustc", HOSTS);
 449         package("rustc-dev", HOSTS);
 450         package("cargo", HOSTS);
 451         package("rust-mingw", MINGW);
 452         package("rust-std", TARGETS);
 453         package("rust-docs", DOCS_TARGETS);
 454         package("rust-src", &["*"]);
 455         package("rls-preview", HOSTS);
 456         package("rust-analyzer-preview", HOSTS);
 457         package("clippy-preview", HOSTS);
 458         package("miri-preview", HOSTS);
 459         package("rustfmt-preview", HOSTS);
 460         package("rust-analysis", TARGETS);
 461         package("llvm-tools-preview", TARGETS);
 462     }
 463
 464     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
 465         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
 466         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
 467         profile(
 468             "default",
 469             &[
 470                 "rustc",
 471                 "cargo",
 472                 "rust-std",
 473                 "rust-mingw",
 474                 "rust-docs",
 475                 "rustfmt-preview",
 476                 "clippy-preview",
 477             ],
 478         );
 479         profile(
 480             "complete",
 481             &[
 482                 "rustc",
 483                 "cargo",
 484                 "rust-std",
 485                 "rust-mingw",
 486                 "rust-docs",
 487                 "rustfmt-preview",
 488                 "clippy-preview",
 489                 "rls-preview",
 490                 "rust-analyzer-preview",
 491                 "rust-src",
 492                 "llvm-tools-preview",
 493                 "rust-analysis",
 494                 "miri-preview",
 495             ],
 496         );
 497
 498         // The compiler libraries are not stable for end users, and they're also huge, so we only
 499         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
 500         // for users to install the additional component manually, if needed.
 501         if self.rust_release == "nightly" {
 502             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
 503         }
 504     }
 505
 506     fn add_renames_to(&self, manifest: &mut Manifest) {
 507         let mut rename = |from: &str, to: &str| {
 508             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
 509         };
 510         rename("rls", "rls-preview");
 511         rename("rustfmt", "rustfmt-preview");
 512         rename("clippy", "clippy-preview");
 513         rename("miri", "miri-preview");
 514     }
 515
 516     fn rust_package(&mut self, manifest: &Manifest) -> Package {
 517         let mut pkg = Package {
 518             version: self
 519                 .cached_version("rust")
 520                 .as_ref()
 521                 .expect("Couldn't find Rust version")
 522                 .clone(),
 523             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 524             target: BTreeMap::new(),
 525         };
 526         for host in HOSTS {
 527             if let Some(target) = self.target_host_combination(host, &manifest) {
 528                 pkg.target.insert(host.to_string(), target);
 529             } else {
 530                 pkg.target.insert(host.to_string(), Target::unavailable());
 531                 continue;
 532             }
 533         }
 534         pkg
 535     }
 536
 537     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
 538         let filename = self.filename("rust", host);
 539         let digest = self.digests.remove(&filename)?;
 540         let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 541         let xz_digest = self.digests.remove(&xz_filename);
 542         let mut components = Vec::new();
 543         let mut extensions = Vec::new();
 544
 545         let host_component = |pkg| Component::from_str(pkg, host);
 546
 547         // rustc/rust-std/cargo/docs are all required,
 548         // and so is rust-mingw if it's available for the target.
 549         components.extend(vec![
 550             host_component("rustc"),
 551             host_component("rust-std"),
 552             host_component("cargo"),
 553             host_component("rust-docs"),
 554         ]);
 555         if host.contains("pc-windows-gnu") {
 556             components.push(host_component("rust-mingw"));
 557         }
 558
 559         // Tools are always present in the manifest,
 560         // but might be marked as unavailable if they weren't built.
 561         extensions.extend(vec![
 562             host_component("clippy-preview"),
 563             host_component("miri-preview"),
 564             host_component("rls-preview"),
 565             host_component("rust-analyzer-preview"),
 566             host_component("rustfmt-preview"),
 567             host_component("llvm-tools-preview"),
 568             host_component("rust-analysis"),
 569         ]);
 570
 571         extensions.extend(
 572             TARGETS
 573                 .iter()
 574                 .filter(|&&target| target != host)
 575                 .map(|target| Component::from_str("rust-std", target)),
 576         );
 577         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
 578         extensions.push(Component::from_str("rust-src", "*"));
 579
 580         // If the components/extensions don't actually exist for this
 581         // particular host/target combination then nix it entirely from our
 582         // lists.
 583         let has_component = |c: &Component| {
 584             if c.target == "*" {
 585                 return true;
 586             }
 587             let pkg = match manifest.pkg.get(&c.pkg) {
 588                 Some(p) => p,
 589                 None => return false,
 590             };
 591             pkg.target.get(&c.target).is_some()
 592         };
 593         extensions.retain(&has_component);
 594         components.retain(&has_component);
 595
 596         Some(Target {
 597             available: true,
 598             url: Some(self.url(&filename)),
 599             hash: Some(digest),
 600             xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 601             xz_hash: xz_digest,
 602             components: Some(components),
 603             extensions: Some(extensions),
 604         })
 605     }
 606
 607     fn profile(
 608         &mut self,
 609         profile_name: &str,
 610         dst: &mut BTreeMap<String, Vec<String>>,
 611         pkgs: &[&str],
 612     ) {
 613         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
 614     }
 615
 616     fn extend_profile(
 617         &mut self,
 618         profile_name: &str,
 619         dst: &mut BTreeMap<String, Vec<String>>,
 620         pkgs: &[&str],
 621     ) {
 622         dst.get_mut(profile_name)
 623             .expect("existing profile")
 624             .extend(pkgs.iter().map(|s| (*s).to_owned()));
 625     }
 626
 627     fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
 628         let (version, mut is_present) = self
 629             .cached_version(pkgname)
 630             .as_ref()
 631             .cloned()
 632             .map(|version| (version, true))
 633             .unwrap_or_default(); // `is_present` defaults to `false` here.
 634
 635         // Never ship nightly-only components for other trains.
 636         if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
 637             is_present = false; // Pretend the component is entirely missing.
 638         }
 639
 640         let targets = targets
 641             .iter()
 642             .map(|name| {
 643                 if is_present {
 644                     // The component generally exists, but it might still be missing for this target.
 645                     let filename = self.filename(pkgname, name);
 646                     let digest = match self.digests.remove(&filename) {
 647                         Some(digest) => digest,
 648                         // This component does not exist for this target -- skip it.
 649                         None => return (name.to_string(), Target::unavailable()),
 650                     };
 651                     let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 652                     let xz_digest = self.digests.remove(&xz_filename);
 653
 654                     (
 655                         name.to_string(),
 656                         Target {
 657                             available: true,
 658                             url: Some(self.url(&filename)),
 659                             hash: Some(digest),
 660                             xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 661                             xz_hash: xz_digest,
 662                             components: None,
 663                             extensions: None,
 664                         },
 665                     )
 666                 } else {
 667                     // If the component is not present for this build add it anyway but mark it as
 668                     // unavailable -- this way rustup won't allow upgrades without --force
 669                     (name.to_string(), Target::unavailable())
 670                 }
 671             })
 672             .collect();
 673
 674         dst.insert(
 675             pkgname.to_string(),
 676             Package {
 677                 version,
 678                 git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 679                 target: targets,
 680             },
 681         );
 682     }
 683
 684     fn url(&self, filename: &str) -> String {
 685         format!("{}/{}/{}", self.s3_address, self.date, filename)
 686     }
 687
 688     fn filename(&self, component: &str, target: &str) -> String {
 689         use PkgType::*;
 690         match PkgType::from_component(component) {
 691             RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
 692             Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
 693             Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
 694             RustAnalyzer => {
 695                 format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target)
 696             }
 697             Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
 698             Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
 699             LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
 700             Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
 701             Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
 702         }
 703     }
 704
 705     fn cached_version(&self, component: &str) -> &Option<String> {
 706         use PkgType::*;
 707         match PkgType::from_component(component) {
 708             Cargo => &self.cargo_version,
 709             Rls => &self.rls_version,
 710             RustAnalyzer => &self.rust_analyzer_version,
 711             Clippy => &self.clippy_version,
 712             Rustfmt => &self.rustfmt_version,
 713             LlvmTools => &self.llvm_tools_version,
 714             Miri => &self.miri_version,
 715             _ => &self.rust_version,
 716         }
 717     }
 718
 719     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 720         use PkgType::*;
 721         match PkgType::from_component(component) {
 722             Cargo => &self.cargo_git_commit_hash,
 723             Rls => &self.rls_git_commit_hash,
 724             RustAnalyzer => &self.rust_analyzer_git_commit_hash,
 725             Clippy => &self.clippy_git_commit_hash,
 726             Rustfmt => &self.rustfmt_git_commit_hash,
 727             LlvmTools => &self.llvm_tools_git_commit_hash,
 728             Miri => &self.miri_git_commit_hash,
 729             _ => &self.rust_git_commit_hash,
 730         }
 731     }
 732
 733     fn version(&self, component: &str, target: &str) -> Option<String> {
 734         self.untar(component, target, |filename| format!("{}/version", filename))
 735     }
 736
 737     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 738         self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
 739     }
 740
 741     fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
 742     where
 743         F: FnOnce(String) -> String,
 744     {
 745         let mut cmd = Command::new("tar");
 746         let filename = self.filename(component, target);
 747         cmd.arg("xf")
 748             .arg(self.input.join(&filename))
 749             .arg(dir(filename.replace(".tar.gz", "")))
 750             .arg("-O");
 751         let output = t!(cmd.output());
 752         if output.status.success() {
 753             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 754         } else {
 755             None
 756         }
 757     }
 758
 759     fn hash(&self, path: &Path) -> String {
 760         let sha = t!(Command::new("shasum")
 761             .arg("-a")
 762             .arg("256")
 763             .arg(path.file_name().unwrap())
 764             .current_dir(path.parent().unwrap())
 765             .output());
 766         assert!(sha.status.success());
 767
 768         let filename = path.file_name().unwrap().to_str().unwrap();
 769         let sha256 = self.output.join(format!("{}.sha256", filename));
 770         t!(fs::write(&sha256, &sha.stdout));
 771
 772         let stdout = String::from_utf8_lossy(&sha.stdout);
 773         stdout.split_whitespace().next().unwrap().to_string()
 774     }
 775
 776     fn sign(&self, path: &Path) {
 777         if !self.should_sign {
 778             return;
 779         }
 780
 781         let filename = path.file_name().unwrap().to_str().unwrap();
 782         let asc = self.output.join(format!("{}.asc", filename));
 783         println!("signing: {:?}", path);
 784         let mut cmd = Command::new("gpg");
 785         cmd.arg("--pinentry-mode=loopback")
 786             .arg("--no-tty")
 787             .arg("--yes")
 788             .arg("--batch")
 789             .arg("--passphrase-fd")
 790             .arg("0")
 791             .arg("--personal-digest-preferences")
 792             .arg("SHA512")
 793             .arg("--armor")
 794             .arg("--output")
 795             .arg(&asc)
 796             .arg("--detach-sign")
 797             .arg(path)
 798             .stdin(Stdio::piped());
 799         let mut child = t!(cmd.spawn());
 800         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 801         assert!(t!(child.wait()).success());
 802     }
 803
 804     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 805         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 806         self.write(&manifest.date, channel_name, "-date.txt");
 807         self.write(
 808             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 809             channel_name,
 810             "-git-commit-hash.txt",
 811         );
 812     }
 813
 814     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 815         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 816         t!(fs::write(&dst, contents));
 817         self.hash(&dst);
 818         self.sign(&dst);
 819     }
 820 }