src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 #[macro_use]
  13 extern crate serde_derive;
  14
  15 use std::collections::BTreeMap;
  16 use std::env;
  17 use std::fs::File;
  18 use std::io::{self, Read, Write};
  19 use std::path::{PathBuf, Path};
  20 use std::process::{Command, Stdio};
  21
  22 static HOSTS: &'static [&'static str] = &[
  23     "aarch64-unknown-linux-gnu",
  24     "arm-unknown-linux-gnueabi",
  25     "arm-unknown-linux-gnueabihf",
  26     "armv7-unknown-linux-gnueabihf",
  27     "i686-apple-darwin",
  28     "i686-pc-windows-gnu",
  29     "i686-pc-windows-msvc",
  30     "i686-unknown-linux-gnu",
  31     "mips-unknown-linux-gnu",
  32     "mips64-unknown-linux-gnuabi64",
  33     "mips64el-unknown-linux-gnuabi64",
  34     "mipsel-unknown-linux-gnu",
  35     "powerpc-unknown-linux-gnu",
  36     "powerpc64-unknown-linux-gnu",
  37     "powerpc64le-unknown-linux-gnu",
  38     "s390x-unknown-linux-gnu",
  39     "x86_64-apple-darwin",
  40     "x86_64-pc-windows-gnu",
  41     "x86_64-pc-windows-msvc",
  42     "x86_64-unknown-freebsd",
  43     "x86_64-unknown-linux-gnu",
  44     "x86_64-unknown-netbsd",
  45 ];
  46
  47 static TARGETS: &'static [&'static str] = &[
  48     "aarch64-apple-ios",
  49     "aarch64-fuchsia",
  50     "aarch64-linux-android",
  51     "aarch64-unknown-cloudabi",
  52     "aarch64-unknown-linux-gnu",
  53     "aarch64-unknown-linux-musl",
  54     "arm-linux-androideabi",
  55     "arm-unknown-linux-gnueabi",
  56     "arm-unknown-linux-gnueabihf",
  57     "arm-unknown-linux-musleabi",
  58     "arm-unknown-linux-musleabihf",
  59     "armv5te-unknown-linux-gnueabi",
  60     "armv5te-unknown-linux-musleabi",
  61     "armv7-apple-ios",
  62     "armv7-linux-androideabi",
  63     "armv7-unknown-cloudabi-eabihf",
  64     "armv7-unknown-linux-gnueabihf",
  65     "armv7-unknown-linux-musleabihf",
  66     "armebv7r-none-eabihf",
  67     "armv7s-apple-ios",
  68     "asmjs-unknown-emscripten",
  69     "i386-apple-ios",
  70     "i586-pc-windows-msvc",
  71     "i586-unknown-linux-gnu",
  72     "i586-unknown-linux-musl",
  73     "i686-apple-darwin",
  74     "i686-linux-android",
  75     "i686-pc-windows-gnu",
  76     "i686-pc-windows-msvc",
  77     "i686-unknown-cloudabi",
  78     "i686-unknown-freebsd",
  79     "i686-unknown-linux-gnu",
  80     "i686-unknown-linux-musl",
  81     "mips-unknown-linux-gnu",
  82     "mips-unknown-linux-musl",
  83     "mips64-unknown-linux-gnuabi64",
  84     "mips64el-unknown-linux-gnuabi64",
  85     "mipsel-unknown-linux-gnu",
  86     "mipsel-unknown-linux-musl",
  87     "powerpc-unknown-linux-gnu",
  88     "powerpc-unknown-linux-gnuspe",
  89     "powerpc64-unknown-linux-gnu",
  90     "powerpc64le-unknown-linux-gnu",
  91     "powerpc64le-unknown-linux-musl",
  92     "riscv32imac-unknown-none-elf",
  93     "s390x-unknown-linux-gnu",
  94     "sparc-unknown-linux-gnu",
  95     "sparc64-unknown-linux-gnu",
  96     "sparcv9-sun-solaris",
  97     "thumbv6m-none-eabi",
  98     "thumbv7em-none-eabi",
  99     "thumbv7em-none-eabihf",
 100     "thumbv7m-none-eabi",
 101     "wasm32-unknown-emscripten",
 102     "wasm32-unknown-unknown",
 103     "x86_64-apple-darwin",
 104     "x86_64-apple-ios",
 105     "x86_64-fuchsia",
 106     "x86_64-linux-android",
 107     "x86_64-pc-windows-gnu",
 108     "x86_64-pc-windows-msvc",
 109     "x86_64-rumprun-netbsd",
 110     "x86_64-sun-solaris",
 111     "x86_64-unknown-cloudabi",
 112     "x86_64-unknown-freebsd",
 113     "x86_64-unknown-hermit",
 114     "x86_64-unknown-linux-gnu",
 115     "x86_64-unknown-linux-gnux32",
 116     "x86_64-unknown-linux-musl",
 117     "x86_64-unknown-netbsd",
 118     "x86_64-unknown-redox",
 119 ];
 120
 121 static MINGW: &'static [&'static str] = &[
 122     "i686-pc-windows-gnu",
 123     "x86_64-pc-windows-gnu",
 124 ];
 125
 126 #[derive(Serialize)]
 127 #[serde(rename_all = "kebab-case")]
 128 struct Manifest {
 129     manifest_version: String,
 130     date: String,
 131     pkg: BTreeMap<String, Package>,
 132     renames: BTreeMap<String, Rename>
 133 }
 134
 135 #[derive(Serialize)]
 136 struct Package {
 137     version: String,
 138     git_commit_hash: Option<String>,
 139     target: BTreeMap<String, Target>,
 140 }
 141
 142 #[derive(Serialize)]
 143 struct Rename {
 144     to: String,
 145 }
 146
 147 #[derive(Serialize)]
 148 struct Target {
 149     available: bool,
 150     url: Option<String>,
 151     hash: Option<String>,
 152     xz_url: Option<String>,
 153     xz_hash: Option<String>,
 154     components: Option<Vec<Component>>,
 155     extensions: Option<Vec<Component>>,
 156 }
 157
 158 impl Target {
 159     fn unavailable() -> Target {
 160         Target {
 161             available: false,
 162             url: None,
 163             hash: None,
 164             xz_url: None,
 165             xz_hash: None,
 166             components: None,
 167             extensions: None,
 168         }
 169     }
 170 }
 171
 172 #[derive(Serialize)]
 173 struct Component {
 174     pkg: String,
 175     target: String,
 176 }
 177
 178 macro_rules! t {
 179     ($e:expr) => (match $e {
 180         Ok(e) => e,
 181         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 182     })
 183 }
 184
 185 struct Builder {
 186     rust_release: String,
 187     cargo_release: String,
 188     rls_release: String,
 189     clippy_release: String,
 190     rustfmt_release: String,
 191     llvm_tools_release: String,
 192
 193     input: PathBuf,
 194     output: PathBuf,
 195     gpg_passphrase: String,
 196     digests: BTreeMap<String, String>,
 197     s3_address: String,
 198     date: String,
 199
 200     rust_version: Option<String>,
 201     cargo_version: Option<String>,
 202     rls_version: Option<String>,
 203     clippy_version: Option<String>,
 204     rustfmt_version: Option<String>,
 205     llvm_tools_version: Option<String>,
 206
 207     rust_git_commit_hash: Option<String>,
 208     cargo_git_commit_hash: Option<String>,
 209     rls_git_commit_hash: Option<String>,
 210     clippy_git_commit_hash: Option<String>,
 211     rustfmt_git_commit_hash: Option<String>,
 212     llvm_tools_git_commit_hash: Option<String>,
 213 }
 214
 215 fn main() {
 216     let mut args = env::args().skip(1);
 217     let input = PathBuf::from(args.next().unwrap());
 218     let output = PathBuf::from(args.next().unwrap());
 219     let date = args.next().unwrap();
 220     let rust_release = args.next().unwrap();
 221     let cargo_release = args.next().unwrap();
 222     let rls_release = args.next().unwrap();
 223     let clippy_release = args.next().unwrap();
 224     let rustfmt_release = args.next().unwrap();
 225     let llvm_tools_release = args.next().unwrap();
 226     let s3_address = args.next().unwrap();
 227     let mut passphrase = String::new();
 228     t!(io::stdin().read_to_string(&mut passphrase));
 229
 230     Builder {
 231         rust_release,
 232         cargo_release,
 233         rls_release,
 234         clippy_release,
 235         rustfmt_release,
 236         llvm_tools_release,
 237
 238         input,
 239         output,
 240         gpg_passphrase: passphrase,
 241         digests: BTreeMap::new(),
 242         s3_address,
 243         date,
 244
 245         rust_version: None,
 246         cargo_version: None,
 247         rls_version: None,
 248         clippy_version: None,
 249         rustfmt_version: None,
 250         llvm_tools_version: None,
 251
 252         rust_git_commit_hash: None,
 253         cargo_git_commit_hash: None,
 254         rls_git_commit_hash: None,
 255         clippy_git_commit_hash: None,
 256         rustfmt_git_commit_hash: None,
 257         llvm_tools_git_commit_hash: None,
 258     }.build();
 259 }
 260
 261 impl Builder {
 262     fn build(&mut self) {
 263         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 264         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 265         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 266         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
 267         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 268         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
 269
 270         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 271         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 272         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 273         self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
 274         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 275         self.llvm_tools_git_commit_hash = self.git_commit_hash("llvm-tools",
 276                                                                "x86_64-unknown-linux-gnu");
 277
 278         self.digest_and_sign();
 279         let manifest = self.build_manifest();
 280         self.write_channel_files(&self.rust_release, &manifest);
 281
 282         if self.rust_release != "beta" && self.rust_release != "nightly" {
 283             self.write_channel_files("stable", &manifest);
 284         }
 285     }
 286
 287     fn digest_and_sign(&mut self) {
 288         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 289             let filename = file.file_name().unwrap().to_str().unwrap();
 290             let digest = self.hash(&file);
 291             self.sign(&file);
 292             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 293         }
 294     }
 295
 296     fn build_manifest(&mut self) -> Manifest {
 297         let mut manifest = Manifest {
 298             manifest_version: "2".to_string(),
 299             date: self.date.to_string(),
 300             pkg: BTreeMap::new(),
 301             renames: BTreeMap::new(),
 302         };
 303
 304         self.package("rustc", &mut manifest.pkg, HOSTS);
 305         self.package("cargo", &mut manifest.pkg, HOSTS);
 306         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 307         self.package("rust-std", &mut manifest.pkg, TARGETS);
 308         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 309         self.package("rust-src", &mut manifest.pkg, &["*"]);
 310         self.package("rls-preview", &mut manifest.pkg, HOSTS);
 311         self.package("clippy-preview", &mut manifest.pkg, HOSTS);
 312         self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
 313         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
 314         self.package("llvm-tools-preview", &mut manifest.pkg, TARGETS);
 315
 316         let clippy_present = manifest.pkg.contains_key("clippy-preview");
 317         let rls_present = manifest.pkg.contains_key("rls-preview");
 318         let rustfmt_present = manifest.pkg.contains_key("rustfmt-preview");
 319         let llvm_tools_present = manifest.pkg.contains_key("llvm-tools-preview");
 320
 321         if rls_present {
 322             manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
 323         }
 324
 325         let mut pkg = Package {
 326             version: self.cached_version("rust")
 327                          .as_ref()
 328                          .expect("Couldn't find Rust version")
 329                          .clone(),
 330             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 331             target: BTreeMap::new(),
 332         };
 333         for host in HOSTS {
 334             let filename = self.filename("rust", host);
 335             let digest = match self.digests.remove(&filename) {
 336                 Some(digest) => digest,
 337                 None => {
 338                     pkg.target.insert(host.to_string(), Target::unavailable());
 339                     continue
 340                 }
 341             };
 342             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 343             let xz_digest = self.digests.remove(&xz_filename);
 344             let mut components = Vec::new();
 345             let mut extensions = Vec::new();
 346
 347             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
 348             // if it's available for the target.
 349             components.extend(vec![
 350                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 351                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 352                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 353                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
 354             ]);
 355             if host.contains("pc-windows-gnu") {
 356                 components.push(Component {
 357                     pkg: "rust-mingw".to_string(),
 358                     target: host.to_string(),
 359                 });
 360             }
 361
 362             if clippy_present {
 363                 extensions.push(Component {
 364                     pkg: "clippy-preview".to_string(),
 365                     target: host.to_string(),
 366                 });
 367             }
 368             if rls_present {
 369                 extensions.push(Component {
 370                     pkg: "rls-preview".to_string(),
 371                     target: host.to_string(),
 372                 });
 373             }
 374             if rustfmt_present {
 375                 extensions.push(Component {
 376                     pkg: "rustfmt-preview".to_string(),
 377                     target: host.to_string(),
 378                 });
 379             }
 380             if llvm_tools_present {
 381                 extensions.push(Component {
 382                     pkg: "llvm-tools-preview".to_string(),
 383                     target: host.to_string(),
 384                 });
 385             }
 386             extensions.push(Component {
 387                 pkg: "rust-analysis".to_string(),
 388                 target: host.to_string(),
 389             });
 390             for target in TARGETS {
 391                 if target != host {
 392                     extensions.push(Component {
 393                         pkg: "rust-std".to_string(),
 394                         target: target.to_string(),
 395                     });
 396                 }
 397             }
 398             extensions.push(Component {
 399                 pkg: "rust-src".to_string(),
 400                 target: "*".to_string(),
 401             });
 402
 403             // If the components/extensions don't actually exist for this
 404             // particular host/target combination then nix it entirely from our
 405             // lists.
 406             {
 407                 let has_component = |c: &Component| {
 408                     if c.target == "*" {
 409                         return true
 410                     }
 411                     let pkg = match manifest.pkg.get(&c.pkg) {
 412                         Some(p) => p,
 413                         None => return false,
 414                     };
 415                     let target = match pkg.target.get(&c.target) {
 416                         Some(t) => t,
 417                         None => return false,
 418                     };
 419                     target.available
 420                 };
 421                 extensions.retain(&has_component);
 422                 components.retain(&has_component);
 423             }
 424
 425             pkg.target.insert(host.to_string(), Target {
 426                 available: true,
 427                 url: Some(self.url(&filename)),
 428                 hash: Some(digest),
 429                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 430                 xz_hash: xz_digest,
 431                 components: Some(components),
 432                 extensions: Some(extensions),
 433             });
 434         }
 435         manifest.pkg.insert("rust".to_string(), pkg);
 436
 437         return manifest;
 438     }
 439
 440     fn package(&mut self,
 441                pkgname: &str,
 442                dst: &mut BTreeMap<String, Package>,
 443                targets: &[&str]) {
 444         let version = match *self.cached_version(pkgname) {
 445             Some(ref version) => version.clone(),
 446             None => {
 447                 println!("Skipping package {}", pkgname);
 448                 return;
 449             }
 450         };
 451
 452         let targets = targets.iter().map(|name| {
 453             let filename = self.filename(pkgname, name);
 454             let digest = match self.digests.remove(&filename) {
 455                 Some(digest) => digest,
 456                 None => return (name.to_string(), Target::unavailable()),
 457             };
 458             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 459             let xz_digest = self.digests.remove(&xz_filename);
 460
 461             (name.to_string(), Target {
 462                 available: true,
 463                 url: Some(self.url(&filename)),
 464                 hash: Some(digest),
 465                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 466                 xz_hash: xz_digest,
 467                 components: None,
 468                 extensions: None,
 469             })
 470         }).collect();
 471
 472         dst.insert(pkgname.to_string(), Package {
 473             version,
 474             git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 475             target: targets,
 476         });
 477     }
 478
 479     fn url(&self, filename: &str) -> String {
 480         format!("{}/{}/{}",
 481                 self.s3_address,
 482                 self.date,
 483                 filename)
 484     }
 485
 486     fn filename(&self, component: &str, target: &str) -> String {
 487         if component == "rust-src" {
 488             format!("rust-src-{}.tar.gz", self.rust_release)
 489         } else if component == "cargo" {
 490             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
 491         } else if component == "rls" || component == "rls-preview" {
 492             format!("rls-{}-{}.tar.gz", self.rls_release, target)
 493         } else if component == "clippy" || component == "clippy-preview" {
 494             format!("clippy-{}-{}.tar.gz", self.clippy_release, target)
 495         } else if component == "rustfmt" || component == "rustfmt-preview" {
 496             format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
 497         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
 498             format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target)
 499         } else {
 500             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
 501         }
 502     }
 503
 504     fn cached_version(&self, component: &str) -> &Option<String> {
 505         if component == "cargo" {
 506             &self.cargo_version
 507         } else if component == "rls" || component == "rls-preview" {
 508             &self.rls_version
 509         } else if component == "clippy" || component == "clippy-preview" {
 510             &self.clippy_version
 511         } else if component == "rustfmt" || component == "rustfmt-preview" {
 512             &self.rustfmt_version
 513         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
 514             &self.llvm_tools_version
 515         } else {
 516             &self.rust_version
 517         }
 518     }
 519
 520     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 521         if component == "cargo" {
 522             &self.cargo_git_commit_hash
 523         } else if component == "rls" || component == "rls-preview" {
 524             &self.rls_git_commit_hash
 525         } else if component == "clippy" || component == "clippy-preview" {
 526             &self.clippy_git_commit_hash
 527         } else if component == "rustfmt" || component == "rustfmt-preview" {
 528             &self.rustfmt_git_commit_hash
 529         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
 530             &self.llvm_tools_git_commit_hash
 531         } else {
 532             &self.rust_git_commit_hash
 533         }
 534     }
 535
 536     fn version(&self, component: &str, target: &str) -> Option<String> {
 537         let mut cmd = Command::new("tar");
 538         let filename = self.filename(component, target);
 539         cmd.arg("xf")
 540            .arg(self.input.join(&filename))
 541            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 542            .arg("-O");
 543         let output = t!(cmd.output());
 544         if output.status.success() {
 545             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 546         } else {
 547             // Perhaps we didn't build this package.
 548             None
 549         }
 550     }
 551
 552     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 553         let mut cmd = Command::new("tar");
 554         let filename = self.filename(component, target);
 555         cmd.arg("xf")
 556            .arg(self.input.join(&filename))
 557            .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
 558            .arg("-O");
 559         let output = t!(cmd.output());
 560         if output.status.success() {
 561             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 562         } else {
 563             None
 564         }
 565     }
 566
 567     fn hash(&self, path: &Path) -> String {
 568         let sha = t!(Command::new("shasum")
 569                         .arg("-a").arg("256")
 570                         .arg(path.file_name().unwrap())
 571                         .current_dir(path.parent().unwrap())
 572                         .output());
 573         assert!(sha.status.success());
 574
 575         let filename = path.file_name().unwrap().to_str().unwrap();
 576         let sha256 = self.output.join(format!("{}.sha256", filename));
 577         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 578
 579         let stdout = String::from_utf8_lossy(&sha.stdout);
 580         stdout.split_whitespace().next().unwrap().to_string()
 581     }
 582
 583     fn sign(&self, path: &Path) {
 584         let filename = path.file_name().unwrap().to_str().unwrap();
 585         let asc = self.output.join(format!("{}.asc", filename));
 586         println!("signing: {:?}", path);
 587         let mut cmd = Command::new("gpg");
 588         cmd.arg("--no-tty")
 589             .arg("--yes")
 590             .arg("--passphrase-fd").arg("0")
 591             .arg("--personal-digest-preferences").arg("SHA512")
 592             .arg("--armor")
 593             .arg("--output").arg(&asc)
 594             .arg("--detach-sign").arg(path)
 595             .stdin(Stdio::piped());
 596         let mut child = t!(cmd.spawn());
 597         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 598         assert!(t!(child.wait()).success());
 599     }
 600
 601     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 602         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 603         self.write(&manifest.date, channel_name, "-date.txt");
 604         self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 605                    channel_name, "-git-commit-hash.txt");
 606     }
 607
 608     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 609         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 610         t!(t!(File::create(&dst)).write_all(contents.as_bytes()));
 611         self.hash(&dst);
 612         self.sign(&dst);
 613     }
 614 }