src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 #[macro_use]
  13 extern crate serde_derive;
  14
  15 use std::collections::BTreeMap;
  16 use std::env;
  17 use std::fs::File;
  18 use std::io::{self, Read, Write};
  19 use std::path::{PathBuf, Path};
  20 use std::process::{Command, Stdio};
  21
  22 static HOSTS: &'static [&'static str] = &[
  23     "aarch64-unknown-linux-gnu",
  24     "arm-unknown-linux-gnueabi",
  25     "arm-unknown-linux-gnueabihf",
  26     "armv7-unknown-linux-gnueabihf",
  27     "i686-apple-darwin",
  28     "i686-pc-windows-gnu",
  29     "i686-pc-windows-msvc",
  30     "i686-unknown-linux-gnu",
  31     "mips-unknown-linux-gnu",
  32     "mips64-unknown-linux-gnuabi64",
  33     "mips64el-unknown-linux-gnuabi64",
  34     "mipsel-unknown-linux-gnu",
  35     "powerpc-unknown-linux-gnu",
  36     "powerpc64-unknown-linux-gnu",
  37     "powerpc64le-unknown-linux-gnu",
  38     "s390x-unknown-linux-gnu",
  39     "x86_64-apple-darwin",
  40     "x86_64-pc-windows-gnu",
  41     "x86_64-pc-windows-msvc",
  42     "x86_64-unknown-freebsd",
  43     "x86_64-unknown-linux-gnu",
  44     "x86_64-unknown-netbsd",
  45 ];
  46
  47 static TARGETS: &'static [&'static str] = &[
  48     "aarch64-apple-ios",
  49     "aarch64-linux-android",
  50     "aarch64-unknown-cloudabi",
  51     "aarch64-unknown-fuchsia",
  52     "aarch64-unknown-linux-gnu",
  53     "aarch64-unknown-linux-musl",
  54     "arm-linux-androideabi",
  55     "arm-unknown-linux-gnueabi",
  56     "arm-unknown-linux-gnueabihf",
  57     "arm-unknown-linux-musleabi",
  58     "arm-unknown-linux-musleabihf",
  59     "armv5te-unknown-linux-gnueabi",
  60     "armv5te-unknown-linux-musleabi",
  61     "armv7-apple-ios",
  62     "armv7-linux-androideabi",
  63     "armv7-unknown-cloudabi-eabihf",
  64     "armv7-unknown-linux-gnueabihf",
  65     "armv7-unknown-linux-musleabihf",
  66     "armebv7r-none-eabihf",
  67     "armv7s-apple-ios",
  68     "asmjs-unknown-emscripten",
  69     "i386-apple-ios",
  70     "i586-pc-windows-msvc",
  71     "i586-unknown-linux-gnu",
  72     "i586-unknown-linux-musl",
  73     "i686-apple-darwin",
  74     "i686-linux-android",
  75     "i686-pc-windows-gnu",
  76     "i686-pc-windows-msvc",
  77     "i686-unknown-cloudabi",
  78     "i686-unknown-freebsd",
  79     "i686-unknown-linux-gnu",
  80     "i686-unknown-linux-musl",
  81     "mips-unknown-linux-gnu",
  82     "mips-unknown-linux-musl",
  83     "mips64-unknown-linux-gnuabi64",
  84     "mips64el-unknown-linux-gnuabi64",
  85     "mipsel-unknown-linux-gnu",
  86     "mipsel-unknown-linux-musl",
  87     "powerpc-unknown-linux-gnu",
  88     "powerpc-unknown-linux-gnuspe",
  89     "powerpc64-unknown-linux-gnu",
  90     "powerpc64le-unknown-linux-gnu",
  91     "s390x-unknown-linux-gnu",
  92     "sparc-unknown-linux-gnu",
  93     "sparc64-unknown-linux-gnu",
  94     "sparcv9-sun-solaris",
  95     "thumbv6m-none-eabi",
  96     "thumbv7em-none-eabi",
  97     "thumbv7em-none-eabihf",
  98     "thumbv7m-none-eabi",
  99     "wasm32-unknown-emscripten",
 100     "wasm32-unknown-unknown",
 101     "x86_64-apple-darwin",
 102     "x86_64-apple-ios",
 103     "x86_64-linux-android",
 104     "x86_64-pc-windows-gnu",
 105     "x86_64-pc-windows-msvc",
 106     "x86_64-rumprun-netbsd",
 107     "x86_64-sun-solaris",
 108     "x86_64-unknown-cloudabi",
 109     "x86_64-unknown-freebsd",
 110     "x86_64-unknown-fuchsia",
 111     "x86_64-unknown-linux-gnu",
 112     "x86_64-unknown-linux-gnux32",
 113     "x86_64-unknown-linux-musl",
 114     "x86_64-unknown-netbsd",
 115     "x86_64-unknown-redox",
 116 ];
 117
 118 static MINGW: &'static [&'static str] = &[
 119     "i686-pc-windows-gnu",
 120     "x86_64-pc-windows-gnu",
 121 ];
 122
 123 #[derive(Serialize)]
 124 #[serde(rename_all = "kebab-case")]
 125 struct Manifest {
 126     manifest_version: String,
 127     date: String,
 128     pkg: BTreeMap<String, Package>,
 129     renames: BTreeMap<String, Rename>
 130 }
 131
 132 #[derive(Serialize)]
 133 struct Package {
 134     version: String,
 135     git_commit_hash: Option<String>,
 136     target: BTreeMap<String, Target>,
 137 }
 138
 139 #[derive(Serialize)]
 140 struct Rename {
 141     to: String,
 142 }
 143
 144 #[derive(Serialize)]
 145 struct Target {
 146     available: bool,
 147     url: Option<String>,
 148     hash: Option<String>,
 149     xz_url: Option<String>,
 150     xz_hash: Option<String>,
 151     components: Option<Vec<Component>>,
 152     extensions: Option<Vec<Component>>,
 153 }
 154
 155 impl Target {
 156     fn unavailable() -> Target {
 157         Target {
 158             available: false,
 159             url: None,
 160             hash: None,
 161             xz_url: None,
 162             xz_hash: None,
 163             components: None,
 164             extensions: None,
 165         }
 166     }
 167 }
 168
 169 #[derive(Serialize)]
 170 struct Component {
 171     pkg: String,
 172     target: String,
 173 }
 174
 175 macro_rules! t {
 176     ($e:expr) => (match $e {
 177         Ok(e) => e,
 178         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 179     })
 180 }
 181
 182 struct Builder {
 183     rust_release: String,
 184     cargo_release: String,
 185     rls_release: String,
 186     rustfmt_release: String,
 187
 188     input: PathBuf,
 189     output: PathBuf,
 190     gpg_passphrase: String,
 191     digests: BTreeMap<String, String>,
 192     s3_address: String,
 193     date: String,
 194
 195     rust_version: Option<String>,
 196     cargo_version: Option<String>,
 197     rls_version: Option<String>,
 198     rustfmt_version: Option<String>,
 199
 200     rust_git_commit_hash: Option<String>,
 201     cargo_git_commit_hash: Option<String>,
 202     rls_git_commit_hash: Option<String>,
 203     rustfmt_git_commit_hash: Option<String>,
 204 }
 205
 206 fn main() {
 207     let mut args = env::args().skip(1);
 208     let input = PathBuf::from(args.next().unwrap());
 209     let output = PathBuf::from(args.next().unwrap());
 210     let date = args.next().unwrap();
 211     let rust_release = args.next().unwrap();
 212     let cargo_release = args.next().unwrap();
 213     let rls_release = args.next().unwrap();
 214     let rustfmt_release = args.next().unwrap();
 215     let _llvm_tools_vers = args.next().unwrap(); // FIXME do something with it?
 216     let s3_address = args.next().unwrap();
 217     let mut passphrase = String::new();
 218     t!(io::stdin().read_to_string(&mut passphrase));
 219
 220     Builder {
 221         rust_release,
 222         cargo_release,
 223         rls_release,
 224         rustfmt_release,
 225
 226         input,
 227         output,
 228         gpg_passphrase: passphrase,
 229         digests: BTreeMap::new(),
 230         s3_address,
 231         date,
 232
 233         rust_version: None,
 234         cargo_version: None,
 235         rls_version: None,
 236         rustfmt_version: None,
 237
 238         rust_git_commit_hash: None,
 239         cargo_git_commit_hash: None,
 240         rls_git_commit_hash: None,
 241         rustfmt_git_commit_hash: None,
 242     }.build();
 243 }
 244
 245 impl Builder {
 246     fn build(&mut self) {
 247         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 248         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 249         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 250         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
 251
 252         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
 253         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
 254         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
 255         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
 256
 257         self.digest_and_sign();
 258         let manifest = self.build_manifest();
 259         self.write_channel_files(&self.rust_release, &manifest);
 260
 261         if self.rust_release != "beta" && self.rust_release != "nightly" {
 262             self.write_channel_files("stable", &manifest);
 263         }
 264     }
 265
 266     fn digest_and_sign(&mut self) {
 267         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 268             let filename = file.file_name().unwrap().to_str().unwrap();
 269             let digest = self.hash(&file);
 270             self.sign(&file);
 271             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 272         }
 273     }
 274
 275     fn build_manifest(&mut self) -> Manifest {
 276         let mut manifest = Manifest {
 277             manifest_version: "2".to_string(),
 278             date: self.date.to_string(),
 279             pkg: BTreeMap::new(),
 280             renames: BTreeMap::new(),
 281         };
 282
 283         self.package("rustc", &mut manifest.pkg, HOSTS);
 284         self.package("cargo", &mut manifest.pkg, HOSTS);
 285         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 286         self.package("rust-std", &mut manifest.pkg, TARGETS);
 287         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 288         self.package("rust-src", &mut manifest.pkg, &["*"]);
 289         self.package("rls-preview", &mut manifest.pkg, HOSTS);
 290         self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
 291         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
 292
 293         let rls_present = manifest.pkg.contains_key("rls-preview");
 294         let rustfmt_present = manifest.pkg.contains_key("rustfmt-preview");
 295
 296         if rls_present {
 297             manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
 298         }
 299
 300         let mut pkg = Package {
 301             version: self.cached_version("rust")
 302                          .as_ref()
 303                          .expect("Couldn't find Rust version")
 304                          .clone(),
 305             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
 306             target: BTreeMap::new(),
 307         };
 308         for host in HOSTS {
 309             let filename = self.filename("rust", host);
 310             let digest = match self.digests.remove(&filename) {
 311                 Some(digest) => digest,
 312                 None => {
 313                     pkg.target.insert(host.to_string(), Target::unavailable());
 314                     continue
 315                 }
 316             };
 317             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 318             let xz_digest = self.digests.remove(&xz_filename);
 319             let mut components = Vec::new();
 320             let mut extensions = Vec::new();
 321
 322             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
 323             // if it's available for the target.
 324             components.extend(vec![
 325                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 326                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 327                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 328                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
 329             ]);
 330             if host.contains("pc-windows-gnu") {
 331                 components.push(Component {
 332                     pkg: "rust-mingw".to_string(),
 333                     target: host.to_string(),
 334                 });
 335             }
 336
 337             if rls_present {
 338                 extensions.push(Component {
 339                     pkg: "rls-preview".to_string(),
 340                     target: host.to_string(),
 341                 });
 342             }
 343             if rustfmt_present {
 344                 extensions.push(Component {
 345                     pkg: "rustfmt-preview".to_string(),
 346                     target: host.to_string(),
 347                 });
 348             }
 349             extensions.push(Component {
 350                 pkg: "rust-analysis".to_string(),
 351                 target: host.to_string(),
 352             });
 353             for target in TARGETS {
 354                 if target != host {
 355                     extensions.push(Component {
 356                         pkg: "rust-std".to_string(),
 357                         target: target.to_string(),
 358                     });
 359                 }
 360             }
 361             extensions.push(Component {
 362                 pkg: "rust-src".to_string(),
 363                 target: "*".to_string(),
 364             });
 365
 366             // If the components/extensions don't actually exist for this
 367             // particular host/target combination then nix it entirely from our
 368             // lists.
 369             {
 370                 let has_component = |c: &Component| {
 371                     if c.target == "*" {
 372                         return true
 373                     }
 374                     let pkg = match manifest.pkg.get(&c.pkg) {
 375                         Some(p) => p,
 376                         None => return false,
 377                     };
 378                     let target = match pkg.target.get(&c.target) {
 379                         Some(t) => t,
 380                         None => return false,
 381                     };
 382                     target.available
 383                 };
 384                 extensions.retain(&has_component);
 385                 components.retain(&has_component);
 386             }
 387
 388             pkg.target.insert(host.to_string(), Target {
 389                 available: true,
 390                 url: Some(self.url(&filename)),
 391                 hash: Some(digest),
 392                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 393                 xz_hash: xz_digest,
 394                 components: Some(components),
 395                 extensions: Some(extensions),
 396             });
 397         }
 398         manifest.pkg.insert("rust".to_string(), pkg);
 399
 400         return manifest;
 401     }
 402
 403     fn package(&mut self,
 404                pkgname: &str,
 405                dst: &mut BTreeMap<String, Package>,
 406                targets: &[&str]) {
 407         let version = match *self.cached_version(pkgname) {
 408             Some(ref version) => version.clone(),
 409             None => {
 410                 println!("Skipping package {}", pkgname);
 411                 return;
 412             }
 413         };
 414
 415         let targets = targets.iter().map(|name| {
 416             let filename = self.filename(pkgname, name);
 417             let digest = match self.digests.remove(&filename) {
 418                 Some(digest) => digest,
 419                 None => return (name.to_string(), Target::unavailable()),
 420             };
 421             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 422             let xz_digest = self.digests.remove(&xz_filename);
 423
 424             (name.to_string(), Target {
 425                 available: true,
 426                 url: Some(self.url(&filename)),
 427                 hash: Some(digest),
 428                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 429                 xz_hash: xz_digest,
 430                 components: None,
 431                 extensions: None,
 432             })
 433         }).collect();
 434
 435         dst.insert(pkgname.to_string(), Package {
 436             version,
 437             git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
 438             target: targets,
 439         });
 440     }
 441
 442     fn url(&self, filename: &str) -> String {
 443         format!("{}/{}/{}",
 444                 self.s3_address,
 445                 self.date,
 446                 filename)
 447     }
 448
 449     fn filename(&self, component: &str, target: &str) -> String {
 450         if component == "rust-src" {
 451             format!("rust-src-{}.tar.gz", self.rust_release)
 452         } else if component == "cargo" {
 453             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
 454         } else if component == "rls" || component == "rls-preview" {
 455             format!("rls-{}-{}.tar.gz", self.rls_release, target)
 456         } else if component == "rustfmt" || component == "rustfmt-preview" {
 457             format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
 458         } else {
 459             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
 460         }
 461     }
 462
 463     fn cached_version(&self, component: &str) -> &Option<String> {
 464         if component == "cargo" {
 465             &self.cargo_version
 466         } else if component == "rls" || component == "rls-preview" {
 467             &self.rls_version
 468         } else if component == "rustfmt" || component == "rustfmt-preview" {
 469             &self.rustfmt_version
 470         } else {
 471             &self.rust_version
 472         }
 473     }
 474
 475     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
 476         if component == "cargo" {
 477             &self.cargo_git_commit_hash
 478         } else if component == "rls" || component == "rls-preview" {
 479             &self.rls_git_commit_hash
 480         } else if component == "rustfmt" || component == "rustfmt-preview" {
 481             &self.rustfmt_git_commit_hash
 482         } else {
 483             &self.rust_git_commit_hash
 484         }
 485     }
 486
 487     fn version(&self, component: &str, target: &str) -> Option<String> {
 488         let mut cmd = Command::new("tar");
 489         let filename = self.filename(component, target);
 490         cmd.arg("xf")
 491            .arg(self.input.join(&filename))
 492            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 493            .arg("-O");
 494         let output = t!(cmd.output());
 495         if output.status.success() {
 496             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 497         } else {
 498             // Perhaps we didn't build this package.
 499             None
 500         }
 501     }
 502
 503     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
 504         let mut cmd = Command::new("tar");
 505         let filename = self.filename(component, target);
 506         cmd.arg("xf")
 507            .arg(self.input.join(&filename))
 508            .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
 509            .arg("-O");
 510         let output = t!(cmd.output());
 511         if output.status.success() {
 512             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
 513         } else {
 514             None
 515         }
 516     }
 517
 518     fn hash(&self, path: &Path) -> String {
 519         let sha = t!(Command::new("shasum")
 520                         .arg("-a").arg("256")
 521                         .arg(path.file_name().unwrap())
 522                         .current_dir(path.parent().unwrap())
 523                         .output());
 524         assert!(sha.status.success());
 525
 526         let filename = path.file_name().unwrap().to_str().unwrap();
 527         let sha256 = self.output.join(format!("{}.sha256", filename));
 528         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 529
 530         let stdout = String::from_utf8_lossy(&sha.stdout);
 531         stdout.split_whitespace().next().unwrap().to_string()
 532     }
 533
 534     fn sign(&self, path: &Path) {
 535         let filename = path.file_name().unwrap().to_str().unwrap();
 536         let asc = self.output.join(format!("{}.asc", filename));
 537         println!("signing: {:?}", path);
 538         let mut cmd = Command::new("gpg");
 539         cmd.arg("--no-tty")
 540             .arg("--yes")
 541             .arg("--passphrase-fd").arg("0")
 542             .arg("--personal-digest-preferences").arg("SHA512")
 543             .arg("--armor")
 544             .arg("--output").arg(&asc)
 545             .arg("--detach-sign").arg(path)
 546             .stdin(Stdio::piped());
 547         let mut child = t!(cmd.spawn());
 548         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 549         assert!(t!(child.wait()).success());
 550     }
 551
 552     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
 553         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
 554         self.write(&manifest.date, channel_name, "-date.txt");
 555         self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
 556                    channel_name, "-git-commit-hash.txt");
 557     }
 558
 559     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
 560         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
 561         t!(t!(File::create(&dst)).write_all(contents.as_bytes()));
 562         self.hash(&dst);
 563         self.sign(&dst);
 564     }
 565 }