src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 extern crate rustc_serialize;
  13
  14 use std::collections::BTreeMap;
  15 use std::env;
  16 use std::fs::File;
  17 use std::io::{self, Read, Write};
  18 use std::path::{PathBuf, Path};
  19 use std::process::{Command, Stdio};
  20
  21 static HOSTS: &'static [&'static str] = &[
  22     "aarch64-unknown-linux-gnu",
  23     "arm-unknown-linux-gnueabi",
  24     "arm-unknown-linux-gnueabihf",
  25     "armv7-unknown-linux-gnueabihf",
  26     "i686-apple-darwin",
  27     "i686-pc-windows-gnu",
  28     "i686-pc-windows-msvc",
  29     "i686-unknown-linux-gnu",
  30     "mips-unknown-linux-gnu",
  31     "mips64-unknown-linux-gnuabi64",
  32     "mips64el-unknown-linux-gnuabi64",
  33     "mipsel-unknown-linux-gnu",
  34     "powerpc-unknown-linux-gnu",
  35     "powerpc64-unknown-linux-gnu",
  36     "powerpc64le-unknown-linux-gnu",
  37     "s390x-unknown-linux-gnu",
  38     "x86_64-apple-darwin",
  39     "x86_64-pc-windows-gnu",
  40     "x86_64-pc-windows-msvc",
  41     "x86_64-unknown-freebsd",
  42     "x86_64-unknown-linux-gnu",
  43     "x86_64-unknown-netbsd",
  44 ];
  45
  46 static TARGETS: &'static [&'static str] = &[
  47     "aarch64-apple-ios",
  48     "aarch64-unknown-fuchsia",
  49     "aarch64-linux-android",
  50     "aarch64-unknown-linux-gnu",
  51     "arm-linux-androideabi",
  52     "arm-unknown-linux-gnueabi",
  53     "arm-unknown-linux-gnueabihf",
  54     "arm-unknown-linux-musleabi",
  55     "arm-unknown-linux-musleabihf",
  56     "armv7-apple-ios",
  57     "armv7-linux-androideabi",
  58     "armv7-unknown-linux-gnueabihf",
  59     "armv7-unknown-linux-musleabihf",
  60     "armv7s-apple-ios",
  61     "asmjs-unknown-emscripten",
  62     "i386-apple-ios",
  63     "i586-pc-windows-msvc",
  64     "i586-unknown-linux-gnu",
  65     "i686-apple-darwin",
  66     "i686-linux-android",
  67     "i686-pc-windows-gnu",
  68     "i686-pc-windows-msvc",
  69     "i686-unknown-freebsd",
  70     "i686-unknown-linux-gnu",
  71     "i686-unknown-linux-musl",
  72     "mips-unknown-linux-gnu",
  73     "mips-unknown-linux-musl",
  74     "mips64-unknown-linux-gnuabi64",
  75     "mips64el-unknown-linux-gnuabi64",
  76     "mipsel-unknown-linux-gnu",
  77     "mipsel-unknown-linux-musl",
  78     "powerpc-unknown-linux-gnu",
  79     "powerpc64-unknown-linux-gnu",
  80     "powerpc64le-unknown-linux-gnu",
  81     "s390x-unknown-linux-gnu",
  82     "sparc64-unknown-linux-gnu",
  83     "wasm32-unknown-emscripten",
  84     "x86_64-linux-android",
  85     "x86_64-apple-darwin",
  86     "x86_64-apple-ios",
  87     "x86_64-pc-windows-gnu",
  88     "x86_64-pc-windows-msvc",
  89     "x86_64-rumprun-netbsd",
  90     "x86_64-unknown-freebsd",
  91     "x86_64-unknown-fuchsia",
  92     "x86_64-unknown-linux-gnu",
  93     "x86_64-unknown-linux-musl",
  94     "x86_64-unknown-netbsd",
  95 ];
  96
  97 static MINGW: &'static [&'static str] = &[
  98     "i686-pc-windows-gnu",
  99     "x86_64-pc-windows-gnu",
 100 ];
 101
 102 struct Manifest {
 103     manifest_version: String,
 104     date: String,
 105     pkg: BTreeMap<String, Package>,
 106 }
 107
 108 #[derive(RustcEncodable)]
 109 struct Package {
 110     version: String,
 111     target: BTreeMap<String, Target>,
 112 }
 113
 114 #[derive(RustcEncodable)]
 115 struct Target {
 116     available: bool,
 117     url: Option<String>,
 118     hash: Option<String>,
 119     xz_url: Option<String>,
 120     xz_hash: Option<String>,
 121     components: Option<Vec<Component>>,
 122     extensions: Option<Vec<Component>>,
 123 }
 124
 125 impl Target {
 126     fn unavailable() -> Target {
 127         Target {
 128             available: false,
 129             url: None,
 130             hash: None,
 131             xz_url: None,
 132             xz_hash: None,
 133             components: None,
 134             extensions: None,
 135         }
 136     }
 137 }
 138
 139 #[derive(RustcEncodable)]
 140 struct Component {
 141     pkg: String,
 142     target: String,
 143 }
 144
 145 macro_rules! t {
 146     ($e:expr) => (match $e {
 147         Ok(e) => e,
 148         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 149     })
 150 }
 151
 152 struct Builder {
 153     rust_release: String,
 154     cargo_release: String,
 155     rls_release: String,
 156     input: PathBuf,
 157     output: PathBuf,
 158     gpg_passphrase: String,
 159     digests: BTreeMap<String, String>,
 160     s3_address: String,
 161     date: String,
 162     rust_version: String,
 163     cargo_version: String,
 164     rls_version: String,
 165 }
 166
 167 fn main() {
 168     let mut args = env::args().skip(1);
 169     let input = PathBuf::from(args.next().unwrap());
 170     let output = PathBuf::from(args.next().unwrap());
 171     let date = args.next().unwrap();
 172     let rust_release = args.next().unwrap();
 173     let cargo_release = args.next().unwrap();
 174     let rls_release = args.next().unwrap();
 175     let s3_address = args.next().unwrap();
 176     let mut passphrase = String::new();
 177     t!(io::stdin().read_to_string(&mut passphrase));
 178
 179     Builder {
 180         rust_release: rust_release,
 181         cargo_release: cargo_release,
 182         rls_release: rls_release,
 183         input: input,
 184         output: output,
 185         gpg_passphrase: passphrase,
 186         digests: BTreeMap::new(),
 187         s3_address: s3_address,
 188         date: date,
 189         rust_version: String::new(),
 190         cargo_version: String::new(),
 191         rls_version: String::new(),
 192     }.build();
 193 }
 194
 195 impl Builder {
 196     fn build(&mut self) {
 197         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 198         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 199         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 200
 201         self.digest_and_sign();
 202         let Manifest { manifest_version, date, pkg } = self.build_manifest();
 203
 204         // Unfortunately we can't use derive(RustcEncodable) here because the
 205         // version field is called `manifest-version`, not `manifest_version`.
 206         // In lieu of that just create the table directly here with a `BTreeMap`
 207         // and wrap it up in a `Value::Table`.
 208         let mut manifest = BTreeMap::new();
 209         manifest.insert("manifest-version".to_string(),
 210                         toml::Value::String(manifest_version));
 211         manifest.insert("date".to_string(), toml::Value::String(date.clone()));
 212         manifest.insert("pkg".to_string(), toml::encode(&pkg));
 213         let manifest = toml::Value::Table(manifest).to_string();
 214
 215         let filename = format!("channel-rust-{}.toml", self.rust_release);
 216         self.write_manifest(&manifest, &filename);
 217
 218         let filename = format!("channel-rust-{}-date.txt", self.rust_release);
 219         self.write_date_stamp(&date, &filename);
 220
 221         if self.rust_release != "beta" && self.rust_release != "nightly" {
 222             self.write_manifest(&manifest, "channel-rust-stable.toml");
 223             self.write_date_stamp(&date, "channel-rust-stable-date.txt");
 224         }
 225     }
 226
 227     fn digest_and_sign(&mut self) {
 228         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 229             let filename = file.file_name().unwrap().to_str().unwrap();
 230             let digest = self.hash(&file);
 231             self.sign(&file);
 232             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 233         }
 234     }
 235
 236     fn build_manifest(&mut self) -> Manifest {
 237         let mut manifest = Manifest {
 238             manifest_version: "2".to_string(),
 239             date: self.date.to_string(),
 240             pkg: BTreeMap::new(),
 241         };
 242
 243         self.package("rustc", &mut manifest.pkg, HOSTS);
 244         self.package("cargo", &mut manifest.pkg, HOSTS);
 245         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 246         self.package("rust-std", &mut manifest.pkg, TARGETS);
 247         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 248         self.package("rust-src", &mut manifest.pkg, &["*"]);
 249         self.package("rls", &mut manifest.pkg, HOSTS);
 250         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
 251
 252         let mut pkg = Package {
 253             version: self.cached_version("rust").to_string(),
 254             target: BTreeMap::new(),
 255         };
 256         for host in HOSTS {
 257             let filename = self.filename("rust", host);
 258             let digest = match self.digests.remove(&filename) {
 259                 Some(digest) => digest,
 260                 None => {
 261                     pkg.target.insert(host.to_string(), Target::unavailable());
 262                     continue
 263                 }
 264             };
 265             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 266             let xz_digest = self.digests.remove(&xz_filename);
 267             let mut components = Vec::new();
 268             let mut extensions = Vec::new();
 269
 270             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
 271             // if it's available for the target.
 272             components.extend(vec![
 273                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 274                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 275                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 276                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
 277             ]);
 278             if host.contains("pc-windows-gnu") {
 279                 components.push(Component {
 280                     pkg: "rust-mingw".to_string(),
 281                     target: host.to_string(),
 282                 });
 283             }
 284
 285             extensions.push(Component {
 286                 pkg: "rls".to_string(),
 287                 target: host.to_string(),
 288             });
 289             extensions.push(Component {
 290                 pkg: "rust-analysis".to_string(),
 291                 target: host.to_string(),
 292             });
 293             for target in TARGETS {
 294                 if target != host {
 295                     extensions.push(Component {
 296                         pkg: "rust-std".to_string(),
 297                         target: target.to_string(),
 298                     });
 299                 }
 300             }
 301             extensions.push(Component {
 302                 pkg: "rust-src".to_string(),
 303                 target: "*".to_string(),
 304             });
 305
 306             pkg.target.insert(host.to_string(), Target {
 307                 available: true,
 308                 url: Some(self.url(&filename)),
 309                 hash: Some(digest),
 310                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 311                 xz_hash: xz_digest,
 312                 components: Some(components),
 313                 extensions: Some(extensions),
 314             });
 315         }
 316         manifest.pkg.insert("rust".to_string(), pkg);
 317
 318         return manifest
 319     }
 320
 321     fn package(&mut self,
 322                pkgname: &str,
 323                dst: &mut BTreeMap<String, Package>,
 324                targets: &[&str]) {
 325         let targets = targets.iter().map(|name| {
 326             let filename = self.filename(pkgname, name);
 327             let digest = match self.digests.remove(&filename) {
 328                 Some(digest) => digest,
 329                 None => return (name.to_string(), Target::unavailable()),
 330             };
 331             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 332             let xz_digest = self.digests.remove(&xz_filename);
 333
 334             (name.to_string(), Target {
 335                 available: true,
 336                 url: Some(self.url(&filename)),
 337                 hash: Some(digest),
 338                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 339                 xz_hash: xz_digest,
 340                 components: None,
 341                 extensions: None,
 342             })
 343         }).collect();
 344
 345         dst.insert(pkgname.to_string(), Package {
 346             version: self.cached_version(pkgname).to_string(),
 347             target: targets,
 348         });
 349     }
 350
 351     fn url(&self, filename: &str) -> String {
 352         format!("{}/{}/{}",
 353                 self.s3_address,
 354                 self.date,
 355                 filename)
 356     }
 357
 358     fn filename(&self, component: &str, target: &str) -> String {
 359         if component == "rust-src" {
 360             format!("rust-src-{}.tar.gz", self.rust_release)
 361         } else if component == "cargo" {
 362             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
 363         } else if component == "rls" {
 364             format!("rls-{}-{}.tar.gz", self.rls_release, target)
 365         } else {
 366             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
 367         }
 368     }
 369
 370     fn cached_version(&self, component: &str) -> &str {
 371         if component == "cargo" {
 372             &self.cargo_version
 373         } else if component == "rls" {
 374             &self.rls_version
 375         } else {
 376             &self.rust_version
 377         }
 378     }
 379
 380     fn version(&self, component: &str, target: &str) -> String {
 381         let mut cmd = Command::new("tar");
 382         let filename = self.filename(component, target);
 383         cmd.arg("xf")
 384            .arg(self.input.join(&filename))
 385            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 386            .arg("-O");
 387         let version = t!(cmd.output());
 388         if !version.status.success() {
 389             panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
 390                    cmd,
 391                    String::from_utf8_lossy(&version.stdout),
 392                    String::from_utf8_lossy(&version.stderr));
 393         }
 394         String::from_utf8_lossy(&version.stdout).trim().to_string()
 395     }
 396
 397     fn hash(&self, path: &Path) -> String {
 398         let sha = t!(Command::new("shasum")
 399                         .arg("-a").arg("256")
 400                         .arg(path.file_name().unwrap())
 401                         .current_dir(path.parent().unwrap())
 402                         .output());
 403         assert!(sha.status.success());
 404
 405         let filename = path.file_name().unwrap().to_str().unwrap();
 406         let sha256 = self.output.join(format!("{}.sha256", filename));
 407         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 408
 409         let stdout = String::from_utf8_lossy(&sha.stdout);
 410         stdout.split_whitespace().next().unwrap().to_string()
 411     }
 412
 413     fn sign(&self, path: &Path) {
 414         let filename = path.file_name().unwrap().to_str().unwrap();
 415         let asc = self.output.join(format!("{}.asc", filename));
 416         println!("signing: {:?}", path);
 417         let mut cmd = Command::new("gpg");
 418         cmd.arg("--no-tty")
 419             .arg("--yes")
 420             .arg("--passphrase-fd").arg("0")
 421             .arg("--armor")
 422             .arg("--output").arg(&asc)
 423             .arg("--detach-sign").arg(path)
 424             .stdin(Stdio::piped());
 425         let mut child = t!(cmd.spawn());
 426         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 427         assert!(t!(child.wait()).success());
 428     }
 429
 430     fn write_manifest(&self, manifest: &str, name: &str) {
 431         let dst = self.output.join(name);
 432         t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
 433         self.hash(&dst);
 434         self.sign(&dst);
 435     }
 436
 437     fn write_date_stamp(&self, date: &str, name: &str) {
 438         let dst = self.output.join(name);
 439         t!(t!(File::create(&dst)).write_all(date.as_bytes()));
 440         self.hash(&dst);
 441         self.sign(&dst);
 442     }
 443 }