src/tools/build-manifest/src/main.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 extern crate toml;
  12 #[macro_use]
  13 extern crate serde_derive;
  14 extern crate serde;
  15
  16 use std::collections::BTreeMap;
  17 use std::env;
  18 use std::fs::File;
  19 use std::io::{self, Read, Write};
  20 use std::path::{PathBuf, Path};
  21 use std::process::{Command, Stdio};
  22
  23 static HOSTS: &'static [&'static str] = &[
  24     "aarch64-unknown-linux-gnu",
  25     "arm-unknown-linux-gnueabi",
  26     "arm-unknown-linux-gnueabihf",
  27     "armv7-unknown-linux-gnueabihf",
  28     "i686-apple-darwin",
  29     "i686-pc-windows-gnu",
  30     "i686-pc-windows-msvc",
  31     "i686-unknown-linux-gnu",
  32     "mips-unknown-linux-gnu",
  33     "mips64-unknown-linux-gnuabi64",
  34     "mips64el-unknown-linux-gnuabi64",
  35     "mipsel-unknown-linux-gnu",
  36     "powerpc-unknown-linux-gnu",
  37     "powerpc64-unknown-linux-gnu",
  38     "powerpc64le-unknown-linux-gnu",
  39     "s390x-unknown-linux-gnu",
  40     "x86_64-apple-darwin",
  41     "x86_64-pc-windows-gnu",
  42     "x86_64-pc-windows-msvc",
  43     "x86_64-unknown-freebsd",
  44     "x86_64-unknown-linux-gnu",
  45     "x86_64-unknown-netbsd",
  46 ];
  47
  48 static TARGETS: &'static [&'static str] = &[
  49     "aarch64-apple-ios",
  50     "aarch64-unknown-fuchsia",
  51     "aarch64-linux-android",
  52     "aarch64-unknown-linux-gnu",
  53     "arm-linux-androideabi",
  54     "arm-unknown-linux-gnueabi",
  55     "arm-unknown-linux-gnueabihf",
  56     "arm-unknown-linux-musleabi",
  57     "arm-unknown-linux-musleabihf",
  58     "armv7-apple-ios",
  59     "armv7-linux-androideabi",
  60     "armv7-unknown-linux-gnueabihf",
  61     "armv7-unknown-linux-musleabihf",
  62     "armv7s-apple-ios",
  63     "asmjs-unknown-emscripten",
  64     "i386-apple-ios",
  65     "i586-pc-windows-msvc",
  66     "i586-unknown-linux-gnu",
  67     "i686-apple-darwin",
  68     "i686-linux-android",
  69     "i686-pc-windows-gnu",
  70     "i686-pc-windows-msvc",
  71     "i686-unknown-freebsd",
  72     "i686-unknown-linux-gnu",
  73     "i686-unknown-linux-musl",
  74     "mips-unknown-linux-gnu",
  75     "mips-unknown-linux-musl",
  76     "mips64-unknown-linux-gnuabi64",
  77     "mips64el-unknown-linux-gnuabi64",
  78     "mipsel-unknown-linux-gnu",
  79     "mipsel-unknown-linux-musl",
  80     "powerpc-unknown-linux-gnu",
  81     "powerpc64-unknown-linux-gnu",
  82     "powerpc64le-unknown-linux-gnu",
  83     "s390x-unknown-linux-gnu",
  84     "sparc64-unknown-linux-gnu",
  85     "wasm32-unknown-emscripten",
  86     "x86_64-linux-android",
  87     "x86_64-apple-darwin",
  88     "x86_64-apple-ios",
  89     "x86_64-pc-windows-gnu",
  90     "x86_64-pc-windows-msvc",
  91     "x86_64-rumprun-netbsd",
  92     "x86_64-unknown-freebsd",
  93     "x86_64-unknown-fuchsia",
  94     "x86_64-unknown-linux-gnu",
  95     "x86_64-unknown-linux-musl",
  96     "x86_64-unknown-netbsd",
  97 ];
  98
  99 static MINGW: &'static [&'static str] = &[
 100     "i686-pc-windows-gnu",
 101     "x86_64-pc-windows-gnu",
 102 ];
 103
 104 #[derive(Serialize)]
 105 #[serde(rename_all = "kebab-case")]
 106 struct Manifest {
 107     manifest_version: String,
 108     date: String,
 109     pkg: BTreeMap<String, Package>,
 110 }
 111
 112 #[derive(Serialize)]
 113 struct Package {
 114     version: String,
 115     target: BTreeMap<String, Target>,
 116 }
 117
 118 #[derive(Serialize)]
 119 struct Target {
 120     available: bool,
 121     url: Option<String>,
 122     hash: Option<String>,
 123     xz_url: Option<String>,
 124     xz_hash: Option<String>,
 125     components: Option<Vec<Component>>,
 126     extensions: Option<Vec<Component>>,
 127 }
 128
 129 impl Target {
 130     fn unavailable() -> Target {
 131         Target {
 132             available: false,
 133             url: None,
 134             hash: None,
 135             xz_url: None,
 136             xz_hash: None,
 137             components: None,
 138             extensions: None,
 139         }
 140     }
 141 }
 142
 143 #[derive(Serialize)]
 144 struct Component {
 145     pkg: String,
 146     target: String,
 147 }
 148
 149 macro_rules! t {
 150     ($e:expr) => (match $e {
 151         Ok(e) => e,
 152         Err(e) => panic!("{} failed with {}", stringify!($e), e),
 153     })
 154 }
 155
 156 struct Builder {
 157     rust_release: String,
 158     cargo_release: String,
 159     rls_release: String,
 160     input: PathBuf,
 161     output: PathBuf,
 162     gpg_passphrase: String,
 163     digests: BTreeMap<String, String>,
 164     s3_address: String,
 165     date: String,
 166     rust_version: String,
 167     cargo_version: String,
 168     rls_version: String,
 169 }
 170
 171 fn main() {
 172     let mut args = env::args().skip(1);
 173     let input = PathBuf::from(args.next().unwrap());
 174     let output = PathBuf::from(args.next().unwrap());
 175     let date = args.next().unwrap();
 176     let rust_release = args.next().unwrap();
 177     let cargo_release = args.next().unwrap();
 178     let rls_release = args.next().unwrap();
 179     let s3_address = args.next().unwrap();
 180     let mut passphrase = String::new();
 181     t!(io::stdin().read_to_string(&mut passphrase));
 182
 183     Builder {
 184         rust_release: rust_release,
 185         cargo_release: cargo_release,
 186         rls_release: rls_release,
 187         input: input,
 188         output: output,
 189         gpg_passphrase: passphrase,
 190         digests: BTreeMap::new(),
 191         s3_address: s3_address,
 192         date: date,
 193         rust_version: String::new(),
 194         cargo_version: String::new(),
 195         rls_version: String::new(),
 196     }.build();
 197 }
 198
 199 impl Builder {
 200     fn build(&mut self) {
 201         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
 202         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
 203         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
 204
 205         self.digest_and_sign();
 206         let manifest = self.build_manifest();
 207         let filename = format!("channel-rust-{}.toml", self.rust_release);
 208         self.write_manifest(&toml::to_string(&manifest).unwrap(), &filename);
 209
 210         let filename = format!("channel-rust-{}-date.txt", self.rust_release);
 211         self.write_date_stamp(&manifest.date, &filename);
 212
 213         if self.rust_release != "beta" && self.rust_release != "nightly" {
 214             self.write_manifest(&toml::to_string(&manifest).unwrap(), "channel-rust-stable.toml");
 215             self.write_date_stamp(&manifest.date, "channel-rust-stable-date.txt");
 216         }
 217     }
 218
 219     fn digest_and_sign(&mut self) {
 220         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
 221             let filename = file.file_name().unwrap().to_str().unwrap();
 222             let digest = self.hash(&file);
 223             self.sign(&file);
 224             assert!(self.digests.insert(filename.to_string(), digest).is_none());
 225         }
 226     }
 227
 228     fn build_manifest(&mut self) -> Manifest {
 229         let mut manifest = Manifest {
 230             manifest_version: "2".to_string(),
 231             date: self.date.to_string(),
 232             pkg: BTreeMap::new(),
 233         };
 234
 235         self.package("rustc", &mut manifest.pkg, HOSTS);
 236         self.package("cargo", &mut manifest.pkg, HOSTS);
 237         self.package("rust-mingw", &mut manifest.pkg, MINGW);
 238         self.package("rust-std", &mut manifest.pkg, TARGETS);
 239         self.package("rust-docs", &mut manifest.pkg, TARGETS);
 240         self.package("rust-src", &mut manifest.pkg, &["*"]);
 241         self.package("rls", &mut manifest.pkg, HOSTS);
 242         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
 243
 244         let mut pkg = Package {
 245             version: self.cached_version("rust").to_string(),
 246             target: BTreeMap::new(),
 247         };
 248         for host in HOSTS {
 249             let filename = self.filename("rust", host);
 250             let digest = match self.digests.remove(&filename) {
 251                 Some(digest) => digest,
 252                 None => {
 253                     pkg.target.insert(host.to_string(), Target::unavailable());
 254                     continue
 255                 }
 256             };
 257             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 258             let xz_digest = self.digests.remove(&xz_filename);
 259             let mut components = Vec::new();
 260             let mut extensions = Vec::new();
 261
 262             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
 263             // if it's available for the target.
 264             components.extend(vec![
 265                 Component { pkg: "rustc".to_string(), target: host.to_string() },
 266                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
 267                 Component { pkg: "cargo".to_string(), target: host.to_string() },
 268                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
 269             ]);
 270             if host.contains("pc-windows-gnu") {
 271                 components.push(Component {
 272                     pkg: "rust-mingw".to_string(),
 273                     target: host.to_string(),
 274                 });
 275             }
 276
 277             extensions.push(Component {
 278                 pkg: "rls".to_string(),
 279                 target: host.to_string(),
 280             });
 281             extensions.push(Component {
 282                 pkg: "rust-analysis".to_string(),
 283                 target: host.to_string(),
 284             });
 285             for target in TARGETS {
 286                 if target != host {
 287                     extensions.push(Component {
 288                         pkg: "rust-std".to_string(),
 289                         target: target.to_string(),
 290                     });
 291                 }
 292             }
 293             extensions.push(Component {
 294                 pkg: "rust-src".to_string(),
 295                 target: "*".to_string(),
 296             });
 297
 298             pkg.target.insert(host.to_string(), Target {
 299                 available: true,
 300                 url: Some(self.url(&filename)),
 301                 hash: Some(digest),
 302                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 303                 xz_hash: xz_digest,
 304                 components: Some(components),
 305                 extensions: Some(extensions),
 306             });
 307         }
 308         manifest.pkg.insert("rust".to_string(), pkg);
 309
 310         return manifest
 311     }
 312
 313     fn package(&mut self,
 314                pkgname: &str,
 315                dst: &mut BTreeMap<String, Package>,
 316                targets: &[&str]) {
 317         let targets = targets.iter().map(|name| {
 318             let filename = self.filename(pkgname, name);
 319             let digest = match self.digests.remove(&filename) {
 320                 Some(digest) => digest,
 321                 None => return (name.to_string(), Target::unavailable()),
 322             };
 323             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
 324             let xz_digest = self.digests.remove(&xz_filename);
 325
 326             (name.to_string(), Target {
 327                 available: true,
 328                 url: Some(self.url(&filename)),
 329                 hash: Some(digest),
 330                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
 331                 xz_hash: xz_digest,
 332                 components: None,
 333                 extensions: None,
 334             })
 335         }).collect();
 336
 337         dst.insert(pkgname.to_string(), Package {
 338             version: self.cached_version(pkgname).to_string(),
 339             target: targets,
 340         });
 341     }
 342
 343     fn url(&self, filename: &str) -> String {
 344         format!("{}/{}/{}",
 345                 self.s3_address,
 346                 self.date,
 347                 filename)
 348     }
 349
 350     fn filename(&self, component: &str, target: &str) -> String {
 351         if component == "rust-src" {
 352             format!("rust-src-{}.tar.gz", self.rust_release)
 353         } else if component == "cargo" {
 354             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
 355         } else if component == "rls" {
 356             format!("rls-{}-{}.tar.gz", self.rls_release, target)
 357         } else {
 358             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
 359         }
 360     }
 361
 362     fn cached_version(&self, component: &str) -> &str {
 363         if component == "cargo" {
 364             &self.cargo_version
 365         } else if component == "rls" {
 366             &self.rls_version
 367         } else {
 368             &self.rust_version
 369         }
 370     }
 371
 372     fn version(&self, component: &str, target: &str) -> String {
 373         let mut cmd = Command::new("tar");
 374         let filename = self.filename(component, target);
 375         cmd.arg("xf")
 376            .arg(self.input.join(&filename))
 377            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
 378            .arg("-O");
 379         let version = t!(cmd.output());
 380         if !version.status.success() {
 381             panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}",
 382                    cmd,
 383                    String::from_utf8_lossy(&version.stdout),
 384                    String::from_utf8_lossy(&version.stderr));
 385         }
 386         String::from_utf8_lossy(&version.stdout).trim().to_string()
 387     }
 388
 389     fn hash(&self, path: &Path) -> String {
 390         let sha = t!(Command::new("shasum")
 391                         .arg("-a").arg("256")
 392                         .arg(path.file_name().unwrap())
 393                         .current_dir(path.parent().unwrap())
 394                         .output());
 395         assert!(sha.status.success());
 396
 397         let filename = path.file_name().unwrap().to_str().unwrap();
 398         let sha256 = self.output.join(format!("{}.sha256", filename));
 399         t!(t!(File::create(&sha256)).write_all(&sha.stdout));
 400
 401         let stdout = String::from_utf8_lossy(&sha.stdout);
 402         stdout.split_whitespace().next().unwrap().to_string()
 403     }
 404
 405     fn sign(&self, path: &Path) {
 406         let filename = path.file_name().unwrap().to_str().unwrap();
 407         let asc = self.output.join(format!("{}.asc", filename));
 408         println!("signing: {:?}", path);
 409         let mut cmd = Command::new("gpg");
 410         cmd.arg("--no-tty")
 411             .arg("--yes")
 412             .arg("--passphrase-fd").arg("0")
 413             .arg("--armor")
 414             .arg("--output").arg(&asc)
 415             .arg("--detach-sign").arg(path)
 416             .stdin(Stdio::piped());
 417         let mut child = t!(cmd.spawn());
 418         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
 419         assert!(t!(child.wait()).success());
 420     }
 421
 422     fn write_manifest(&self, manifest: &str, name: &str) {
 423         let dst = self.output.join(name);
 424         t!(t!(File::create(&dst)).write_all(manifest.as_bytes()));
 425         self.hash(&dst);
 426         self.sign(&dst);
 427     }
 428
 429     fn write_date_stamp(&self, date: &str, name: &str) {
 430         let dst = self.output.join(name);
 431         t!(t!(File::create(&dst)).write_all(date.as_bytes()));
 432         self.hash(&dst);
 433         self.sign(&dst);
 434     }
 435 }