1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
13 extern crate serde_derive;
15 use std::collections::BTreeMap;
18 use std::io::{self, Read, Write};
19 use std::path::{PathBuf, Path};
20 use std::process::{Command, Stdio};
22 static HOSTS: &'static [&'static str] = &[
23 "aarch64-unknown-linux-gnu",
24 "arm-unknown-linux-gnueabi",
25 "arm-unknown-linux-gnueabihf",
26 "armv7-unknown-linux-gnueabihf",
28 "i686-pc-windows-gnu",
29 "i686-pc-windows-msvc",
30 "i686-unknown-linux-gnu",
31 "mips-unknown-linux-gnu",
32 "mips64-unknown-linux-gnuabi64",
33 "mips64el-unknown-linux-gnuabi64",
34 "mipsel-unknown-linux-gnu",
35 "powerpc-unknown-linux-gnu",
36 "powerpc64-unknown-linux-gnu",
37 "powerpc64le-unknown-linux-gnu",
38 "s390x-unknown-linux-gnu",
39 "x86_64-apple-darwin",
40 "x86_64-pc-windows-gnu",
41 "x86_64-pc-windows-msvc",
42 "x86_64-unknown-freebsd",
43 "x86_64-unknown-linux-gnu",
44 "x86_64-unknown-netbsd",
47 static TARGETS: &'static [&'static str] = &[
50 "aarch64-linux-android",
51 "aarch64-pc-windows-msvc",
52 "aarch64-unknown-cloudabi",
53 "aarch64-unknown-linux-gnu",
54 "aarch64-unknown-linux-musl",
55 "arm-linux-androideabi",
56 "arm-unknown-linux-gnueabi",
57 "arm-unknown-linux-gnueabihf",
58 "arm-unknown-linux-musleabi",
59 "arm-unknown-linux-musleabihf",
60 "armv5te-unknown-linux-gnueabi",
61 "armv5te-unknown-linux-musleabi",
63 "armv7-linux-androideabi",
64 "armv7-unknown-linux-gnueabihf",
65 "armv7-unknown-linux-musleabihf",
67 "armebv7r-none-eabihf",
71 "asmjs-unknown-emscripten",
73 "i586-pc-windows-msvc",
74 "i586-unknown-linux-gnu",
75 "i586-unknown-linux-musl",
78 "i686-pc-windows-gnu",
79 "i686-pc-windows-msvc",
80 "i686-unknown-freebsd",
81 "i686-unknown-linux-gnu",
82 "i686-unknown-linux-musl",
83 "mips-unknown-linux-gnu",
84 "mips-unknown-linux-musl",
85 "mips64-unknown-linux-gnuabi64",
86 "mips64el-unknown-linux-gnuabi64",
87 "mipsel-unknown-linux-gnu",
88 "mipsel-unknown-linux-musl",
89 "powerpc-unknown-linux-gnu",
90 "powerpc64-unknown-linux-gnu",
91 "powerpc64le-unknown-linux-gnu",
92 "riscv32imc-unknown-none-elf",
93 "riscv32imac-unknown-none-elf",
94 "s390x-unknown-linux-gnu",
95 "sparc64-unknown-linux-gnu",
96 "sparcv9-sun-solaris",
98 "thumbv7em-none-eabi",
99 "thumbv7em-none-eabihf",
100 "thumbv7m-none-eabi",
101 "wasm32-unknown-emscripten",
102 "wasm32-unknown-unknown",
103 "x86_64-apple-darwin",
106 "x86_64-linux-android",
107 "x86_64-pc-windows-gnu",
108 "x86_64-pc-windows-msvc",
109 "x86_64-rumprun-netbsd",
110 "x86_64-sun-solaris",
111 "x86_64-unknown-cloudabi",
112 "x86_64-unknown-freebsd",
113 "x86_64-unknown-linux-gnu",
114 "x86_64-unknown-linux-gnux32",
115 "x86_64-unknown-linux-musl",
116 "x86_64-unknown-netbsd",
117 "x86_64-unknown-redox",
120 static DOCS_TARGETS: &'static [&'static str] = &[
122 "i686-pc-windows-gnu",
123 "i686-pc-windows-msvc",
124 "i686-unknown-linux-gnu",
125 "x86_64-apple-darwin",
126 "x86_64-pc-windows-gnu",
127 "x86_64-pc-windows-msvc",
128 "x86_64-unknown-linux-gnu",
131 static MINGW: &'static [&'static str] = &[
132 "i686-pc-windows-gnu",
133 "x86_64-pc-windows-gnu",
137 #[serde(rename_all = "kebab-case")]
139 manifest_version: String,
141 pkg: BTreeMap<String, Package>,
142 renames: BTreeMap<String, Rename>
148 git_commit_hash: Option<String>,
149 target: BTreeMap<String, Target>,
161 hash: Option<String>,
162 xz_url: Option<String>,
163 xz_hash: Option<String>,
164 components: Option<Vec<Component>>,
165 extensions: Option<Vec<Component>>,
169 fn unavailable() -> Target {
189 ($e:expr) => (match $e {
191 Err(e) => panic!("{} failed with {}", stringify!($e), e),
196 rust_release: String,
197 cargo_release: String,
199 clippy_release: String,
200 rustfmt_release: String,
201 llvm_tools_release: String,
202 lldb_release: String,
206 gpg_passphrase: String,
207 digests: BTreeMap<String, String>,
211 rust_version: Option<String>,
212 cargo_version: Option<String>,
213 rls_version: Option<String>,
214 clippy_version: Option<String>,
215 rustfmt_version: Option<String>,
216 llvm_tools_version: Option<String>,
217 lldb_version: Option<String>,
219 rust_git_commit_hash: Option<String>,
220 cargo_git_commit_hash: Option<String>,
221 rls_git_commit_hash: Option<String>,
222 clippy_git_commit_hash: Option<String>,
223 rustfmt_git_commit_hash: Option<String>,
224 llvm_tools_git_commit_hash: Option<String>,
225 lldb_git_commit_hash: Option<String>,
231 // Avoid signing packages while manually testing
232 // Do NOT set this envvar in CI
233 let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
235 // Safety check to ensure signing is always enabled on CI
236 // The CI environment variable is set by both Travis and AppVeyor
237 if !should_sign && env::var("CI").is_ok() {
238 println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
239 println!("If you're not running this on CI, unset the 'CI' env var.");
243 let mut args = env::args().skip(1);
244 let input = PathBuf::from(args.next().unwrap());
245 let output = PathBuf::from(args.next().unwrap());
246 let date = args.next().unwrap();
247 let rust_release = args.next().unwrap();
248 let cargo_release = args.next().unwrap();
249 let rls_release = args.next().unwrap();
250 let clippy_release = args.next().unwrap();
251 let rustfmt_release = args.next().unwrap();
252 let llvm_tools_release = args.next().unwrap();
253 let lldb_release = args.next().unwrap();
254 let s3_address = args.next().unwrap();
256 // Do not ask for a passphrase while manually testing
257 let mut passphrase = String::new();
259 t!(io::stdin().read_to_string(&mut passphrase));
273 gpg_passphrase: passphrase,
274 digests: BTreeMap::new(),
281 clippy_version: None,
282 rustfmt_version: None,
283 llvm_tools_version: None,
286 rust_git_commit_hash: None,
287 cargo_git_commit_hash: None,
288 rls_git_commit_hash: None,
289 clippy_git_commit_hash: None,
290 rustfmt_git_commit_hash: None,
291 llvm_tools_git_commit_hash: None,
292 lldb_git_commit_hash: None,
299 fn build(&mut self) {
300 self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
301 self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
302 self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
303 self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
304 self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
305 self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
306 // lldb is only built for macOS.
307 self.lldb_version = self.version("lldb", "x86_64-apple-darwin");
309 self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
310 self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
311 self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
312 self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
313 self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
314 self.llvm_tools_git_commit_hash = self.git_commit_hash("llvm-tools",
315 "x86_64-unknown-linux-gnu");
316 self.lldb_git_commit_hash = self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
318 self.digest_and_sign();
319 let manifest = self.build_manifest();
320 self.write_channel_files(&self.rust_release, &manifest);
322 if self.rust_release != "beta" && self.rust_release != "nightly" {
323 self.write_channel_files("stable", &manifest);
327 fn digest_and_sign(&mut self) {
328 for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
329 let filename = file.file_name().unwrap().to_str().unwrap();
330 let digest = self.hash(&file);
332 assert!(self.digests.insert(filename.to_string(), digest).is_none());
336 fn build_manifest(&mut self) -> Manifest {
337 let mut manifest = Manifest {
338 manifest_version: "2".to_string(),
339 date: self.date.to_string(),
340 pkg: BTreeMap::new(),
341 renames: BTreeMap::new(),
344 self.package("rustc", &mut manifest.pkg, HOSTS);
345 self.package("cargo", &mut manifest.pkg, HOSTS);
346 self.package("rust-mingw", &mut manifest.pkg, MINGW);
347 self.package("rust-std", &mut manifest.pkg, TARGETS);
348 self.package("rust-docs", &mut manifest.pkg, DOCS_TARGETS);
349 self.package("rust-src", &mut manifest.pkg, &["*"]);
350 self.package("rls-preview", &mut manifest.pkg, HOSTS);
351 self.package("clippy-preview", &mut manifest.pkg, HOSTS);
352 self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
353 self.package("rust-analysis", &mut manifest.pkg, TARGETS);
354 self.package("llvm-tools-preview", &mut manifest.pkg, TARGETS);
355 self.package("lldb-preview", &mut manifest.pkg, TARGETS);
357 manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
358 manifest.renames.insert("rustfmt".to_owned(), Rename { to: "rustfmt-preview".to_owned() });
359 manifest.renames.insert("clippy".to_owned(), Rename { to: "clippy-preview".to_owned() });
361 let mut pkg = Package {
362 version: self.cached_version("rust")
364 .expect("Couldn't find Rust version")
366 git_commit_hash: self.cached_git_commit_hash("rust").clone(),
367 target: BTreeMap::new(),
370 let filename = self.filename("rust", host);
371 let digest = match self.digests.remove(&filename) {
372 Some(digest) => digest,
374 pkg.target.insert(host.to_string(), Target::unavailable());
378 let xz_filename = filename.replace(".tar.gz", ".tar.xz");
379 let xz_digest = self.digests.remove(&xz_filename);
380 let mut components = Vec::new();
381 let mut extensions = Vec::new();
383 // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
384 // if it's available for the target.
385 components.extend(vec![
386 Component { pkg: "rustc".to_string(), target: host.to_string() },
387 Component { pkg: "rust-std".to_string(), target: host.to_string() },
388 Component { pkg: "cargo".to_string(), target: host.to_string() },
389 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
391 if host.contains("pc-windows-gnu") {
392 components.push(Component {
393 pkg: "rust-mingw".to_string(),
394 target: host.to_string(),
398 // Tools are always present in the manifest, but might be marked as unavailable if they
400 extensions.extend(vec![
401 Component { pkg: "clippy-preview".to_string(), target: host.to_string() },
402 Component { pkg: "rls-preview".to_string(), target: host.to_string() },
403 Component { pkg: "rustfmt-preview".to_string(), target: host.to_string() },
404 Component { pkg: "llvm-tools-preview".to_string(), target: host.to_string() },
405 Component { pkg: "lldb-preview".to_string(), target: host.to_string() },
406 Component { pkg: "rust-analysis".to_string(), target: host.to_string() },
409 for target in TARGETS {
411 extensions.push(Component {
412 pkg: "rust-std".to_string(),
413 target: target.to_string(),
417 extensions.push(Component {
418 pkg: "rust-src".to_string(),
419 target: "*".to_string(),
422 // If the components/extensions don't actually exist for this
423 // particular host/target combination then nix it entirely from our
426 let has_component = |c: &Component| {
430 let pkg = match manifest.pkg.get(&c.pkg) {
432 None => return false,
434 pkg.target.get(&c.target).is_some()
436 extensions.retain(&has_component);
437 components.retain(&has_component);
440 pkg.target.insert(host.to_string(), Target {
442 url: Some(self.url(&filename)),
444 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
446 components: Some(components),
447 extensions: Some(extensions),
450 manifest.pkg.insert("rust".to_string(), pkg);
455 fn package(&mut self,
457 dst: &mut BTreeMap<String, Package>,
459 let (version, is_present) = match *self.cached_version(pkgname) {
460 Some(ref version) => (version.clone(), true),
461 None => (String::new(), false),
464 let targets = targets.iter().map(|name| {
466 let filename = self.filename(pkgname, name);
467 let digest = match self.digests.remove(&filename) {
468 Some(digest) => digest,
469 None => return (name.to_string(), Target::unavailable()),
471 let xz_filename = filename.replace(".tar.gz", ".tar.xz");
472 let xz_digest = self.digests.remove(&xz_filename);
474 (name.to_string(), Target {
476 url: Some(self.url(&filename)),
478 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
484 // If the component is not present for this build add it anyway but mark it as
485 // unavailable -- this way rustup won't allow upgrades without --force
486 (name.to_string(), Target {
498 dst.insert(pkgname.to_string(), Package {
500 git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
505 fn url(&self, filename: &str) -> String {
512 fn filename(&self, component: &str, target: &str) -> String {
513 if component == "rust-src" {
514 format!("rust-src-{}.tar.gz", self.rust_release)
515 } else if component == "cargo" {
516 format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
517 } else if component == "rls" || component == "rls-preview" {
518 format!("rls-{}-{}.tar.gz", self.rls_release, target)
519 } else if component == "clippy" || component == "clippy-preview" {
520 format!("clippy-{}-{}.tar.gz", self.clippy_release, target)
521 } else if component == "rustfmt" || component == "rustfmt-preview" {
522 format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
523 } else if component == "llvm-tools" || component == "llvm-tools-preview" {
524 format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target)
525 } else if component == "lldb" || component == "lldb-preview" {
526 format!("lldb-{}-{}.tar.gz", self.lldb_release, target)
528 format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
532 fn cached_version(&self, component: &str) -> &Option<String> {
533 if component == "cargo" {
535 } else if component == "rls" || component == "rls-preview" {
537 } else if component == "clippy" || component == "clippy-preview" {
539 } else if component == "rustfmt" || component == "rustfmt-preview" {
540 &self.rustfmt_version
541 } else if component == "llvm-tools" || component == "llvm-tools-preview" {
542 &self.llvm_tools_version
543 } else if component == "lldb" || component == "lldb-preview" {
550 fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
551 if component == "cargo" {
552 &self.cargo_git_commit_hash
553 } else if component == "rls" || component == "rls-preview" {
554 &self.rls_git_commit_hash
555 } else if component == "clippy" || component == "clippy-preview" {
556 &self.clippy_git_commit_hash
557 } else if component == "rustfmt" || component == "rustfmt-preview" {
558 &self.rustfmt_git_commit_hash
559 } else if component == "llvm-tools" || component == "llvm-tools-preview" {
560 &self.llvm_tools_git_commit_hash
561 } else if component == "lldb" || component == "lldb-preview" {
562 &self.lldb_git_commit_hash
564 &self.rust_git_commit_hash
568 fn version(&self, component: &str, target: &str) -> Option<String> {
569 let mut cmd = Command::new("tar");
570 let filename = self.filename(component, target);
572 .arg(self.input.join(&filename))
573 .arg(format!("{}/version", filename.replace(".tar.gz", "")))
575 let output = t!(cmd.output());
576 if output.status.success() {
577 Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
579 // Perhaps we didn't build this package.
584 fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
585 let mut cmd = Command::new("tar");
586 let filename = self.filename(component, target);
588 .arg(self.input.join(&filename))
589 .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
591 let output = t!(cmd.output());
592 if output.status.success() {
593 Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
599 fn hash(&self, path: &Path) -> String {
600 let sha = t!(Command::new("shasum")
601 .arg("-a").arg("256")
602 .arg(path.file_name().unwrap())
603 .current_dir(path.parent().unwrap())
605 assert!(sha.status.success());
607 let filename = path.file_name().unwrap().to_str().unwrap();
608 let sha256 = self.output.join(format!("{}.sha256", filename));
609 t!(fs::write(&sha256, &sha.stdout));
611 let stdout = String::from_utf8_lossy(&sha.stdout);
612 stdout.split_whitespace().next().unwrap().to_string()
615 fn sign(&self, path: &Path) {
616 if !self.should_sign {
620 let filename = path.file_name().unwrap().to_str().unwrap();
621 let asc = self.output.join(format!("{}.asc", filename));
622 println!("signing: {:?}", path);
623 let mut cmd = Command::new("gpg");
624 cmd.arg("--pinentry-mode=loopback")
628 .arg("--passphrase-fd").arg("0")
629 .arg("--personal-digest-preferences").arg("SHA512")
631 .arg("--output").arg(&asc)
632 .arg("--detach-sign").arg(path)
633 .stdin(Stdio::piped());
634 let mut child = t!(cmd.spawn());
635 t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
636 assert!(t!(child.wait()).success());
639 fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
640 self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
641 self.write(&manifest.date, channel_name, "-date.txt");
642 self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
643 channel_name, "-git-commit-hash.txt");
646 fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
647 let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
648 t!(fs::write(&dst, contents));