1 //! Build a dist manifest, hash and sign everything.
2 //! This gets called by `promote-release`
3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
5 //! by rustbuild (in `src/bootstrap/dist.rs`).
10 use crate::manifest::{Component, FileHash, Manifest, Package, Rename, Target};
11 use crate::versions::{PkgType, Versions};
12 use rayon::prelude::*;
14 use std::collections::{BTreeMap, HashMap, HashSet};
16 use std::error::Error;
17 use std::fs::{self, File};
18 use std::io::{self, BufReader, Read, Write};
19 use std::path::{Path, PathBuf};
20 use std::process::{Command, Stdio};
22 use std::time::Instant;
24 static HOSTS: &[&str] = &[
25 "aarch64-apple-darwin",
26 "aarch64-pc-windows-msvc",
27 "aarch64-unknown-linux-gnu",
28 "aarch64-unknown-linux-musl",
29 "arm-unknown-linux-gnueabi",
30 "arm-unknown-linux-gnueabihf",
31 "armv7-unknown-linux-gnueabihf",
33 "i686-pc-windows-gnu",
34 "i686-pc-windows-msvc",
35 "i686-unknown-linux-gnu",
36 "mips-unknown-linux-gnu",
37 "mips64-unknown-linux-gnuabi64",
38 "mips64el-unknown-linux-gnuabi64",
39 "mipsel-unknown-linux-gnu",
40 "mipsisa32r6-unknown-linux-gnu",
41 "mipsisa32r6el-unknown-linux-gnu",
42 "mipsisa64r6-unknown-linux-gnuabi64",
43 "mipsisa64r6el-unknown-linux-gnuabi64",
44 "powerpc-unknown-linux-gnu",
45 "powerpc64-unknown-linux-gnu",
46 "powerpc64le-unknown-linux-gnu",
47 "riscv64gc-unknown-linux-gnu",
48 "s390x-unknown-linux-gnu",
49 "x86_64-apple-darwin",
50 "x86_64-pc-windows-gnu",
51 "x86_64-pc-windows-msvc",
52 "x86_64-unknown-freebsd",
53 "x86_64-unknown-illumos",
54 "x86_64-unknown-linux-gnu",
55 "x86_64-unknown-linux-musl",
56 "x86_64-unknown-netbsd",
59 static TARGETS: &[&str] = &[
60 "aarch64-apple-darwin",
63 "aarch64-linux-android",
64 "aarch64-pc-windows-msvc",
65 "aarch64-unknown-cloudabi",
66 "aarch64-unknown-hermit",
67 "aarch64-unknown-linux-gnu",
68 "aarch64-unknown-linux-musl",
69 "aarch64-unknown-none",
70 "aarch64-unknown-none-softfloat",
71 "aarch64-unknown-redox",
72 "arm-linux-androideabi",
73 "arm-unknown-linux-gnueabi",
74 "arm-unknown-linux-gnueabihf",
75 "arm-unknown-linux-musleabi",
76 "arm-unknown-linux-musleabihf",
77 "armv5te-unknown-linux-gnueabi",
78 "armv5te-unknown-linux-musleabi",
80 "armv7-linux-androideabi",
81 "thumbv7neon-linux-androideabi",
82 "armv7-unknown-linux-gnueabi",
83 "armv7-unknown-linux-gnueabihf",
85 "thumbv7neon-unknown-linux-gnueabihf",
86 "armv7-unknown-linux-musleabi",
87 "armv7-unknown-linux-musleabihf",
89 "armebv7r-none-eabihf",
93 "asmjs-unknown-emscripten",
95 "i586-pc-windows-msvc",
96 "i586-unknown-linux-gnu",
97 "i586-unknown-linux-musl",
100 "i686-pc-windows-gnu",
101 "i686-pc-windows-msvc",
102 "i686-unknown-freebsd",
103 "i686-unknown-linux-gnu",
104 "i686-unknown-linux-musl",
105 "mips-unknown-linux-gnu",
106 "mips-unknown-linux-musl",
107 "mips64-unknown-linux-gnuabi64",
108 "mips64-unknown-linux-muslabi64",
109 "mips64el-unknown-linux-gnuabi64",
110 "mips64el-unknown-linux-muslabi64",
111 "mipsisa32r6-unknown-linux-gnu",
112 "mipsisa32r6el-unknown-linux-gnu",
113 "mipsisa64r6-unknown-linux-gnuabi64",
114 "mipsisa64r6el-unknown-linux-gnuabi64",
115 "mipsel-unknown-linux-gnu",
116 "mipsel-unknown-linux-musl",
117 "nvptx64-nvidia-cuda",
118 "powerpc-unknown-linux-gnu",
119 "powerpc64-unknown-linux-gnu",
120 "powerpc64le-unknown-linux-gnu",
121 "riscv32i-unknown-none-elf",
122 "riscv32imc-unknown-none-elf",
123 "riscv32imac-unknown-none-elf",
124 "riscv32gc-unknown-linux-gnu",
125 "riscv64imac-unknown-none-elf",
126 "riscv64gc-unknown-none-elf",
127 "riscv64gc-unknown-linux-gnu",
128 "s390x-unknown-linux-gnu",
129 "sparc64-unknown-linux-gnu",
130 "sparcv9-sun-solaris",
131 "thumbv6m-none-eabi",
132 "thumbv7em-none-eabi",
133 "thumbv7em-none-eabihf",
134 "thumbv7m-none-eabi",
135 "thumbv8m.base-none-eabi",
136 "thumbv8m.main-none-eabi",
137 "thumbv8m.main-none-eabihf",
138 "wasm32-unknown-emscripten",
139 "wasm32-unknown-unknown",
141 "x86_64-apple-darwin",
143 "x86_64-fortanix-unknown-sgx",
145 "x86_64-linux-android",
146 "x86_64-pc-windows-gnu",
147 "x86_64-pc-windows-msvc",
148 "x86_64-rumprun-netbsd",
149 "x86_64-sun-solaris",
151 "x86_64-unknown-cloudabi",
152 "x86_64-unknown-freebsd",
153 "x86_64-unknown-illumos",
154 "x86_64-unknown-linux-gnu",
155 "x86_64-unknown-linux-gnux32",
156 "x86_64-unknown-linux-musl",
157 "x86_64-unknown-netbsd",
158 "x86_64-unknown-redox",
159 "x86_64-unknown-hermit",
162 static DOCS_TARGETS: &[&str] = &[
164 "i686-pc-windows-gnu",
165 "i686-pc-windows-msvc",
166 "i686-unknown-linux-gnu",
167 "x86_64-apple-darwin",
168 "x86_64-pc-windows-gnu",
169 "x86_64-pc-windows-msvc",
170 "x86_64-unknown-linux-gnu",
171 "x86_64-unknown-linux-musl",
174 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
176 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
182 Err(e) => panic!("{} failed with {}", stringify!($e), e),
196 legacy_gpg_passphrase: String,
200 // Up until Rust 1.48 the release process relied on build-manifest to create the SHA256
201 // checksums of released files and to sign the tarballs. That was moved over to promote-release
202 // in time for the branching of Rust 1.48, but the old release process still had to work the
205 // When running build-manifest through the old ./x.py dist hash-and-sign the environment
206 // variable will be set, enabling the legacy behavior of generating the .sha256 files and
207 // signing the tarballs.
209 // Once the old release process is fully decommissioned, the environment variable, all the
210 // related code in this tool and ./x.py dist hash-and-sign can be removed.
211 let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
213 let num_threads = if legacy {
214 // Avoid overloading the old server in legacy mode.
216 } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
217 num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
221 rayon::ThreadPoolBuilder::new()
222 .num_threads(num_threads)
224 .expect("failed to initialize Rayon");
226 let mut args = env::args().skip(1);
227 let input = PathBuf::from(args.next().unwrap());
228 let output = PathBuf::from(args.next().unwrap());
229 let date = args.next().unwrap();
230 let s3_address = args.next().unwrap();
231 let channel = args.next().unwrap();
233 // Do not ask for a passphrase while manually testing
234 let mut passphrase = String::new();
236 // `x.py` passes the passphrase via stdin.
237 t!(io::stdin().read_to_string(&mut passphrase));
241 versions: Versions::new(&channel, &input).unwrap(),
249 legacy_gpg_passphrase: passphrase,
255 fn build(&mut self) {
256 self.check_toolstate();
258 self.digest_and_sign();
260 let manifest = self.build_manifest();
262 self.write_channel_files(self.versions.channel(), &manifest);
263 if self.versions.channel() == "stable" {
264 // channel-rust-1.XX.YY.toml
265 let rust_version = self.versions.rustc_version();
266 self.write_channel_files(rust_version, &manifest);
268 // channel-rust-1.XX.toml
269 let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
270 self.write_channel_files(&major_minor, &manifest);
274 /// If a tool does not pass its tests, don't ship it.
275 /// Right now, we do this only for Miri.
276 fn check_toolstate(&mut self) {
277 let toolstates: Option<HashMap<String, String>> =
278 File::open(self.input.join("toolstates-linux.json"))
280 .and_then(|f| serde_json::from_reader(&f).ok());
281 let toolstates = toolstates.unwrap_or_else(|| {
283 "WARNING: `toolstates-linux.json` missing/malformed; \
284 assuming all tools failed"
286 HashMap::default() // Use empty map if anything went wrong.
288 // Mark some tools as missing based on toolstate.
289 if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
290 println!("Miri tests are not passing, removing component");
291 self.versions.disable_version(&PkgType::Miri);
295 /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
296 fn digest_and_sign(&mut self) {
297 for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
298 file.file_name().unwrap().to_str().unwrap();
304 fn build_manifest(&mut self) -> Manifest {
305 let mut manifest = Manifest {
306 manifest_version: "2".to_string(),
307 date: self.date.to_string(),
308 pkg: BTreeMap::new(),
309 renames: BTreeMap::new(),
310 profiles: BTreeMap::new(),
312 self.add_packages_to(&mut manifest);
313 self.add_profiles_to(&mut manifest);
314 self.add_renames_to(&mut manifest);
315 manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
317 self.fill_missing_hashes(&mut manifest);
322 fn add_packages_to(&mut self, manifest: &mut Manifest) {
323 let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
324 package("rustc", HOSTS);
325 package("rustc-dev", HOSTS);
326 package("rustc-docs", HOSTS);
327 package("cargo", HOSTS);
328 package("rust-mingw", MINGW);
329 package("rust-std", TARGETS);
330 package("rust-docs", DOCS_TARGETS);
331 package("rust-src", &["*"]);
332 package("rls-preview", HOSTS);
333 package("rust-analyzer-preview", HOSTS);
334 package("clippy-preview", HOSTS);
335 package("miri-preview", HOSTS);
336 package("rustfmt-preview", HOSTS);
337 package("rust-analysis", TARGETS);
338 package("llvm-tools-preview", TARGETS);
341 fn add_profiles_to(&mut self, manifest: &mut Manifest) {
342 let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
343 profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
367 "rust-analyzer-preview",
369 "llvm-tools-preview",
375 // The compiler libraries are not stable for end users, and they're also huge, so we only
376 // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
377 // for users to install the additional component manually, if needed.
378 if self.versions.channel() == "nightly" {
379 self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
380 self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
384 fn add_renames_to(&self, manifest: &mut Manifest) {
385 let mut rename = |from: &str, to: &str| {
386 manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
388 rename("rls", "rls-preview");
389 rename("rustfmt", "rustfmt-preview");
390 rename("clippy", "clippy-preview");
391 rename("miri", "miri-preview");
394 fn rust_package(&mut self, manifest: &Manifest) -> Package {
395 let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
396 let mut pkg = Package {
397 version: version_info.version.expect("missing Rust version"),
398 git_commit_hash: version_info.git_commit,
399 target: BTreeMap::new(),
402 if let Some(target) = self.target_host_combination(host, &manifest) {
403 pkg.target.insert(host.to_string(), target);
405 pkg.target.insert(host.to_string(), Target::unavailable());
412 fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
413 let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
415 let mut target = Target::from_compressed_tar(self, &filename);
416 if !target.available {
420 let mut components = Vec::new();
421 let mut extensions = Vec::new();
423 let host_component = |pkg| Component::from_str(pkg, host);
425 // rustc/rust-std/cargo/docs are all required,
426 // and so is rust-mingw if it's available for the target.
427 components.extend(vec![
428 host_component("rustc"),
429 host_component("rust-std"),
430 host_component("cargo"),
431 host_component("rust-docs"),
433 if host.contains("pc-windows-gnu") {
434 components.push(host_component("rust-mingw"));
437 // Tools are always present in the manifest,
438 // but might be marked as unavailable if they weren't built.
439 extensions.extend(vec![
440 host_component("clippy-preview"),
441 host_component("miri-preview"),
442 host_component("rls-preview"),
443 host_component("rust-analyzer-preview"),
444 host_component("rustfmt-preview"),
445 host_component("llvm-tools-preview"),
446 host_component("rust-analysis"),
452 .filter(|&&target| target != host)
453 .map(|target| Component::from_str("rust-std", target)),
455 extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
456 extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
457 extensions.push(Component::from_str("rust-src", "*"));
459 // If the components/extensions don't actually exist for this
460 // particular host/target combination then nix it entirely from our
462 let has_component = |c: &Component| {
466 let pkg = match manifest.pkg.get(&c.pkg) {
468 None => return false,
470 pkg.target.get(&c.target).is_some()
472 extensions.retain(&has_component);
473 components.retain(&has_component);
475 target.components = Some(components);
476 target.extensions = Some(extensions);
483 dst: &mut BTreeMap<String, Vec<String>>,
486 dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
492 dst: &mut BTreeMap<String, Vec<String>>,
495 dst.get_mut(profile_name)
496 .expect("existing profile")
497 .extend(pkgs.iter().map(|s| (*s).to_owned()));
500 fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
501 let version_info = self
503 .version(&PkgType::from_component(pkgname))
504 .expect("failed to load package version");
505 let mut is_present = version_info.present;
507 // Never ship nightly-only components for other trains.
508 if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
509 is_present = false; // Pretend the component is entirely missing.
512 let targets = targets
515 let target = if is_present {
518 .tarball_name(&PkgType::from_component(pkgname), name)
521 Target::from_compressed_tar(self, &filename)
523 // If the component is not present for this build add it anyway but mark it as
524 // unavailable -- this way rustup won't allow upgrades without --force
525 Target::unavailable()
527 (name.to_string(), target)
534 version: version_info.version.unwrap_or_default(),
535 git_commit_hash: version_info.git_commit,
541 fn url(&self, path: &Path) -> String {
542 let file_name = path.file_name().unwrap().to_str().unwrap();
543 format!("{}/{}/{}", self.s3_address, self.date, file_name)
546 fn hash(&self, path: &Path) -> String {
547 let sha = t!(Command::new("shasum")
550 .arg(path.file_name().unwrap())
551 .current_dir(path.parent().unwrap())
553 assert!(sha.status.success());
555 let filename = path.file_name().unwrap().to_str().unwrap();
556 let sha256 = self.output.join(format!("{}.sha256", filename));
557 t!(fs::write(&sha256, &sha.stdout));
559 let stdout = String::from_utf8_lossy(&sha.stdout);
560 stdout.split_whitespace().next().unwrap().to_string()
563 fn sign(&self, path: &Path) {
568 let filename = path.file_name().unwrap().to_str().unwrap();
569 let asc = self.output.join(format!("{}.asc", filename));
570 println!("signing: {:?}", path);
571 let mut cmd = Command::new("gpg");
572 cmd.arg("--pinentry-mode=loopback")
576 .arg("--passphrase-fd")
578 .arg("--personal-digest-preferences")
583 .arg("--detach-sign")
585 .stdin(Stdio::piped());
586 let mut child = t!(cmd.spawn());
587 t!(child.stdin.take().unwrap().write_all(self.legacy_gpg_passphrase.as_bytes()));
588 assert!(t!(child.wait()).success());
591 fn fill_missing_hashes(&self, manifest: &mut Manifest) {
592 // First collect all files that need hashes
593 let mut need_hashes = HashSet::new();
594 crate::manifest::visit_file_hashes(manifest, |file_hash| {
595 if let FileHash::Missing(path) = file_hash {
596 need_hashes.insert(path.clone());
600 let collected = Mutex::new(HashMap::new());
601 let collection_start = Instant::now();
603 "collecting hashes for {} tarballs across {} threads",
605 rayon::current_num_threads().min(need_hashes.len()),
607 need_hashes.par_iter().for_each(|path| match fetch_hash(path) {
609 collected.lock().unwrap().insert(path, hash);
611 Err(err) => eprintln!("error while fetching the hash for {}: {}", path.display(), err),
613 let collected = collected.into_inner().unwrap();
614 println!("collected {} hashes in {:.2?}", collected.len(), collection_start.elapsed());
616 crate::manifest::visit_file_hashes(manifest, |file_hash| {
617 if let FileHash::Missing(path) = file_hash {
618 match collected.get(path) {
619 Some(hash) => *file_hash = FileHash::Present(hash.clone()),
620 None => panic!("missing hash for file {}", path.display()),
626 fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
627 self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
628 self.write(&manifest.date, channel_name, "-date.txt");
630 manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
632 "-git-commit-hash.txt",
636 fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
637 let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
638 t!(fs::write(&dst, contents));
646 fn fetch_hash(path: &Path) -> Result<String, Box<dyn Error>> {
647 let mut file = BufReader::new(File::open(path)?);
648 let mut sha256 = sha2::Sha256::default();
649 std::io::copy(&mut file, &mut sha256)?;
650 Ok(hex::encode(sha256.finalize()))