]> git.lizzy.rs Git - rust.git/blob - src/tools/build-manifest/src/main.rs
Rollup merge of #77951 - ehuss:update-books, r=ehuss
[rust.git] / src / tools / build-manifest / src / main.rs
1 //! Build a dist manifest, hash and sign everything.
2 //! This gets called by `promote-release`
3 //! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
4 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
5 //! by rustbuild (in `src/bootstrap/dist.rs`).
6
7 mod manifest;
8 mod versions;
9
10 use crate::manifest::{Component, FileHash, Manifest, Package, Rename, Target};
11 use crate::versions::{PkgType, Versions};
12 use rayon::prelude::*;
13 use sha2::Digest;
14 use std::collections::{BTreeMap, HashMap, HashSet};
15 use std::env;
16 use std::error::Error;
17 use std::fs::{self, File};
18 use std::io::{self, BufReader, Read, Write};
19 use std::path::{Path, PathBuf};
20 use std::process::{Command, Stdio};
21 use std::sync::Mutex;
22 use std::time::Instant;
23
24 static HOSTS: &[&str] = &[
25     "aarch64-apple-darwin",
26     "aarch64-pc-windows-msvc",
27     "aarch64-unknown-linux-gnu",
28     "aarch64-unknown-linux-musl",
29     "arm-unknown-linux-gnueabi",
30     "arm-unknown-linux-gnueabihf",
31     "armv7-unknown-linux-gnueabihf",
32     "i686-apple-darwin",
33     "i686-pc-windows-gnu",
34     "i686-pc-windows-msvc",
35     "i686-unknown-linux-gnu",
36     "mips-unknown-linux-gnu",
37     "mips64-unknown-linux-gnuabi64",
38     "mips64el-unknown-linux-gnuabi64",
39     "mipsel-unknown-linux-gnu",
40     "mipsisa32r6-unknown-linux-gnu",
41     "mipsisa32r6el-unknown-linux-gnu",
42     "mipsisa64r6-unknown-linux-gnuabi64",
43     "mipsisa64r6el-unknown-linux-gnuabi64",
44     "powerpc-unknown-linux-gnu",
45     "powerpc64-unknown-linux-gnu",
46     "powerpc64le-unknown-linux-gnu",
47     "riscv64gc-unknown-linux-gnu",
48     "s390x-unknown-linux-gnu",
49     "x86_64-apple-darwin",
50     "x86_64-pc-windows-gnu",
51     "x86_64-pc-windows-msvc",
52     "x86_64-unknown-freebsd",
53     "x86_64-unknown-illumos",
54     "x86_64-unknown-linux-gnu",
55     "x86_64-unknown-linux-musl",
56     "x86_64-unknown-netbsd",
57 ];
58
59 static TARGETS: &[&str] = &[
60     "aarch64-apple-darwin",
61     "aarch64-apple-ios",
62     "aarch64-fuchsia",
63     "aarch64-linux-android",
64     "aarch64-pc-windows-msvc",
65     "aarch64-unknown-cloudabi",
66     "aarch64-unknown-hermit",
67     "aarch64-unknown-linux-gnu",
68     "aarch64-unknown-linux-musl",
69     "aarch64-unknown-none",
70     "aarch64-unknown-none-softfloat",
71     "aarch64-unknown-redox",
72     "arm-linux-androideabi",
73     "arm-unknown-linux-gnueabi",
74     "arm-unknown-linux-gnueabihf",
75     "arm-unknown-linux-musleabi",
76     "arm-unknown-linux-musleabihf",
77     "armv5te-unknown-linux-gnueabi",
78     "armv5te-unknown-linux-musleabi",
79     "armv7-apple-ios",
80     "armv7-linux-androideabi",
81     "thumbv7neon-linux-androideabi",
82     "armv7-unknown-linux-gnueabi",
83     "armv7-unknown-linux-gnueabihf",
84     "armv7a-none-eabi",
85     "thumbv7neon-unknown-linux-gnueabihf",
86     "armv7-unknown-linux-musleabi",
87     "armv7-unknown-linux-musleabihf",
88     "armebv7r-none-eabi",
89     "armebv7r-none-eabihf",
90     "armv7r-none-eabi",
91     "armv7r-none-eabihf",
92     "armv7s-apple-ios",
93     "asmjs-unknown-emscripten",
94     "i386-apple-ios",
95     "i586-pc-windows-msvc",
96     "i586-unknown-linux-gnu",
97     "i586-unknown-linux-musl",
98     "i686-apple-darwin",
99     "i686-linux-android",
100     "i686-pc-windows-gnu",
101     "i686-pc-windows-msvc",
102     "i686-unknown-freebsd",
103     "i686-unknown-linux-gnu",
104     "i686-unknown-linux-musl",
105     "mips-unknown-linux-gnu",
106     "mips-unknown-linux-musl",
107     "mips64-unknown-linux-gnuabi64",
108     "mips64-unknown-linux-muslabi64",
109     "mips64el-unknown-linux-gnuabi64",
110     "mips64el-unknown-linux-muslabi64",
111     "mipsisa32r6-unknown-linux-gnu",
112     "mipsisa32r6el-unknown-linux-gnu",
113     "mipsisa64r6-unknown-linux-gnuabi64",
114     "mipsisa64r6el-unknown-linux-gnuabi64",
115     "mipsel-unknown-linux-gnu",
116     "mipsel-unknown-linux-musl",
117     "nvptx64-nvidia-cuda",
118     "powerpc-unknown-linux-gnu",
119     "powerpc64-unknown-linux-gnu",
120     "powerpc64le-unknown-linux-gnu",
121     "riscv32i-unknown-none-elf",
122     "riscv32imc-unknown-none-elf",
123     "riscv32imac-unknown-none-elf",
124     "riscv32gc-unknown-linux-gnu",
125     "riscv64imac-unknown-none-elf",
126     "riscv64gc-unknown-none-elf",
127     "riscv64gc-unknown-linux-gnu",
128     "s390x-unknown-linux-gnu",
129     "sparc64-unknown-linux-gnu",
130     "sparcv9-sun-solaris",
131     "thumbv6m-none-eabi",
132     "thumbv7em-none-eabi",
133     "thumbv7em-none-eabihf",
134     "thumbv7m-none-eabi",
135     "thumbv8m.base-none-eabi",
136     "thumbv8m.main-none-eabi",
137     "thumbv8m.main-none-eabihf",
138     "wasm32-unknown-emscripten",
139     "wasm32-unknown-unknown",
140     "wasm32-wasi",
141     "x86_64-apple-darwin",
142     "x86_64-apple-ios",
143     "x86_64-fortanix-unknown-sgx",
144     "x86_64-fuchsia",
145     "x86_64-linux-android",
146     "x86_64-pc-windows-gnu",
147     "x86_64-pc-windows-msvc",
148     "x86_64-rumprun-netbsd",
149     "x86_64-sun-solaris",
150     "x86_64-pc-solaris",
151     "x86_64-unknown-cloudabi",
152     "x86_64-unknown-freebsd",
153     "x86_64-unknown-illumos",
154     "x86_64-unknown-linux-gnu",
155     "x86_64-unknown-linux-gnux32",
156     "x86_64-unknown-linux-musl",
157     "x86_64-unknown-netbsd",
158     "x86_64-unknown-redox",
159     "x86_64-unknown-hermit",
160 ];
161
162 static DOCS_TARGETS: &[&str] = &[
163     "i686-apple-darwin",
164     "i686-pc-windows-gnu",
165     "i686-pc-windows-msvc",
166     "i686-unknown-linux-gnu",
167     "x86_64-apple-darwin",
168     "x86_64-pc-windows-gnu",
169     "x86_64-pc-windows-msvc",
170     "x86_64-unknown-linux-gnu",
171     "x86_64-unknown-linux-musl",
172 ];
173
174 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
175
176 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
177
178 macro_rules! t {
179     ($e:expr) => {
180         match $e {
181             Ok(e) => e,
182             Err(e) => panic!("{} failed with {}", stringify!($e), e),
183         }
184     };
185 }
186
187 struct Builder {
188     versions: Versions,
189
190     input: PathBuf,
191     output: PathBuf,
192     s3_address: String,
193     date: String,
194
195     legacy: bool,
196     legacy_gpg_passphrase: String,
197 }
198
199 fn main() {
200     // Up until Rust 1.48 the release process relied on build-manifest to create the SHA256
201     // checksums of released files and to sign the tarballs. That was moved over to promote-release
202     // in time for the branching of Rust 1.48, but the old release process still had to work the
203     // old way.
204     //
205     // When running build-manifest through the old ./x.py dist hash-and-sign the environment
206     // variable will be set, enabling the legacy behavior of generating the .sha256 files and
207     // signing the tarballs.
208     //
209     // Once the old release process is fully decommissioned, the environment variable, all the
210     // related code in this tool and ./x.py dist hash-and-sign can be removed.
211     let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
212
213     let num_threads = if legacy {
214         // Avoid overloading the old server in legacy mode.
215         1
216     } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
217         num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
218     } else {
219         num_cpus::get()
220     };
221     rayon::ThreadPoolBuilder::new()
222         .num_threads(num_threads)
223         .build_global()
224         .expect("failed to initialize Rayon");
225
226     let mut args = env::args().skip(1);
227     let input = PathBuf::from(args.next().unwrap());
228     let output = PathBuf::from(args.next().unwrap());
229     let date = args.next().unwrap();
230     let s3_address = args.next().unwrap();
231     let channel = args.next().unwrap();
232
233     // Do not ask for a passphrase while manually testing
234     let mut passphrase = String::new();
235     if legacy {
236         // `x.py` passes the passphrase via stdin.
237         t!(io::stdin().read_to_string(&mut passphrase));
238     }
239
240     Builder {
241         versions: Versions::new(&channel, &input).unwrap(),
242
243         input,
244         output,
245         s3_address,
246         date,
247
248         legacy,
249         legacy_gpg_passphrase: passphrase,
250     }
251     .build();
252 }
253
254 impl Builder {
255     fn build(&mut self) {
256         self.check_toolstate();
257         if self.legacy {
258             self.digest_and_sign();
259         }
260         let manifest = self.build_manifest();
261
262         self.write_channel_files(self.versions.channel(), &manifest);
263         if self.versions.channel() == "stable" {
264             // channel-rust-1.XX.YY.toml
265             let rust_version = self.versions.rustc_version();
266             self.write_channel_files(rust_version, &manifest);
267
268             // channel-rust-1.XX.toml
269             let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
270             self.write_channel_files(&major_minor, &manifest);
271         }
272     }
273
274     /// If a tool does not pass its tests, don't ship it.
275     /// Right now, we do this only for Miri.
276     fn check_toolstate(&mut self) {
277         let toolstates: Option<HashMap<String, String>> =
278             File::open(self.input.join("toolstates-linux.json"))
279                 .ok()
280                 .and_then(|f| serde_json::from_reader(&f).ok());
281         let toolstates = toolstates.unwrap_or_else(|| {
282             println!(
283                 "WARNING: `toolstates-linux.json` missing/malformed; \
284                 assuming all tools failed"
285             );
286             HashMap::default() // Use empty map if anything went wrong.
287         });
288         // Mark some tools as missing based on toolstate.
289         if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
290             println!("Miri tests are not passing, removing component");
291             self.versions.disable_version(&PkgType::Miri);
292         }
293     }
294
295     /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
296     fn digest_and_sign(&mut self) {
297         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
298             file.file_name().unwrap().to_str().unwrap();
299             self.hash(&file);
300             self.sign(&file);
301         }
302     }
303
304     fn build_manifest(&mut self) -> Manifest {
305         let mut manifest = Manifest {
306             manifest_version: "2".to_string(),
307             date: self.date.to_string(),
308             pkg: BTreeMap::new(),
309             renames: BTreeMap::new(),
310             profiles: BTreeMap::new(),
311         };
312         self.add_packages_to(&mut manifest);
313         self.add_profiles_to(&mut manifest);
314         self.add_renames_to(&mut manifest);
315         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
316
317         self.fill_missing_hashes(&mut manifest);
318
319         manifest
320     }
321
322     fn add_packages_to(&mut self, manifest: &mut Manifest) {
323         let mut package = |name, targets| self.package(name, &mut manifest.pkg, targets);
324         package("rustc", HOSTS);
325         package("rustc-dev", HOSTS);
326         package("rustc-docs", HOSTS);
327         package("cargo", HOSTS);
328         package("rust-mingw", MINGW);
329         package("rust-std", TARGETS);
330         package("rust-docs", DOCS_TARGETS);
331         package("rust-src", &["*"]);
332         package("rls-preview", HOSTS);
333         package("rust-analyzer-preview", HOSTS);
334         package("clippy-preview", HOSTS);
335         package("miri-preview", HOSTS);
336         package("rustfmt-preview", HOSTS);
337         package("rust-analysis", TARGETS);
338         package("llvm-tools-preview", TARGETS);
339     }
340
341     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
342         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
343         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
344         profile(
345             "default",
346             &[
347                 "rustc",
348                 "cargo",
349                 "rust-std",
350                 "rust-mingw",
351                 "rust-docs",
352                 "rustfmt-preview",
353                 "clippy-preview",
354             ],
355         );
356         profile(
357             "complete",
358             &[
359                 "rustc",
360                 "cargo",
361                 "rust-std",
362                 "rust-mingw",
363                 "rust-docs",
364                 "rustfmt-preview",
365                 "clippy-preview",
366                 "rls-preview",
367                 "rust-analyzer-preview",
368                 "rust-src",
369                 "llvm-tools-preview",
370                 "rust-analysis",
371                 "miri-preview",
372             ],
373         );
374
375         // The compiler libraries are not stable for end users, and they're also huge, so we only
376         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
377         // for users to install the additional component manually, if needed.
378         if self.versions.channel() == "nightly" {
379             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
380             self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
381         }
382     }
383
384     fn add_renames_to(&self, manifest: &mut Manifest) {
385         let mut rename = |from: &str, to: &str| {
386             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
387         };
388         rename("rls", "rls-preview");
389         rename("rustfmt", "rustfmt-preview");
390         rename("clippy", "clippy-preview");
391         rename("miri", "miri-preview");
392     }
393
394     fn rust_package(&mut self, manifest: &Manifest) -> Package {
395         let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
396         let mut pkg = Package {
397             version: version_info.version.expect("missing Rust version"),
398             git_commit_hash: version_info.git_commit,
399             target: BTreeMap::new(),
400         };
401         for host in HOSTS {
402             if let Some(target) = self.target_host_combination(host, &manifest) {
403                 pkg.target.insert(host.to_string(), target);
404             } else {
405                 pkg.target.insert(host.to_string(), Target::unavailable());
406                 continue;
407             }
408         }
409         pkg
410     }
411
412     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
413         let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
414
415         let mut target = Target::from_compressed_tar(self, &filename);
416         if !target.available {
417             return None;
418         }
419
420         let mut components = Vec::new();
421         let mut extensions = Vec::new();
422
423         let host_component = |pkg| Component::from_str(pkg, host);
424
425         // rustc/rust-std/cargo/docs are all required,
426         // and so is rust-mingw if it's available for the target.
427         components.extend(vec![
428             host_component("rustc"),
429             host_component("rust-std"),
430             host_component("cargo"),
431             host_component("rust-docs"),
432         ]);
433         if host.contains("pc-windows-gnu") {
434             components.push(host_component("rust-mingw"));
435         }
436
437         // Tools are always present in the manifest,
438         // but might be marked as unavailable if they weren't built.
439         extensions.extend(vec![
440             host_component("clippy-preview"),
441             host_component("miri-preview"),
442             host_component("rls-preview"),
443             host_component("rust-analyzer-preview"),
444             host_component("rustfmt-preview"),
445             host_component("llvm-tools-preview"),
446             host_component("rust-analysis"),
447         ]);
448
449         extensions.extend(
450             TARGETS
451                 .iter()
452                 .filter(|&&target| target != host)
453                 .map(|target| Component::from_str("rust-std", target)),
454         );
455         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
456         extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
457         extensions.push(Component::from_str("rust-src", "*"));
458
459         // If the components/extensions don't actually exist for this
460         // particular host/target combination then nix it entirely from our
461         // lists.
462         let has_component = |c: &Component| {
463             if c.target == "*" {
464                 return true;
465             }
466             let pkg = match manifest.pkg.get(&c.pkg) {
467                 Some(p) => p,
468                 None => return false,
469             };
470             pkg.target.get(&c.target).is_some()
471         };
472         extensions.retain(&has_component);
473         components.retain(&has_component);
474
475         target.components = Some(components);
476         target.extensions = Some(extensions);
477         Some(target)
478     }
479
480     fn profile(
481         &mut self,
482         profile_name: &str,
483         dst: &mut BTreeMap<String, Vec<String>>,
484         pkgs: &[&str],
485     ) {
486         dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
487     }
488
489     fn extend_profile(
490         &mut self,
491         profile_name: &str,
492         dst: &mut BTreeMap<String, Vec<String>>,
493         pkgs: &[&str],
494     ) {
495         dst.get_mut(profile_name)
496             .expect("existing profile")
497             .extend(pkgs.iter().map(|s| (*s).to_owned()));
498     }
499
500     fn package(&mut self, pkgname: &str, dst: &mut BTreeMap<String, Package>, targets: &[&str]) {
501         let version_info = self
502             .versions
503             .version(&PkgType::from_component(pkgname))
504             .expect("failed to load package version");
505         let mut is_present = version_info.present;
506
507         // Never ship nightly-only components for other trains.
508         if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
509             is_present = false; // Pretend the component is entirely missing.
510         }
511
512         let targets = targets
513             .iter()
514             .map(|name| {
515                 let target = if is_present {
516                     let filename = self
517                         .versions
518                         .tarball_name(&PkgType::from_component(pkgname), name)
519                         .unwrap();
520
521                     Target::from_compressed_tar(self, &filename)
522                 } else {
523                     // If the component is not present for this build add it anyway but mark it as
524                     // unavailable -- this way rustup won't allow upgrades without --force
525                     Target::unavailable()
526                 };
527                 (name.to_string(), target)
528             })
529             .collect();
530
531         dst.insert(
532             pkgname.to_string(),
533             Package {
534                 version: version_info.version.unwrap_or_default(),
535                 git_commit_hash: version_info.git_commit,
536                 target: targets,
537             },
538         );
539     }
540
541     fn url(&self, path: &Path) -> String {
542         let file_name = path.file_name().unwrap().to_str().unwrap();
543         format!("{}/{}/{}", self.s3_address, self.date, file_name)
544     }
545
546     fn hash(&self, path: &Path) -> String {
547         let sha = t!(Command::new("shasum")
548             .arg("-a")
549             .arg("256")
550             .arg(path.file_name().unwrap())
551             .current_dir(path.parent().unwrap())
552             .output());
553         assert!(sha.status.success());
554
555         let filename = path.file_name().unwrap().to_str().unwrap();
556         let sha256 = self.output.join(format!("{}.sha256", filename));
557         t!(fs::write(&sha256, &sha.stdout));
558
559         let stdout = String::from_utf8_lossy(&sha.stdout);
560         stdout.split_whitespace().next().unwrap().to_string()
561     }
562
563     fn sign(&self, path: &Path) {
564         if !self.legacy {
565             return;
566         }
567
568         let filename = path.file_name().unwrap().to_str().unwrap();
569         let asc = self.output.join(format!("{}.asc", filename));
570         println!("signing: {:?}", path);
571         let mut cmd = Command::new("gpg");
572         cmd.arg("--pinentry-mode=loopback")
573             .arg("--no-tty")
574             .arg("--yes")
575             .arg("--batch")
576             .arg("--passphrase-fd")
577             .arg("0")
578             .arg("--personal-digest-preferences")
579             .arg("SHA512")
580             .arg("--armor")
581             .arg("--output")
582             .arg(&asc)
583             .arg("--detach-sign")
584             .arg(path)
585             .stdin(Stdio::piped());
586         let mut child = t!(cmd.spawn());
587         t!(child.stdin.take().unwrap().write_all(self.legacy_gpg_passphrase.as_bytes()));
588         assert!(t!(child.wait()).success());
589     }
590
591     fn fill_missing_hashes(&self, manifest: &mut Manifest) {
592         // First collect all files that need hashes
593         let mut need_hashes = HashSet::new();
594         crate::manifest::visit_file_hashes(manifest, |file_hash| {
595             if let FileHash::Missing(path) = file_hash {
596                 need_hashes.insert(path.clone());
597             }
598         });
599
600         let collected = Mutex::new(HashMap::new());
601         let collection_start = Instant::now();
602         println!(
603             "collecting hashes for {} tarballs across {} threads",
604             need_hashes.len(),
605             rayon::current_num_threads().min(need_hashes.len()),
606         );
607         need_hashes.par_iter().for_each(|path| match fetch_hash(path) {
608             Ok(hash) => {
609                 collected.lock().unwrap().insert(path, hash);
610             }
611             Err(err) => eprintln!("error while fetching the hash for {}: {}", path.display(), err),
612         });
613         let collected = collected.into_inner().unwrap();
614         println!("collected {} hashes in {:.2?}", collected.len(), collection_start.elapsed());
615
616         crate::manifest::visit_file_hashes(manifest, |file_hash| {
617             if let FileHash::Missing(path) = file_hash {
618                 match collected.get(path) {
619                     Some(hash) => *file_hash = FileHash::Present(hash.clone()),
620                     None => panic!("missing hash for file {}", path.display()),
621                 }
622             }
623         })
624     }
625
626     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
627         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
628         self.write(&manifest.date, channel_name, "-date.txt");
629         self.write(
630             manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
631             channel_name,
632             "-git-commit-hash.txt",
633         );
634     }
635
636     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
637         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
638         t!(fs::write(&dst, contents));
639         if self.legacy {
640             self.hash(&dst);
641             self.sign(&dst);
642         }
643     }
644 }
645
646 fn fetch_hash(path: &Path) -> Result<String, Box<dyn Error>> {
647     let mut file = BufReader::new(File::open(path)?);
648     let mut sha256 = sha2::Sha256::default();
649     std::io::copy(&mut file, &mut sha256)?;
650     Ok(hex::encode(sha256.finalize()))
651 }