]> git.lizzy.rs Git - rust.git/blob - src/tools/build-manifest/src/main.rs
Auto merge of #57129 - RalfJung:check-bounds, r=oli-obk
[rust.git] / src / tools / build-manifest / src / main.rs
1 extern crate toml;
2 #[macro_use]
3 extern crate serde_derive;
4
5 use std::collections::BTreeMap;
6 use std::env;
7 use std::fs;
8 use std::io::{self, Read, Write};
9 use std::path::{PathBuf, Path};
10 use std::process::{Command, Stdio};
11
12 static HOSTS: &'static [&'static str] = &[
13     "aarch64-unknown-linux-gnu",
14     "arm-unknown-linux-gnueabi",
15     "arm-unknown-linux-gnueabihf",
16     "armv7-unknown-linux-gnueabihf",
17     "i686-apple-darwin",
18     "i686-pc-windows-gnu",
19     "i686-pc-windows-msvc",
20     "i686-unknown-linux-gnu",
21     "mips-unknown-linux-gnu",
22     "mips64-unknown-linux-gnuabi64",
23     "mips64el-unknown-linux-gnuabi64",
24     "mipsel-unknown-linux-gnu",
25     "powerpc-unknown-linux-gnu",
26     "powerpc64-unknown-linux-gnu",
27     "powerpc64le-unknown-linux-gnu",
28     "s390x-unknown-linux-gnu",
29     "x86_64-apple-darwin",
30     "x86_64-pc-windows-gnu",
31     "x86_64-pc-windows-msvc",
32     "x86_64-unknown-freebsd",
33     "x86_64-unknown-linux-gnu",
34     "x86_64-unknown-netbsd",
35 ];
36
37 static TARGETS: &'static [&'static str] = &[
38     "aarch64-apple-ios",
39     "aarch64-fuchsia",
40     "aarch64-linux-android",
41     "aarch64-pc-windows-msvc",
42     "aarch64-unknown-cloudabi",
43     "aarch64-unknown-linux-gnu",
44     "aarch64-unknown-linux-musl",
45     "arm-linux-androideabi",
46     "arm-unknown-linux-gnueabi",
47     "arm-unknown-linux-gnueabihf",
48     "arm-unknown-linux-musleabi",
49     "arm-unknown-linux-musleabihf",
50     "armv5te-unknown-linux-gnueabi",
51     "armv5te-unknown-linux-musleabi",
52     "armv7-apple-ios",
53     "armv7-linux-androideabi",
54     "thumbv7neon-linux-androideabi",
55     "armv7-unknown-linux-gnueabihf",
56     "thumbv7neon-unknown-linux-gnueabihf",
57     "armv7-unknown-linux-musleabihf",
58     "armebv7r-none-eabi",
59     "armebv7r-none-eabihf",
60     "armv7r-none-eabi",
61     "armv7r-none-eabihf",
62     "armv7s-apple-ios",
63     "asmjs-unknown-emscripten",
64     "i386-apple-ios",
65     "i586-pc-windows-msvc",
66     "i586-unknown-linux-gnu",
67     "i586-unknown-linux-musl",
68     "i686-apple-darwin",
69     "i686-linux-android",
70     "i686-pc-windows-gnu",
71     "i686-pc-windows-msvc",
72     "i686-unknown-freebsd",
73     "i686-unknown-linux-gnu",
74     "i686-unknown-linux-musl",
75     "mips-unknown-linux-gnu",
76     "mips-unknown-linux-musl",
77     "mips64-unknown-linux-gnuabi64",
78     "mips64el-unknown-linux-gnuabi64",
79     "mipsel-unknown-linux-gnu",
80     "mipsel-unknown-linux-musl",
81     "powerpc-unknown-linux-gnu",
82     "powerpc64-unknown-linux-gnu",
83     "powerpc64le-unknown-linux-gnu",
84     "riscv32imc-unknown-none-elf",
85     "riscv32imac-unknown-none-elf",
86     "s390x-unknown-linux-gnu",
87     "sparc64-unknown-linux-gnu",
88     "sparcv9-sun-solaris",
89     "thumbv6m-none-eabi",
90     "thumbv7em-none-eabi",
91     "thumbv7em-none-eabihf",
92     "thumbv7m-none-eabi",
93     "wasm32-unknown-emscripten",
94     "wasm32-unknown-unknown",
95     "x86_64-apple-darwin",
96     "x86_64-apple-ios",
97     "x86_64-fuchsia",
98     "x86_64-linux-android",
99     "x86_64-pc-windows-gnu",
100     "x86_64-pc-windows-msvc",
101     "x86_64-rumprun-netbsd",
102     "x86_64-sun-solaris",
103     "x86_64-unknown-cloudabi",
104     "x86_64-unknown-freebsd",
105     "x86_64-unknown-linux-gnu",
106     "x86_64-unknown-linux-gnux32",
107     "x86_64-unknown-linux-musl",
108     "x86_64-unknown-netbsd",
109     "x86_64-unknown-redox",
110 ];
111
112 static DOCS_TARGETS: &'static [&'static str] = &[
113     "i686-apple-darwin",
114     "i686-pc-windows-gnu",
115     "i686-pc-windows-msvc",
116     "i686-unknown-linux-gnu",
117     "x86_64-apple-darwin",
118     "x86_64-pc-windows-gnu",
119     "x86_64-pc-windows-msvc",
120     "x86_64-unknown-linux-gnu",
121 ];
122
123 static MINGW: &'static [&'static str] = &[
124     "i686-pc-windows-gnu",
125     "x86_64-pc-windows-gnu",
126 ];
127
128 #[derive(Serialize)]
129 #[serde(rename_all = "kebab-case")]
130 struct Manifest {
131     manifest_version: String,
132     date: String,
133     pkg: BTreeMap<String, Package>,
134     renames: BTreeMap<String, Rename>
135 }
136
137 #[derive(Serialize)]
138 struct Package {
139     version: String,
140     git_commit_hash: Option<String>,
141     target: BTreeMap<String, Target>,
142 }
143
144 #[derive(Serialize)]
145 struct Rename {
146     to: String,
147 }
148
149 #[derive(Serialize)]
150 struct Target {
151     available: bool,
152     url: Option<String>,
153     hash: Option<String>,
154     xz_url: Option<String>,
155     xz_hash: Option<String>,
156     components: Option<Vec<Component>>,
157     extensions: Option<Vec<Component>>,
158 }
159
160 impl Target {
161     fn unavailable() -> Target {
162         Target {
163             available: false,
164             url: None,
165             hash: None,
166             xz_url: None,
167             xz_hash: None,
168             components: None,
169             extensions: None,
170         }
171     }
172 }
173
174 #[derive(Serialize)]
175 struct Component {
176     pkg: String,
177     target: String,
178 }
179
180 macro_rules! t {
181     ($e:expr) => (match $e {
182         Ok(e) => e,
183         Err(e) => panic!("{} failed with {}", stringify!($e), e),
184     })
185 }
186
187 struct Builder {
188     rust_release: String,
189     cargo_release: String,
190     rls_release: String,
191     clippy_release: String,
192     rustfmt_release: String,
193     llvm_tools_release: String,
194     lldb_release: String,
195
196     input: PathBuf,
197     output: PathBuf,
198     gpg_passphrase: String,
199     digests: BTreeMap<String, String>,
200     s3_address: String,
201     date: String,
202
203     rust_version: Option<String>,
204     cargo_version: Option<String>,
205     rls_version: Option<String>,
206     clippy_version: Option<String>,
207     rustfmt_version: Option<String>,
208     llvm_tools_version: Option<String>,
209     lldb_version: Option<String>,
210
211     rust_git_commit_hash: Option<String>,
212     cargo_git_commit_hash: Option<String>,
213     rls_git_commit_hash: Option<String>,
214     clippy_git_commit_hash: Option<String>,
215     rustfmt_git_commit_hash: Option<String>,
216     llvm_tools_git_commit_hash: Option<String>,
217     lldb_git_commit_hash: Option<String>,
218
219     should_sign: bool,
220 }
221
222 fn main() {
223     // Avoid signing packages while manually testing
224     // Do NOT set this envvar in CI
225     let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
226
227     // Safety check to ensure signing is always enabled on CI
228     // The CI environment variable is set by both Travis and AppVeyor
229     if !should_sign && env::var("CI").is_ok() {
230         println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
231         println!("If you're not running this on CI, unset the 'CI' env var.");
232         panic!();
233     }
234
235     let mut args = env::args().skip(1);
236     let input = PathBuf::from(args.next().unwrap());
237     let output = PathBuf::from(args.next().unwrap());
238     let date = args.next().unwrap();
239     let rust_release = args.next().unwrap();
240     let cargo_release = args.next().unwrap();
241     let rls_release = args.next().unwrap();
242     let clippy_release = args.next().unwrap();
243     let rustfmt_release = args.next().unwrap();
244     let llvm_tools_release = args.next().unwrap();
245     let lldb_release = args.next().unwrap();
246     let s3_address = args.next().unwrap();
247
248     // Do not ask for a passphrase while manually testing
249     let mut passphrase = String::new();
250     if should_sign {
251         t!(io::stdin().read_to_string(&mut passphrase));
252     }
253
254     Builder {
255         rust_release,
256         cargo_release,
257         rls_release,
258         clippy_release,
259         rustfmt_release,
260         llvm_tools_release,
261         lldb_release,
262
263         input,
264         output,
265         gpg_passphrase: passphrase,
266         digests: BTreeMap::new(),
267         s3_address,
268         date,
269
270         rust_version: None,
271         cargo_version: None,
272         rls_version: None,
273         clippy_version: None,
274         rustfmt_version: None,
275         llvm_tools_version: None,
276         lldb_version: None,
277
278         rust_git_commit_hash: None,
279         cargo_git_commit_hash: None,
280         rls_git_commit_hash: None,
281         clippy_git_commit_hash: None,
282         rustfmt_git_commit_hash: None,
283         llvm_tools_git_commit_hash: None,
284         lldb_git_commit_hash: None,
285
286         should_sign,
287     }.build();
288 }
289
290 impl Builder {
291     fn build(&mut self) {
292         self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu");
293         self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu");
294         self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu");
295         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
296         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
297         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
298         // lldb is only built for macOS.
299         self.lldb_version = self.version("lldb", "x86_64-apple-darwin");
300
301         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
302         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
303         self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu");
304         self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu");
305         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
306         self.llvm_tools_git_commit_hash = self.git_commit_hash("llvm-tools",
307                                                                "x86_64-unknown-linux-gnu");
308         self.lldb_git_commit_hash = self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
309
310         self.digest_and_sign();
311         let manifest = self.build_manifest();
312         self.write_channel_files(&self.rust_release, &manifest);
313
314         if self.rust_release != "beta" && self.rust_release != "nightly" {
315             self.write_channel_files("stable", &manifest);
316         }
317     }
318
319     fn digest_and_sign(&mut self) {
320         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
321             let filename = file.file_name().unwrap().to_str().unwrap();
322             let digest = self.hash(&file);
323             self.sign(&file);
324             assert!(self.digests.insert(filename.to_string(), digest).is_none());
325         }
326     }
327
328     fn build_manifest(&mut self) -> Manifest {
329         let mut manifest = Manifest {
330             manifest_version: "2".to_string(),
331             date: self.date.to_string(),
332             pkg: BTreeMap::new(),
333             renames: BTreeMap::new(),
334         };
335
336         self.package("rustc", &mut manifest.pkg, HOSTS);
337         self.package("cargo", &mut manifest.pkg, HOSTS);
338         self.package("rust-mingw", &mut manifest.pkg, MINGW);
339         self.package("rust-std", &mut manifest.pkg, TARGETS);
340         self.package("rust-docs", &mut manifest.pkg, DOCS_TARGETS);
341         self.package("rust-src", &mut manifest.pkg, &["*"]);
342         self.package("rls-preview", &mut manifest.pkg, HOSTS);
343         self.package("clippy-preview", &mut manifest.pkg, HOSTS);
344         self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
345         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
346         self.package("llvm-tools-preview", &mut manifest.pkg, TARGETS);
347         self.package("lldb-preview", &mut manifest.pkg, TARGETS);
348
349         manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
350         manifest.renames.insert("rustfmt".to_owned(), Rename { to: "rustfmt-preview".to_owned() });
351         manifest.renames.insert("clippy".to_owned(), Rename { to: "clippy-preview".to_owned() });
352
353         let mut pkg = Package {
354             version: self.cached_version("rust")
355                          .as_ref()
356                          .expect("Couldn't find Rust version")
357                          .clone(),
358             git_commit_hash: self.cached_git_commit_hash("rust").clone(),
359             target: BTreeMap::new(),
360         };
361         for host in HOSTS {
362             let filename = self.filename("rust", host);
363             let digest = match self.digests.remove(&filename) {
364                 Some(digest) => digest,
365                 None => {
366                     pkg.target.insert(host.to_string(), Target::unavailable());
367                     continue
368                 }
369             };
370             let xz_filename = filename.replace(".tar.gz", ".tar.xz");
371             let xz_digest = self.digests.remove(&xz_filename);
372             let mut components = Vec::new();
373             let mut extensions = Vec::new();
374
375             // rustc/rust-std/cargo/docs are all required, and so is rust-mingw
376             // if it's available for the target.
377             components.extend(vec![
378                 Component { pkg: "rustc".to_string(), target: host.to_string() },
379                 Component { pkg: "rust-std".to_string(), target: host.to_string() },
380                 Component { pkg: "cargo".to_string(), target: host.to_string() },
381                 Component { pkg: "rust-docs".to_string(), target: host.to_string() },
382             ]);
383             if host.contains("pc-windows-gnu") {
384                 components.push(Component {
385                     pkg: "rust-mingw".to_string(),
386                     target: host.to_string(),
387                 });
388             }
389
390             // Tools are always present in the manifest, but might be marked as unavailable if they
391             // weren't built
392             extensions.extend(vec![
393                 Component { pkg: "clippy-preview".to_string(), target: host.to_string() },
394                 Component { pkg: "rls-preview".to_string(), target: host.to_string() },
395                 Component { pkg: "rustfmt-preview".to_string(), target: host.to_string() },
396                 Component { pkg: "llvm-tools-preview".to_string(), target: host.to_string() },
397                 Component { pkg: "lldb-preview".to_string(), target: host.to_string() },
398                 Component { pkg: "rust-analysis".to_string(), target: host.to_string() },
399             ]);
400
401             for target in TARGETS {
402                 if target != host {
403                     extensions.push(Component {
404                         pkg: "rust-std".to_string(),
405                         target: target.to_string(),
406                     });
407                 }
408             }
409             extensions.push(Component {
410                 pkg: "rust-src".to_string(),
411                 target: "*".to_string(),
412             });
413
414             // If the components/extensions don't actually exist for this
415             // particular host/target combination then nix it entirely from our
416             // lists.
417             {
418                 let has_component = |c: &Component| {
419                     if c.target == "*" {
420                         return true
421                     }
422                     let pkg = match manifest.pkg.get(&c.pkg) {
423                         Some(p) => p,
424                         None => return false,
425                     };
426                     pkg.target.get(&c.target).is_some()
427                 };
428                 extensions.retain(&has_component);
429                 components.retain(&has_component);
430             }
431
432             pkg.target.insert(host.to_string(), Target {
433                 available: true,
434                 url: Some(self.url(&filename)),
435                 hash: Some(digest),
436                 xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
437                 xz_hash: xz_digest,
438                 components: Some(components),
439                 extensions: Some(extensions),
440             });
441         }
442         manifest.pkg.insert("rust".to_string(), pkg);
443
444         return manifest;
445     }
446
447     fn package(&mut self,
448                pkgname: &str,
449                dst: &mut BTreeMap<String, Package>,
450                targets: &[&str]) {
451         let (version, is_present) = match *self.cached_version(pkgname) {
452             Some(ref version) => (version.clone(), true),
453             None => (String::new(), false),
454         };
455
456         let targets = targets.iter().map(|name| {
457             if is_present {
458                 let filename = self.filename(pkgname, name);
459                 let digest = match self.digests.remove(&filename) {
460                     Some(digest) => digest,
461                     None => return (name.to_string(), Target::unavailable()),
462                 };
463                 let xz_filename = filename.replace(".tar.gz", ".tar.xz");
464                 let xz_digest = self.digests.remove(&xz_filename);
465
466                 (name.to_string(), Target {
467                     available: true,
468                     url: Some(self.url(&filename)),
469                     hash: Some(digest),
470                     xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
471                     xz_hash: xz_digest,
472                     components: None,
473                     extensions: None,
474                 })
475             } else {
476                 // If the component is not present for this build add it anyway but mark it as
477                 // unavailable -- this way rustup won't allow upgrades without --force
478                 (name.to_string(), Target {
479                     available: false,
480                     url: None,
481                     hash: None,
482                     xz_url: None,
483                     xz_hash: None,
484                     components: None,
485                     extensions: None,
486                 })
487             }
488         }).collect();
489
490         dst.insert(pkgname.to_string(), Package {
491             version,
492             git_commit_hash: self.cached_git_commit_hash(pkgname).clone(),
493             target: targets,
494         });
495     }
496
497     fn url(&self, filename: &str) -> String {
498         format!("{}/{}/{}",
499                 self.s3_address,
500                 self.date,
501                 filename)
502     }
503
504     fn filename(&self, component: &str, target: &str) -> String {
505         if component == "rust-src" {
506             format!("rust-src-{}.tar.gz", self.rust_release)
507         } else if component == "cargo" {
508             format!("cargo-{}-{}.tar.gz", self.cargo_release, target)
509         } else if component == "rls" || component == "rls-preview" {
510             format!("rls-{}-{}.tar.gz", self.rls_release, target)
511         } else if component == "clippy" || component == "clippy-preview" {
512             format!("clippy-{}-{}.tar.gz", self.clippy_release, target)
513         } else if component == "rustfmt" || component == "rustfmt-preview" {
514             format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
515         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
516             format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target)
517         } else if component == "lldb" || component == "lldb-preview" {
518             format!("lldb-{}-{}.tar.gz", self.lldb_release, target)
519         } else {
520             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
521         }
522     }
523
524     fn cached_version(&self, component: &str) -> &Option<String> {
525         if component == "cargo" {
526             &self.cargo_version
527         } else if component == "rls" || component == "rls-preview" {
528             &self.rls_version
529         } else if component == "clippy" || component == "clippy-preview" {
530             &self.clippy_version
531         } else if component == "rustfmt" || component == "rustfmt-preview" {
532             &self.rustfmt_version
533         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
534             &self.llvm_tools_version
535         } else if component == "lldb" || component == "lldb-preview" {
536             &self.lldb_version
537         } else {
538             &self.rust_version
539         }
540     }
541
542     fn cached_git_commit_hash(&self, component: &str) -> &Option<String> {
543         if component == "cargo" {
544             &self.cargo_git_commit_hash
545         } else if component == "rls" || component == "rls-preview" {
546             &self.rls_git_commit_hash
547         } else if component == "clippy" || component == "clippy-preview" {
548             &self.clippy_git_commit_hash
549         } else if component == "rustfmt" || component == "rustfmt-preview" {
550             &self.rustfmt_git_commit_hash
551         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
552             &self.llvm_tools_git_commit_hash
553         } else if component == "lldb" || component == "lldb-preview" {
554             &self.lldb_git_commit_hash
555         } else {
556             &self.rust_git_commit_hash
557         }
558     }
559
560     fn version(&self, component: &str, target: &str) -> Option<String> {
561         let mut cmd = Command::new("tar");
562         let filename = self.filename(component, target);
563         cmd.arg("xf")
564            .arg(self.input.join(&filename))
565            .arg(format!("{}/version", filename.replace(".tar.gz", "")))
566            .arg("-O");
567         let output = t!(cmd.output());
568         if output.status.success() {
569             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
570         } else {
571             // Perhaps we didn't build this package.
572             None
573         }
574     }
575
576     fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
577         let mut cmd = Command::new("tar");
578         let filename = self.filename(component, target);
579         cmd.arg("xf")
580            .arg(self.input.join(&filename))
581            .arg(format!("{}/git-commit-hash", filename.replace(".tar.gz", "")))
582            .arg("-O");
583         let output = t!(cmd.output());
584         if output.status.success() {
585             Some(String::from_utf8_lossy(&output.stdout).trim().to_string())
586         } else {
587             None
588         }
589     }
590
591     fn hash(&self, path: &Path) -> String {
592         let sha = t!(Command::new("shasum")
593                         .arg("-a").arg("256")
594                         .arg(path.file_name().unwrap())
595                         .current_dir(path.parent().unwrap())
596                         .output());
597         assert!(sha.status.success());
598
599         let filename = path.file_name().unwrap().to_str().unwrap();
600         let sha256 = self.output.join(format!("{}.sha256", filename));
601         t!(fs::write(&sha256, &sha.stdout));
602
603         let stdout = String::from_utf8_lossy(&sha.stdout);
604         stdout.split_whitespace().next().unwrap().to_string()
605     }
606
607     fn sign(&self, path: &Path) {
608         if !self.should_sign {
609             return;
610         }
611
612         let filename = path.file_name().unwrap().to_str().unwrap();
613         let asc = self.output.join(format!("{}.asc", filename));
614         println!("signing: {:?}", path);
615         let mut cmd = Command::new("gpg");
616         cmd.arg("--pinentry-mode=loopback")
617             .arg("--no-tty")
618             .arg("--yes")
619             .arg("--batch")
620             .arg("--passphrase-fd").arg("0")
621             .arg("--personal-digest-preferences").arg("SHA512")
622             .arg("--armor")
623             .arg("--output").arg(&asc)
624             .arg("--detach-sign").arg(path)
625             .stdin(Stdio::piped());
626         let mut child = t!(cmd.spawn());
627         t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
628         assert!(t!(child.wait()).success());
629     }
630
631     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
632         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
633         self.write(&manifest.date, channel_name, "-date.txt");
634         self.write(manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
635                    channel_name, "-git-commit-hash.txt");
636     }
637
638     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
639         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
640         t!(fs::write(&dst, contents));
641         self.hash(&dst);
642         self.sign(&dst);
643     }
644 }