]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/tool.rs
Auto merge of #50710 - Zoxc:value_to_constvalue, r=oli-obk
[rust.git] / src / bootstrap / tool.rs
1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use std::fs;
12 use std::env;
13 use std::iter;
14 use std::path::PathBuf;
15 use std::process::{Command, exit};
16
17 use Mode;
18 use Compiler;
19 use builder::{Step, RunConfig, ShouldRun, Builder};
20 use util::{exe, add_lib_path};
21 use compile::{self, libtest_stamp, libstd_stamp, librustc_stamp};
22 use native;
23 use channel::GitInfo;
24 use cache::Interned;
25 use toolstate::ToolState;
26
27 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
28 pub struct CleanTools {
29     pub compiler: Compiler,
30     pub target: Interned<String>,
31     pub mode: Mode,
32 }
33
34 impl Step for CleanTools {
35     type Output = ();
36
37     fn should_run(run: ShouldRun) -> ShouldRun {
38         run.never()
39     }
40
41     fn run(self, builder: &Builder) {
42         let compiler = self.compiler;
43         let target = self.target;
44         let mode = self.mode;
45
46         // This is for the original compiler, but if we're forced to use stage 1, then
47         // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
48         // we copy the libs forward.
49         let tools_dir = builder.stage_out(compiler, Mode::Tool);
50         let compiler = if builder.force_use_stage1(compiler, target) {
51             builder.compiler(1, compiler.host)
52         } else {
53             compiler
54         };
55
56         for &cur_mode in &[Mode::Libstd, Mode::Libtest, Mode::Librustc] {
57             let stamp = match cur_mode {
58                 Mode::Libstd => libstd_stamp(builder, compiler, target),
59                 Mode::Libtest => libtest_stamp(builder, compiler, target),
60                 Mode::Librustc => librustc_stamp(builder, compiler, target),
61                 _ => panic!(),
62             };
63
64             if builder.clear_if_dirty(&tools_dir, &stamp) {
65                 break;
66             }
67
68             // If we are a rustc tool, and std changed, we also need to clear ourselves out -- our
69             // dependencies depend on std. Therefore, we iterate up until our own mode.
70             if mode == cur_mode {
71                 break;
72             }
73         }
74     }
75 }
76
77 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
78 struct ToolBuild {
79     compiler: Compiler,
80     target: Interned<String>,
81     tool: &'static str,
82     path: &'static str,
83     mode: Mode,
84     is_ext_tool: bool,
85     extra_features: Vec<String>,
86 }
87
88 impl Step for ToolBuild {
89     type Output = Option<PathBuf>;
90
91     fn should_run(run: ShouldRun) -> ShouldRun {
92         run.never()
93     }
94
95     /// Build a tool in `src/tools`
96     ///
97     /// This will build the specified tool with the specified `host` compiler in
98     /// `stage` into the normal cargo output directory.
99     fn run(self, builder: &Builder) -> Option<PathBuf> {
100         let compiler = self.compiler;
101         let target = self.target;
102         let tool = self.tool;
103         let path = self.path;
104         let is_ext_tool = self.is_ext_tool;
105
106         match self.mode {
107             Mode::Libstd => builder.ensure(compile::Std { compiler, target }),
108             Mode::Libtest => builder.ensure(compile::Test { compiler, target }),
109             Mode::Librustc => builder.ensure(compile::Rustc { compiler, target }),
110             Mode::Tool => panic!("unexpected Mode::Tool for tool build")
111         }
112
113         let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path);
114         cargo.arg("--features").arg(self.extra_features.join(" "));
115
116         let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
117         builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
118         let mut duplicates = Vec::new();
119         let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
120             // Only care about big things like the RLS/Cargo for now
121             if tool != "rls" && tool != "cargo" {
122                 return
123             }
124             let (id, features, filenames) = match msg {
125                 compile::CargoMessage::CompilerArtifact {
126                     package_id,
127                     features,
128                     filenames
129                 } => {
130                     (package_id, features, filenames)
131                 }
132                 _ => return,
133             };
134             let features = features.iter().map(|s| s.to_string()).collect::<Vec<_>>();
135
136             for path in filenames {
137                 let val = (tool, PathBuf::from(&*path), features.clone());
138                 // we're only interested in deduplicating rlibs for now
139                 if val.1.extension().and_then(|s| s.to_str()) != Some("rlib") {
140                     continue
141                 }
142
143                 // Don't worry about libs that turn out to be host dependencies
144                 // or build scripts, we only care about target dependencies that
145                 // are in `deps`.
146                 if let Some(maybe_target) = val.1
147                     .parent()                   // chop off file name
148                     .and_then(|p| p.parent())   // chop off `deps`
149                     .and_then(|p| p.parent())   // chop off `release`
150                     .and_then(|p| p.file_name())
151                     .and_then(|p| p.to_str())
152                 {
153                     if maybe_target != &*target {
154                         continue
155                     }
156                 }
157
158                 let mut artifacts = builder.tool_artifacts.borrow_mut();
159                 let prev_artifacts = artifacts
160                     .entry(target)
161                     .or_insert_with(Default::default);
162                 if let Some(prev) = prev_artifacts.get(&*id) {
163                     if prev.1 != val.1 {
164                         duplicates.push((
165                             id.to_string(),
166                             val,
167                             prev.clone(),
168                         ));
169                     }
170                     return
171                 }
172                 prev_artifacts.insert(id.to_string(), val);
173             }
174         });
175
176         if is_expected && duplicates.len() != 0 {
177             println!("duplicate artfacts found when compiling a tool, this \
178                       typically means that something was recompiled because \
179                       a transitive dependency has different features activated \
180                       than in a previous build:\n");
181             for (id, cur, prev) in duplicates {
182                 println!("  {}", id);
183                 println!("    `{}` enabled features {:?} at {:?}",
184                          cur.0, cur.2, cur.1);
185                 println!("    `{}` enabled features {:?} at {:?}",
186                          prev.0, prev.2, prev.1);
187             }
188             println!("");
189             panic!("tools should not compile multiple copies of the same crate");
190         }
191
192         builder.save_toolstate(tool, if is_expected {
193             ToolState::TestFail
194         } else {
195             ToolState::BuildFail
196         });
197
198         if !is_expected {
199             if !is_ext_tool {
200                 exit(1);
201             } else {
202                 return None;
203             }
204         } else {
205             let cargo_out = builder.cargo_out(compiler, Mode::Tool, target)
206                 .join(exe(tool, &compiler.host));
207             let bin = builder.tools_dir(compiler).join(exe(tool, &compiler.host));
208             builder.copy(&cargo_out, &bin);
209             Some(bin)
210         }
211     }
212 }
213
214 pub fn prepare_tool_cargo(
215     builder: &Builder,
216     compiler: Compiler,
217     target: Interned<String>,
218     command: &'static str,
219     path: &'static str,
220 ) -> Command {
221     let mut cargo = builder.cargo(compiler, Mode::Tool, target, command);
222     let dir = builder.src.join(path);
223     cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
224
225     // We don't want to build tools dynamically as they'll be running across
226     // stages and such and it's just easier if they're not dynamically linked.
227     cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
228
229     if let Some(dir) = builder.openssl_install_dir(target) {
230         cargo.env("OPENSSL_STATIC", "1");
231         cargo.env("OPENSSL_DIR", dir);
232         cargo.env("LIBZ_SYS_STATIC", "1");
233     }
234
235     // if tools are using lzma we want to force the build script to build its
236     // own copy
237     cargo.env("LZMA_API_STATIC", "1");
238
239     cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel);
240     cargo.env("CFG_VERSION", builder.rust_version());
241
242     let info = GitInfo::new(&builder.config, &dir);
243     if let Some(sha) = info.sha() {
244         cargo.env("CFG_COMMIT_HASH", sha);
245     }
246     if let Some(sha_short) = info.sha_short() {
247         cargo.env("CFG_SHORT_COMMIT_HASH", sha_short);
248     }
249     if let Some(date) = info.commit_date() {
250         cargo.env("CFG_COMMIT_DATE", date);
251     }
252     cargo
253 }
254
255 macro_rules! tool {
256     ($($name:ident, $path:expr, $tool_name:expr, $mode:expr;)+) => {
257         #[derive(Copy, Clone)]
258         pub enum Tool {
259             $(
260                 $name,
261             )+
262         }
263
264         impl<'a> Builder<'a> {
265             pub fn tool_exe(&self, tool: Tool) -> PathBuf {
266                 let stage = self.tool_default_stage(tool);
267                 match tool {
268                     $(Tool::$name =>
269                         self.ensure($name {
270                             compiler: self.compiler(stage, self.config.build),
271                             target: self.config.build,
272                         }),
273                     )+
274                 }
275             }
276
277             pub fn tool_default_stage(&self, tool: Tool) -> u32 {
278                 // Compile the error-index in the same stage as rustdoc to avoid
279                 // recompiling rustdoc twice if we can. Otherwise compile
280                 // everything else in stage0 as there's no need to rebootstrap
281                 // everything.
282                 match tool {
283                     Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
284                     _ => 0,
285                 }
286             }
287         }
288
289         $(
290             #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
291         pub struct $name {
292             pub compiler: Compiler,
293             pub target: Interned<String>,
294         }
295
296         impl Step for $name {
297             type Output = PathBuf;
298
299             fn should_run(run: ShouldRun) -> ShouldRun {
300                 run.path($path)
301             }
302
303             fn make_run(run: RunConfig) {
304                 run.builder.ensure($name {
305                     compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
306                     target: run.target,
307                 });
308             }
309
310             fn run(self, builder: &Builder) -> PathBuf {
311                 builder.ensure(ToolBuild {
312                     compiler: self.compiler,
313                     target: self.target,
314                     tool: $tool_name,
315                     mode: $mode,
316                     path: $path,
317                     is_ext_tool: false,
318                     extra_features: Vec::new(),
319                 }).expect("expected to build -- essential tool")
320             }
321         }
322         )+
323     }
324 }
325
326 tool!(
327     Rustbook, "src/tools/rustbook", "rustbook", Mode::Librustc;
328     ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::Librustc;
329     UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen", Mode::Libstd;
330     Tidy, "src/tools/tidy", "tidy", Mode::Libstd;
331     Linkchecker, "src/tools/linkchecker", "linkchecker", Mode::Libstd;
332     CargoTest, "src/tools/cargotest", "cargotest", Mode::Libstd;
333     Compiletest, "src/tools/compiletest", "compiletest", Mode::Libtest;
334     BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::Libstd;
335     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::Libstd;
336     RustInstaller, "src/tools/rust-installer", "fabricate", Mode::Libstd;
337     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::Libstd;
338 );
339
340 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
341 pub struct RemoteTestServer {
342     pub compiler: Compiler,
343     pub target: Interned<String>,
344 }
345
346 impl Step for RemoteTestServer {
347     type Output = PathBuf;
348
349     fn should_run(run: ShouldRun) -> ShouldRun {
350         run.path("src/tools/remote-test-server")
351     }
352
353     fn make_run(run: RunConfig) {
354         run.builder.ensure(RemoteTestServer {
355             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
356             target: run.target,
357         });
358     }
359
360     fn run(self, builder: &Builder) -> PathBuf {
361         builder.ensure(ToolBuild {
362             compiler: self.compiler,
363             target: self.target,
364             tool: "remote-test-server",
365             mode: Mode::Libstd,
366             path: "src/tools/remote-test-server",
367             is_ext_tool: false,
368             extra_features: Vec::new(),
369         }).expect("expected to build -- essential tool")
370     }
371 }
372
373 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
374 pub struct Rustdoc {
375     pub host: Interned<String>,
376 }
377
378 impl Step for Rustdoc {
379     type Output = PathBuf;
380     const DEFAULT: bool = true;
381     const ONLY_HOSTS: bool = true;
382
383     fn should_run(run: ShouldRun) -> ShouldRun {
384         run.path("src/tools/rustdoc")
385     }
386
387     fn make_run(run: RunConfig) {
388         run.builder.ensure(Rustdoc {
389             host: run.host,
390         });
391     }
392
393     fn run(self, builder: &Builder) -> PathBuf {
394         let target_compiler = builder.compiler(builder.top_stage, self.host);
395         let target = target_compiler.host;
396         let build_compiler = if target_compiler.stage == 0 {
397             builder.compiler(0, builder.config.build)
398         } else if target_compiler.stage >= 2 {
399             // Past stage 2, we consider the compiler to be ABI-compatible and hence capable of
400             // building rustdoc itself.
401             builder.compiler(target_compiler.stage, builder.config.build)
402         } else {
403             // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
404             // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
405             // compilers, which isn't what we want.
406             builder.compiler(target_compiler.stage - 1, builder.config.build)
407         };
408
409         builder.ensure(compile::Rustc { compiler: build_compiler, target });
410         builder.ensure(compile::Rustc {
411             compiler: build_compiler,
412             target: builder.config.build,
413         });
414
415         let mut cargo = prepare_tool_cargo(builder,
416                                            build_compiler,
417                                            target,
418                                            "build",
419                                            "src/tools/rustdoc");
420
421         // Most tools don't get debuginfo, but rustdoc should.
422         cargo.env("RUSTC_DEBUGINFO", builder.config.rust_debuginfo.to_string())
423              .env("RUSTC_DEBUGINFO_LINES", builder.config.rust_debuginfo_lines.to_string());
424
425         let _folder = builder.fold_output(|| format!("stage{}-rustdoc", target_compiler.stage));
426         builder.info(&format!("Building rustdoc for stage{} ({})",
427             target_compiler.stage, target_compiler.host));
428         builder.run(&mut cargo);
429
430         // Cargo adds a number of paths to the dylib search path on windows, which results in
431         // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
432         // rustdoc a different name.
433         let tool_rustdoc = builder.cargo_out(build_compiler, Mode::Tool, target)
434             .join(exe("rustdoc-tool-binary", &target_compiler.host));
435
436         // don't create a stage0-sysroot/bin directory.
437         if target_compiler.stage > 0 {
438             let sysroot = builder.sysroot(target_compiler);
439             let bindir = sysroot.join("bin");
440             t!(fs::create_dir_all(&bindir));
441             let bin_rustdoc = bindir.join(exe("rustdoc", &*target_compiler.host));
442             let _ = fs::remove_file(&bin_rustdoc);
443             builder.copy(&tool_rustdoc, &bin_rustdoc);
444             bin_rustdoc
445         } else {
446             tool_rustdoc
447         }
448     }
449 }
450
451 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
452 pub struct Cargo {
453     pub compiler: Compiler,
454     pub target: Interned<String>,
455 }
456
457 impl Step for Cargo {
458     type Output = PathBuf;
459     const DEFAULT: bool = true;
460     const ONLY_HOSTS: bool = true;
461
462     fn should_run(run: ShouldRun) -> ShouldRun {
463         let builder = run.builder;
464         run.path("src/tools/cargo").default_condition(builder.config.extended)
465     }
466
467     fn make_run(run: RunConfig) {
468         run.builder.ensure(Cargo {
469             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
470             target: run.target,
471         });
472     }
473
474     fn run(self, builder: &Builder) -> PathBuf {
475         builder.ensure(native::Openssl {
476             target: self.target,
477         });
478         // Cargo depends on procedural macros, which requires a full host
479         // compiler to be available, so we need to depend on that.
480         builder.ensure(compile::Rustc {
481             compiler: self.compiler,
482             target: builder.config.build,
483         });
484         builder.ensure(ToolBuild {
485             compiler: self.compiler,
486             target: self.target,
487             tool: "cargo",
488             mode: Mode::Librustc,
489             path: "src/tools/cargo",
490             is_ext_tool: false,
491             extra_features: Vec::new(),
492         }).expect("expected to build -- essential tool")
493     }
494 }
495
496 macro_rules! tool_extended {
497     (($sel:ident, $builder:ident),
498        $($name:ident,
499        $toolstate:ident,
500        $path:expr,
501        $tool_name:expr,
502        $extra_deps:block;)+) => {
503         $(
504             #[derive(Debug, Clone, Hash, PartialEq, Eq)]
505         pub struct $name {
506             pub compiler: Compiler,
507             pub target: Interned<String>,
508             pub extra_features: Vec<String>,
509         }
510
511         impl Step for $name {
512             type Output = Option<PathBuf>;
513             const DEFAULT: bool = true;
514             const ONLY_HOSTS: bool = true;
515
516             fn should_run(run: ShouldRun) -> ShouldRun {
517                 let builder = run.builder;
518                 run.path($path).default_condition(builder.config.extended)
519             }
520
521             fn make_run(run: RunConfig) {
522                 run.builder.ensure($name {
523                     compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
524                     target: run.target,
525                     extra_features: Vec::new(),
526                 });
527             }
528
529             #[allow(unused_mut)]
530             fn run(mut $sel, $builder: &Builder) -> Option<PathBuf> {
531                 $extra_deps
532                 $builder.ensure(ToolBuild {
533                     compiler: $sel.compiler,
534                     target: $sel.target,
535                     tool: $tool_name,
536                     mode: Mode::Librustc,
537                     path: $path,
538                     extra_features: $sel.extra_features,
539                     is_ext_tool: true,
540                 })
541             }
542         }
543         )+
544     }
545 }
546
547 tool_extended!((self, builder),
548     Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", {};
549     Clippy, clippy, "src/tools/clippy", "clippy-driver", {
550         // Clippy depends on procedural macros (serde), which requires a full host
551         // compiler to be available, so we need to depend on that.
552         builder.ensure(compile::Rustc {
553             compiler: self.compiler,
554             target: builder.config.build,
555         });
556     };
557     Miri, miri, "src/tools/miri", "miri", {};
558     Rls, rls, "src/tools/rls", "rls", {
559         builder.ensure(native::Openssl {
560             target: self.target,
561         });
562         // RLS depends on procedural macros, which requires a full host
563         // compiler to be available, so we need to depend on that.
564         builder.ensure(compile::Rustc {
565             compiler: self.compiler,
566             target: builder.config.build,
567         });
568     };
569     Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", {};
570 );
571
572 impl<'a> Builder<'a> {
573     /// Get a `Command` which is ready to run `tool` in `stage` built for
574     /// `host`.
575     pub fn tool_cmd(&self, tool: Tool) -> Command {
576         let mut cmd = Command::new(self.tool_exe(tool));
577         let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
578         self.prepare_tool_cmd(compiler, &mut cmd);
579         cmd
580     }
581
582     /// Prepares the `cmd` provided to be able to run the `compiler` provided.
583     ///
584     /// Notably this munges the dynamic library lookup path to point to the
585     /// right location to run `compiler`.
586     fn prepare_tool_cmd(&self, compiler: Compiler, cmd: &mut Command) {
587         let host = &compiler.host;
588         let mut lib_paths: Vec<PathBuf> = vec![
589             PathBuf::from(&self.sysroot_libdir(compiler, compiler.host)),
590             self.cargo_out(compiler, Mode::Tool, *host).join("deps"),
591         ];
592
593         // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make
594         // mode) and that C compiler may need some extra PATH modification. Do
595         // so here.
596         if compiler.host.contains("msvc") {
597             let curpaths = env::var_os("PATH").unwrap_or_default();
598             let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>();
599             for &(ref k, ref v) in self.cc[&compiler.host].env() {
600                 if k != "PATH" {
601                     continue
602                 }
603                 for path in env::split_paths(v) {
604                     if !curpaths.contains(&path) {
605                         lib_paths.push(path);
606                     }
607                 }
608             }
609         }
610
611         // Add the llvm/bin directory to PATH since it contains lots of
612         // useful, platform-independent tools
613         if let Some(llvm_bin_path) = self.llvm_bin_path() {
614             if host.contains("windows") {
615                 // On Windows, PATH and the dynamic library path are the same,
616                 // so we just add the LLVM bin path to lib_path
617                 lib_paths.push(llvm_bin_path);
618             } else {
619                 let old_path = env::var_os("PATH").unwrap_or_default();
620                 let new_path = env::join_paths(iter::once(llvm_bin_path)
621                         .chain(env::split_paths(&old_path)))
622                     .expect("Could not add LLVM bin path to PATH");
623                 cmd.env("PATH", new_path);
624             }
625         }
626
627         add_lib_path(lib_paths, cmd);
628     }
629
630     fn llvm_bin_path(&self) -> Option<PathBuf> {
631         if self.config.llvm_enabled && !self.config.dry_run {
632             let llvm_config = self.ensure(native::Llvm {
633                 target: self.config.build,
634                 emscripten: false,
635             });
636
637             // Add the llvm/bin directory to PATH since it contains lots of
638             // useful, platform-independent tools
639             let llvm_bin_path = llvm_config.parent()
640                 .expect("Expected llvm-config to be contained in directory");
641             assert!(llvm_bin_path.is_dir());
642             Some(llvm_bin_path.to_path_buf())
643         } else {
644             None
645         }
646     }
647 }