]> git.lizzy.rs Git - rust.git/blob - src/bootstrap/tool.rs
Vertical list of tools to check
[rust.git] / src / bootstrap / tool.rs
1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use std::fs;
12 use std::env;
13 use std::iter;
14 use std::path::PathBuf;
15 use std::process::{Command, exit};
16
17 use Mode;
18 use Compiler;
19 use builder::{Step, RunConfig, ShouldRun, Builder};
20 use util::{exe, add_lib_path};
21 use compile::{self, libtest_stamp, libstd_stamp, librustc_stamp};
22 use native;
23 use channel::GitInfo;
24 use cache::Interned;
25 use toolstate::ToolState;
26
27 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
28 pub struct CleanTools {
29     pub compiler: Compiler,
30     pub target: Interned<String>,
31     pub cause: Mode,
32 }
33
34 impl Step for CleanTools {
35     type Output = ();
36
37     fn should_run(run: ShouldRun) -> ShouldRun {
38         run.never()
39     }
40
41     fn run(self, builder: &Builder) {
42         let compiler = self.compiler;
43         let target = self.target;
44         let cause = self.cause;
45
46         // This is for the original compiler, but if we're forced to use stage 1, then
47         // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
48         // we copy the libs forward.
49         let tools_dir = builder.stage_out(compiler, Mode::ToolRustc);
50         let compiler = if builder.force_use_stage1(compiler, target) {
51             builder.compiler(1, compiler.host)
52         } else {
53             compiler
54         };
55
56         for &cur_mode in &[Mode::Std, Mode::Test, Mode::Rustc] {
57             let stamp = match cur_mode {
58                 Mode::Std => libstd_stamp(builder, compiler, target),
59                 Mode::Test => libtest_stamp(builder, compiler, target),
60                 Mode::Rustc => librustc_stamp(builder, compiler, target),
61                 _ => panic!(),
62             };
63
64             if builder.clear_if_dirty(&tools_dir, &stamp) {
65                 break;
66             }
67
68             // If we are a rustc tool, and std changed, we also need to clear ourselves out -- our
69             // dependencies depend on std. Therefore, we iterate up until our own mode.
70             if cause == cur_mode {
71                 break;
72             }
73         }
74     }
75 }
76
77 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
78 struct ToolBuild {
79     compiler: Compiler,
80     target: Interned<String>,
81     tool: &'static str,
82     path: &'static str,
83     mode: Mode,
84     is_ext_tool: bool,
85     extra_features: Vec<String>,
86 }
87
88 impl Step for ToolBuild {
89     type Output = Option<PathBuf>;
90
91     fn should_run(run: ShouldRun) -> ShouldRun {
92         run.never()
93     }
94
95     /// Build a tool in `src/tools`
96     ///
97     /// This will build the specified tool with the specified `host` compiler in
98     /// `stage` into the normal cargo output directory.
99     fn run(self, builder: &Builder) -> Option<PathBuf> {
100         let compiler = self.compiler;
101         let target = self.target;
102         let tool = self.tool;
103         let path = self.path;
104         let is_ext_tool = self.is_ext_tool;
105
106         match self.mode {
107             Mode::ToolStd => builder.ensure(compile::Std { compiler, target }),
108             Mode::ToolTest => builder.ensure(compile::Test { compiler, target }),
109             Mode::ToolRustc => builder.ensure(compile::Rustc { compiler, target }),
110             _ => panic!("unexpected Mode for tool build")
111         }
112
113         let mut cargo = prepare_tool_cargo(builder, compiler, self.mode, target, "build", path);
114         cargo.arg("--features").arg(self.extra_features.join(" "));
115
116         let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
117         builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
118         let mut duplicates = Vec::new();
119         let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
120             // Only care about big things like the RLS/Cargo for now
121             match tool {
122                 | "rls"
123                 | "cargo"
124                 | "clippy-driver"
125                 => {}
126
127                 _ => return,
128             }
129             let (id, features, filenames) = match msg {
130                 compile::CargoMessage::CompilerArtifact {
131                     package_id,
132                     features,
133                     filenames
134                 } => {
135                     (package_id, features, filenames)
136                 }
137                 _ => return,
138             };
139             let features = features.iter().map(|s| s.to_string()).collect::<Vec<_>>();
140
141             for path in filenames {
142                 let val = (tool, PathBuf::from(&*path), features.clone());
143                 // we're only interested in deduplicating rlibs for now
144                 if val.1.extension().and_then(|s| s.to_str()) != Some("rlib") {
145                     continue
146                 }
147
148                 // Don't worry about libs that turn out to be host dependencies
149                 // or build scripts, we only care about target dependencies that
150                 // are in `deps`.
151                 if let Some(maybe_target) = val.1
152                     .parent()                   // chop off file name
153                     .and_then(|p| p.parent())   // chop off `deps`
154                     .and_then(|p| p.parent())   // chop off `release`
155                     .and_then(|p| p.file_name())
156                     .and_then(|p| p.to_str())
157                 {
158                     if maybe_target != &*target {
159                         continue
160                     }
161                 }
162
163                 let mut artifacts = builder.tool_artifacts.borrow_mut();
164                 let prev_artifacts = artifacts
165                     .entry(target)
166                     .or_insert_with(Default::default);
167                 if let Some(prev) = prev_artifacts.get(&*id) {
168                     if prev.1 != val.1 {
169                         duplicates.push((
170                             id.to_string(),
171                             val,
172                             prev.clone(),
173                         ));
174                     }
175                     return
176                 }
177                 prev_artifacts.insert(id.to_string(), val);
178             }
179         });
180
181         if is_expected && duplicates.len() != 0 {
182             println!("duplicate artfacts found when compiling a tool, this \
183                       typically means that something was recompiled because \
184                       a transitive dependency has different features activated \
185                       than in a previous build:\n");
186             for (id, cur, prev) in duplicates {
187                 println!("  {}", id);
188                 println!("    `{}` enabled features {:?} at {:?}",
189                          cur.0, cur.2, cur.1);
190                 println!("    `{}` enabled features {:?} at {:?}",
191                          prev.0, prev.2, prev.1);
192             }
193             println!("");
194             panic!("tools should not compile multiple copies of the same crate");
195         }
196
197         builder.save_toolstate(tool, if is_expected {
198             ToolState::TestFail
199         } else {
200             ToolState::BuildFail
201         });
202
203         if !is_expected {
204             if !is_ext_tool {
205                 exit(1);
206             } else {
207                 return None;
208             }
209         } else {
210             let cargo_out = builder.cargo_out(compiler, self.mode, target)
211                 .join(exe(tool, &compiler.host));
212             let bin = builder.tools_dir(compiler).join(exe(tool, &compiler.host));
213             builder.copy(&cargo_out, &bin);
214             Some(bin)
215         }
216     }
217 }
218
219 pub fn prepare_tool_cargo(
220     builder: &Builder,
221     compiler: Compiler,
222     mode: Mode,
223     target: Interned<String>,
224     command: &'static str,
225     path: &'static str,
226 ) -> Command {
227     let mut cargo = builder.cargo(compiler, mode, target, command);
228     let dir = builder.src.join(path);
229     cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
230
231     // We don't want to build tools dynamically as they'll be running across
232     // stages and such and it's just easier if they're not dynamically linked.
233     cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
234
235     if let Some(dir) = builder.openssl_install_dir(target) {
236         cargo.env("OPENSSL_STATIC", "1");
237         cargo.env("OPENSSL_DIR", dir);
238         cargo.env("LIBZ_SYS_STATIC", "1");
239     }
240
241     // if tools are using lzma we want to force the build script to build its
242     // own copy
243     cargo.env("LZMA_API_STATIC", "1");
244
245     cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel);
246     cargo.env("CFG_VERSION", builder.rust_version());
247
248     let info = GitInfo::new(&builder.config, &dir);
249     if let Some(sha) = info.sha() {
250         cargo.env("CFG_COMMIT_HASH", sha);
251     }
252     if let Some(sha_short) = info.sha_short() {
253         cargo.env("CFG_SHORT_COMMIT_HASH", sha_short);
254     }
255     if let Some(date) = info.commit_date() {
256         cargo.env("CFG_COMMIT_DATE", date);
257     }
258     cargo
259 }
260
261 macro_rules! tool {
262     ($($name:ident, $path:expr, $tool_name:expr, $mode:expr $(,llvm_tools = $llvm:expr)*;)+) => {
263         #[derive(Copy, Clone)]
264         pub enum Tool {
265             $(
266                 $name,
267             )+
268         }
269
270         impl Tool {
271             pub fn get_mode(&self) -> Mode {
272                 let mode = match self {
273                     $(Tool::$name => $mode,)+
274                 };
275                 mode
276             }
277
278             /// Whether this tool requires LLVM to run
279             pub fn uses_llvm_tools(&self) -> bool {
280                 match self {
281                     $(Tool::$name => false $(|| $llvm)*,)+
282                 }
283             }
284         }
285
286         impl<'a> Builder<'a> {
287             pub fn tool_exe(&self, tool: Tool) -> PathBuf {
288                 let stage = self.tool_default_stage(tool);
289                 match tool {
290                     $(Tool::$name =>
291                         self.ensure($name {
292                             compiler: self.compiler(stage, self.config.build),
293                             target: self.config.build,
294                         }),
295                     )+
296                 }
297             }
298
299             pub fn tool_default_stage(&self, tool: Tool) -> u32 {
300                 // Compile the error-index in the same stage as rustdoc to avoid
301                 // recompiling rustdoc twice if we can. Otherwise compile
302                 // everything else in stage0 as there's no need to rebootstrap
303                 // everything.
304                 match tool {
305                     Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
306                     _ => 0,
307                 }
308             }
309         }
310
311         $(
312             #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
313         pub struct $name {
314             pub compiler: Compiler,
315             pub target: Interned<String>,
316         }
317
318         impl Step for $name {
319             type Output = PathBuf;
320
321             fn should_run(run: ShouldRun) -> ShouldRun {
322                 run.path($path)
323             }
324
325             fn make_run(run: RunConfig) {
326                 run.builder.ensure($name {
327                     compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
328                     target: run.target,
329                 });
330             }
331
332             fn run(self, builder: &Builder) -> PathBuf {
333                 builder.ensure(ToolBuild {
334                     compiler: self.compiler,
335                     target: self.target,
336                     tool: $tool_name,
337                     mode: $mode,
338                     path: $path,
339                     is_ext_tool: false,
340                     extra_features: Vec::new(),
341                 }).expect("expected to build -- essential tool")
342             }
343         }
344         )+
345     }
346 }
347
348 tool!(
349     Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolRustc;
350     ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;
351     UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen", Mode::ToolStd;
352     Tidy, "src/tools/tidy", "tidy", Mode::ToolStd;
353     Linkchecker, "src/tools/linkchecker", "linkchecker", Mode::ToolStd;
354     CargoTest, "src/tools/cargotest", "cargotest", Mode::ToolStd;
355     Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolTest, llvm_tools = true;
356     BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolStd;
357     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolStd;
358     RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolStd;
359     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolStd;
360 );
361
362 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
363 pub struct RemoteTestServer {
364     pub compiler: Compiler,
365     pub target: Interned<String>,
366 }
367
368 impl Step for RemoteTestServer {
369     type Output = PathBuf;
370
371     fn should_run(run: ShouldRun) -> ShouldRun {
372         run.path("src/tools/remote-test-server")
373     }
374
375     fn make_run(run: RunConfig) {
376         run.builder.ensure(RemoteTestServer {
377             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
378             target: run.target,
379         });
380     }
381
382     fn run(self, builder: &Builder) -> PathBuf {
383         builder.ensure(ToolBuild {
384             compiler: self.compiler,
385             target: self.target,
386             tool: "remote-test-server",
387             mode: Mode::ToolStd,
388             path: "src/tools/remote-test-server",
389             is_ext_tool: false,
390             extra_features: Vec::new(),
391         }).expect("expected to build -- essential tool")
392     }
393 }
394
395 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
396 pub struct Rustdoc {
397     pub host: Interned<String>,
398 }
399
400 impl Step for Rustdoc {
401     type Output = PathBuf;
402     const DEFAULT: bool = true;
403     const ONLY_HOSTS: bool = true;
404
405     fn should_run(run: ShouldRun) -> ShouldRun {
406         run.path("src/tools/rustdoc")
407     }
408
409     fn make_run(run: RunConfig) {
410         run.builder.ensure(Rustdoc {
411             host: run.host,
412         });
413     }
414
415     fn run(self, builder: &Builder) -> PathBuf {
416         let target_compiler = builder.compiler(builder.top_stage, self.host);
417         let target = target_compiler.host;
418         let build_compiler = if target_compiler.stage == 0 {
419             builder.compiler(0, builder.config.build)
420         } else if target_compiler.stage >= 2 {
421             // Past stage 2, we consider the compiler to be ABI-compatible and hence capable of
422             // building rustdoc itself.
423             builder.compiler(target_compiler.stage, builder.config.build)
424         } else {
425             // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
426             // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
427             // compilers, which isn't what we want.
428             builder.compiler(target_compiler.stage - 1, builder.config.build)
429         };
430
431         builder.ensure(compile::Rustc { compiler: build_compiler, target });
432         builder.ensure(compile::Rustc {
433             compiler: build_compiler,
434             target: builder.config.build,
435         });
436
437         let mut cargo = prepare_tool_cargo(builder,
438                                            build_compiler,
439                                            Mode::ToolRustc,
440                                            target,
441                                            "build",
442                                            "src/tools/rustdoc");
443
444         // Most tools don't get debuginfo, but rustdoc should.
445         cargo.env("RUSTC_DEBUGINFO", builder.config.rust_debuginfo.to_string())
446              .env("RUSTC_DEBUGINFO_LINES", builder.config.rust_debuginfo_lines.to_string());
447
448         let _folder = builder.fold_output(|| format!("stage{}-rustdoc", target_compiler.stage));
449         builder.info(&format!("Building rustdoc for stage{} ({})",
450             target_compiler.stage, target_compiler.host));
451         builder.run(&mut cargo);
452
453         // Cargo adds a number of paths to the dylib search path on windows, which results in
454         // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
455         // rustdoc a different name.
456         let tool_rustdoc = builder.cargo_out(build_compiler, Mode::ToolRustc, target)
457             .join(exe("rustdoc_tool_binary", &target_compiler.host));
458
459         // don't create a stage0-sysroot/bin directory.
460         if target_compiler.stage > 0 {
461             let sysroot = builder.sysroot(target_compiler);
462             let bindir = sysroot.join("bin");
463             t!(fs::create_dir_all(&bindir));
464             let bin_rustdoc = bindir.join(exe("rustdoc", &*target_compiler.host));
465             let _ = fs::remove_file(&bin_rustdoc);
466             builder.copy(&tool_rustdoc, &bin_rustdoc);
467             bin_rustdoc
468         } else {
469             tool_rustdoc
470         }
471     }
472 }
473
474 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
475 pub struct Cargo {
476     pub compiler: Compiler,
477     pub target: Interned<String>,
478 }
479
480 impl Step for Cargo {
481     type Output = PathBuf;
482     const DEFAULT: bool = true;
483     const ONLY_HOSTS: bool = true;
484
485     fn should_run(run: ShouldRun) -> ShouldRun {
486         let builder = run.builder;
487         run.path("src/tools/cargo").default_condition(builder.config.extended)
488     }
489
490     fn make_run(run: RunConfig) {
491         run.builder.ensure(Cargo {
492             compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
493             target: run.target,
494         });
495     }
496
497     fn run(self, builder: &Builder) -> PathBuf {
498         builder.ensure(native::Openssl {
499             target: self.target,
500         });
501         // Cargo depends on procedural macros, which requires a full host
502         // compiler to be available, so we need to depend on that.
503         builder.ensure(compile::Rustc {
504             compiler: self.compiler,
505             target: builder.config.build,
506         });
507         builder.ensure(ToolBuild {
508             compiler: self.compiler,
509             target: self.target,
510             tool: "cargo",
511             mode: Mode::ToolRustc,
512             path: "src/tools/cargo",
513             is_ext_tool: false,
514             extra_features: Vec::new(),
515         }).expect("expected to build -- essential tool")
516     }
517 }
518
519 macro_rules! tool_extended {
520     (($sel:ident, $builder:ident),
521        $($name:ident,
522        $toolstate:ident,
523        $path:expr,
524        $tool_name:expr,
525        $extra_deps:block;)+) => {
526         $(
527             #[derive(Debug, Clone, Hash, PartialEq, Eq)]
528         pub struct $name {
529             pub compiler: Compiler,
530             pub target: Interned<String>,
531             pub extra_features: Vec<String>,
532         }
533
534         impl Step for $name {
535             type Output = Option<PathBuf>;
536             const DEFAULT: bool = true;
537             const ONLY_HOSTS: bool = true;
538
539             fn should_run(run: ShouldRun) -> ShouldRun {
540                 let builder = run.builder;
541                 run.path($path).default_condition(builder.config.extended)
542             }
543
544             fn make_run(run: RunConfig) {
545                 run.builder.ensure($name {
546                     compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
547                     target: run.target,
548                     extra_features: Vec::new(),
549                 });
550             }
551
552             #[allow(unused_mut)]
553             fn run(mut $sel, $builder: &Builder) -> Option<PathBuf> {
554                 $extra_deps
555                 $builder.ensure(ToolBuild {
556                     compiler: $sel.compiler,
557                     target: $sel.target,
558                     tool: $tool_name,
559                     mode: Mode::ToolRustc,
560                     path: $path,
561                     extra_features: $sel.extra_features,
562                     is_ext_tool: true,
563                 })
564             }
565         }
566         )+
567     }
568 }
569
570 tool_extended!((self, builder),
571     Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", {};
572     Clippy, clippy, "src/tools/clippy", "clippy-driver", {
573         // Clippy depends on procedural macros (serde), which requires a full host
574         // compiler to be available, so we need to depend on that.
575         builder.ensure(compile::Rustc {
576             compiler: self.compiler,
577             target: builder.config.build,
578         });
579     };
580     Miri, miri, "src/tools/miri", "miri", {};
581     Rls, rls, "src/tools/rls", "rls", {
582         builder.ensure(native::Openssl {
583             target: self.target,
584         });
585         // RLS depends on procedural macros, which requires a full host
586         // compiler to be available, so we need to depend on that.
587         builder.ensure(compile::Rustc {
588             compiler: self.compiler,
589             target: builder.config.build,
590         });
591     };
592     Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", {};
593 );
594
595 impl<'a> Builder<'a> {
596     /// Get a `Command` which is ready to run `tool` in `stage` built for
597     /// `host`.
598     pub fn tool_cmd(&self, tool: Tool) -> Command {
599         let mut cmd = Command::new(self.tool_exe(tool));
600         let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
601         self.prepare_tool_cmd(compiler, tool, &mut cmd);
602         cmd
603     }
604
605     /// Prepares the `cmd` provided to be able to run the `compiler` provided.
606     ///
607     /// Notably this munges the dynamic library lookup path to point to the
608     /// right location to run `compiler`.
609     fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) {
610         let host = &compiler.host;
611         let mut lib_paths: Vec<PathBuf> = vec![
612             PathBuf::from(&self.sysroot_libdir(compiler, compiler.host)),
613             self.cargo_out(compiler, tool.get_mode(), *host).join("deps"),
614         ];
615
616         // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make
617         // mode) and that C compiler may need some extra PATH modification. Do
618         // so here.
619         if compiler.host.contains("msvc") {
620             let curpaths = env::var_os("PATH").unwrap_or_default();
621             let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>();
622             for &(ref k, ref v) in self.cc[&compiler.host].env() {
623                 if k != "PATH" {
624                     continue
625                 }
626                 for path in env::split_paths(v) {
627                     if !curpaths.contains(&path) {
628                         lib_paths.push(path);
629                     }
630                 }
631             }
632         }
633
634         // Add the llvm/bin directory to PATH since it contains lots of
635         // useful, platform-independent tools
636         if tool.uses_llvm_tools() {
637             if let Some(llvm_bin_path) = self.llvm_bin_path() {
638                 if host.contains("windows") {
639                     // On Windows, PATH and the dynamic library path are the same,
640                     // so we just add the LLVM bin path to lib_path
641                     lib_paths.push(llvm_bin_path);
642                 } else {
643                     let old_path = env::var_os("PATH").unwrap_or_default();
644                     let new_path = env::join_paths(iter::once(llvm_bin_path)
645                             .chain(env::split_paths(&old_path)))
646                         .expect("Could not add LLVM bin path to PATH");
647                     cmd.env("PATH", new_path);
648                 }
649             }
650         }
651
652         add_lib_path(lib_paths, cmd);
653     }
654
655     fn llvm_bin_path(&self) -> Option<PathBuf> {
656         if self.config.llvm_enabled && !self.config.dry_run {
657             let llvm_config = self.ensure(native::Llvm {
658                 target: self.config.build,
659                 emscripten: false,
660             });
661
662             // Add the llvm/bin directory to PATH since it contains lots of
663             // useful, platform-independent tools
664             let llvm_bin_path = llvm_config.parent()
665                 .expect("Expected llvm-config to be contained in directory");
666             assert!(llvm_bin_path.is_dir());
667             Some(llvm_bin_path.to_path_buf())
668         } else {
669             None
670         }
671     }
672 }