src/bootstrap/step.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Major workhorse of rustbuild, definition and dependencies between stages of
  12 //! the copmile.
  13 //!
  14 //! The primary purpose of this module is to define the various `Step`s of
  15 //! execution of the build. Each `Step` has a corresponding `Source` indicating
  16 //! what it's actually doing along with a number of dependencies which must be
  17 //! executed first.
  18 //!
  19 //! This module will take the CLI as input and calculate the steps required for
  20 //! the build requested, ensuring that all intermediate pieces are in place.
  21 //! Essentially this module is a `make`-replacement, but not as good.
  22
  23 use std::collections::HashSet;
  24
  25 use {Build, Compiler};
  26
  27 #[derive(Hash, Eq, PartialEq, Clone, Debug)]
  28 pub struct Step<'a> {
  29     pub src: Source<'a>,
  30     pub target: &'a str,
  31 }
  32
  33 /// Macro used to iterate over all targets that are recognized by the build
  34 /// system.
  35 ///
  36 /// Whenever a new step is added it will involve adding an entry here, updating
  37 /// the dependencies section below, and then adding an implementation of the
  38 /// step in `build/mod.rs`.
  39 ///
  40 /// This macro takes another macro as an argument and then calls that macro with
  41 /// all steps that the build system knows about.
  42 macro_rules! targets {
  43     ($m:ident) => {
  44         $m! {
  45             // Step representing building the stageN compiler. This is just the
  46             // compiler executable itself, not any of the support libraries
  47             (rustc, Rustc { stage: u32 }),
  48
  49             // Steps for the two main cargo builds. These are parameterized over
  50             // the compiler which is producing the artifact.
  51             (libstd, Libstd { compiler: Compiler<'a> }),
  52             (libtest, Libtest { compiler: Compiler<'a> }),
  53             (librustc, Librustc { compiler: Compiler<'a> }),
  54
  55             // Links the target produced by the compiler provided into the
  56             // host's directory also provided.
  57             (libstd_link, LibstdLink {
  58                 compiler: Compiler<'a>,
  59                 host: &'a str
  60             }),
  61             (libtest_link, LibtestLink {
  62                 compiler: Compiler<'a>,
  63                 host: &'a str
  64             }),
  65             (librustc_link, LibrustcLink {
  66                 compiler: Compiler<'a>,
  67                 host: &'a str
  68             }),
  69
  70             // Various tools that we can build as part of the build.
  71             (tool_linkchecker, ToolLinkchecker { stage: u32 }),
  72             (tool_rustbook, ToolRustbook { stage: u32 }),
  73             (tool_error_index, ToolErrorIndex { stage: u32 }),
  74             (tool_cargotest, ToolCargoTest { stage: u32 }),
  75             (tool_tidy, ToolTidy { stage: u32 }),
  76             (tool_compiletest, ToolCompiletest { stage: u32 }),
  77
  78             // Steps for long-running native builds. Ideally these wouldn't
  79             // actually exist and would be part of build scripts, but for now
  80             // these are here.
  81             //
  82             // There aren't really any parameters to this, but empty structs
  83             // with braces are unstable so we just pick something that works.
  84             (llvm, Llvm { _dummy: () }),
  85             (compiler_rt, CompilerRt { _dummy: () }),
  86             (test_helpers, TestHelpers { _dummy: () }),
  87             (debugger_scripts, DebuggerScripts { stage: u32 }),
  88
  89             // Steps for various pieces of documentation that we can generate,
  90             // the 'doc' step is just a pseudo target to depend on a bunch of
  91             // others.
  92             (doc, Doc { stage: u32 }),
  93             (doc_book, DocBook { stage: u32 }),
  94             (doc_nomicon, DocNomicon { stage: u32 }),
  95             (doc_style, DocStyle { stage: u32 }),
  96             (doc_standalone, DocStandalone { stage: u32 }),
  97             (doc_std, DocStd { stage: u32 }),
  98             (doc_test, DocTest { stage: u32 }),
  99             (doc_rustc, DocRustc { stage: u32 }),
 100             (doc_error_index, DocErrorIndex { stage: u32 }),
 101
 102             // Steps for running tests. The 'check' target is just a pseudo
 103             // target to depend on a bunch of others.
 104             (check, Check { stage: u32, compiler: Compiler<'a> }),
 105             (check_target, CheckTarget { stage: u32, compiler: Compiler<'a> }),
 106             (check_linkcheck, CheckLinkcheck { stage: u32 }),
 107             (check_cargotest, CheckCargoTest { stage: u32 }),
 108             (check_tidy, CheckTidy { stage: u32 }),
 109             (check_rpass, CheckRPass { compiler: Compiler<'a> }),
 110             (check_rpass_full, CheckRPassFull { compiler: Compiler<'a> }),
 111             (check_rpass_valgrind, CheckRPassValgrind { compiler: Compiler<'a> }),
 112             (check_rfail, CheckRFail { compiler: Compiler<'a> }),
 113             (check_rfail_full, CheckRFailFull { compiler: Compiler<'a> }),
 114             (check_cfail, CheckCFail { compiler: Compiler<'a> }),
 115             (check_cfail_full, CheckCFailFull { compiler: Compiler<'a> }),
 116             (check_pfail, CheckPFail { compiler: Compiler<'a> }),
 117             (check_pretty, CheckPretty { compiler: Compiler<'a> }),
 118             (check_pretty_rpass, CheckPrettyRPass { compiler: Compiler<'a> }),
 119             (check_pretty_rpass_full, CheckPrettyRPassFull { compiler: Compiler<'a> }),
 120             (check_pretty_rfail, CheckPrettyRFail { compiler: Compiler<'a> }),
 121             (check_pretty_rfail_full, CheckPrettyRFailFull { compiler: Compiler<'a> }),
 122             (check_pretty_rpass_valgrind, CheckPrettyRPassValgrind { compiler: Compiler<'a> }),
 123             (check_codegen, CheckCodegen { compiler: Compiler<'a> }),
 124             (check_codegen_units, CheckCodegenUnits { compiler: Compiler<'a> }),
 125             (check_incremental, CheckIncremental { compiler: Compiler<'a> }),
 126             (check_ui, CheckUi { compiler: Compiler<'a> }),
 127             (check_mir_opt, CheckMirOpt { compiler: Compiler<'a> }),
 128             (check_debuginfo, CheckDebuginfo { compiler: Compiler<'a> }),
 129             (check_rustdoc, CheckRustdoc { compiler: Compiler<'a> }),
 130             (check_docs, CheckDocs { compiler: Compiler<'a> }),
 131             (check_error_index, CheckErrorIndex { compiler: Compiler<'a> }),
 132             (check_rmake, CheckRMake { compiler: Compiler<'a> }),
 133             (check_crate_std, CheckCrateStd { compiler: Compiler<'a> }),
 134             (check_crate_test, CheckCrateTest { compiler: Compiler<'a> }),
 135             (check_crate_rustc, CheckCrateRustc { compiler: Compiler<'a> }),
 136
 137             // Distribution targets, creating tarballs
 138             (dist, Dist { stage: u32 }),
 139             (dist_docs, DistDocs { stage: u32 }),
 140             (dist_mingw, DistMingw { _dummy: () }),
 141             (dist_rustc, DistRustc { stage: u32 }),
 142             (dist_std, DistStd { compiler: Compiler<'a> }),
 143             (dist_src, DistSrc { _dummy: () }),
 144
 145             // Misc targets
 146             (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }),
 147         }
 148     }
 149 }
 150
 151 // Define the `Source` enum by iterating over all the steps and peeling out just
 152 // the types that we want to define.
 153
 154 macro_rules! item { ($a:item) => ($a) }
 155
 156 macro_rules! define_source {
 157     ($(($short:ident, $name:ident { $($args:tt)* }),)*) => {
 158         item! {
 159             #[derive(Hash, Eq, PartialEq, Clone, Debug)]
 160             pub enum Source<'a> {
 161                 $($name { $($args)* }),*
 162             }
 163         }
 164     }
 165 }
 166
 167 targets!(define_source);
 168
 169 /// Calculate a list of all steps described by `build`.
 170 ///
 171 /// This will inspect the flags passed in on the command line and use that to
 172 /// build up a list of steps to execute. These steps will then be transformed
 173 /// into a topologically sorted list which when executed left-to-right will
 174 /// correctly sequence the entire build.
 175 pub fn all(build: &Build) -> Vec<Step> {
 176     let mut ret = Vec::new();
 177     let mut all = HashSet::new();
 178     for target in top_level(build) {
 179         fill(build, &target, &mut ret, &mut all);
 180     }
 181     return ret;
 182
 183     fn fill<'a>(build: &'a Build,
 184                 target: &Step<'a>,
 185                 ret: &mut Vec<Step<'a>>,
 186                 set: &mut HashSet<Step<'a>>) {
 187         if set.insert(target.clone()) {
 188             for dep in target.deps(build) {
 189                 fill(build, &dep, ret, set);
 190             }
 191             ret.push(target.clone());
 192         }
 193     }
 194 }
 195
 196 /// Determines what top-level targets are requested as part of this build,
 197 /// returning them as a list.
 198 fn top_level(build: &Build) -> Vec<Step> {
 199     let mut targets = Vec::new();
 200     let stage = build.flags.stage.unwrap_or(2);
 201
 202     let host = Step {
 203         src: Source::Llvm { _dummy: () },
 204         target: build.flags.host.iter().next()
 205                      .unwrap_or(&build.config.build),
 206     };
 207     let target = Step {
 208         src: Source::Llvm { _dummy: () },
 209         target: build.flags.target.iter().next().map(|x| &x[..])
 210                      .unwrap_or(host.target)
 211     };
 212
 213     // First, try to find steps on the command line.
 214     add_steps(build, stage, &host, &target, &mut targets);
 215
 216     // If none are specified, then build everything.
 217     if targets.len() == 0 {
 218         let t = Step {
 219             src: Source::Llvm { _dummy: () },
 220             target: &build.config.build,
 221         };
 222         if build.config.docs {
 223           targets.push(t.doc(stage));
 224         }
 225         for host in build.config.host.iter() {
 226             if !build.flags.host.contains(host) {
 227                 continue
 228             }
 229             let host = t.target(host);
 230             if host.target == build.config.build {
 231                 targets.push(host.librustc(host.compiler(stage)));
 232             } else {
 233                 targets.push(host.librustc_link(t.compiler(stage), host.target));
 234             }
 235             for target in build.config.target.iter() {
 236                 if !build.flags.target.contains(target) {
 237                     continue
 238                 }
 239
 240                 if host.target == build.config.build {
 241                     targets.push(host.target(target)
 242                                      .libtest(host.compiler(stage)));
 243                 } else {
 244                     targets.push(host.target(target)
 245                                      .libtest_link(t.compiler(stage), host.target));
 246                 }
 247             }
 248         }
 249     }
 250
 251     return targets
 252
 253 }
 254
 255 fn add_steps<'a>(build: &'a Build,
 256                  stage: u32,
 257                  host: &Step<'a>,
 258                  target: &Step<'a>,
 259                  targets: &mut Vec<Step<'a>>) {
 260     struct Context<'a> {
 261         stage: u32,
 262         compiler: Compiler<'a>,
 263         _dummy: (),
 264         host: &'a str,
 265     }
 266     for step in build.flags.step.iter() {
 267
 268         // The macro below insists on hygienic access to all local variables, so
 269         // we shove them all in a struct and subvert hygiene by accessing struct
 270         // fields instead,
 271         let cx = Context {
 272             stage: stage,
 273             compiler: host.target(&build.config.build).compiler(stage),
 274             _dummy: (),
 275             host: host.target,
 276         };
 277         macro_rules! add_step {
 278             ($(($short:ident, $name:ident { $($arg:ident: $t:ty),* }),)*) => ({$(
 279                 let name = stringify!($short).replace("_", "-");
 280                 if &step[..] == &name[..] {
 281                     targets.push(target.$short($(cx.$arg),*));
 282                     continue
 283                 }
 284                 drop(name);
 285             )*})
 286         }
 287
 288         targets!(add_step);
 289
 290         panic!("unknown step: {}", step);
 291     }
 292 }
 293
 294 macro_rules! constructors {
 295     ($(($short:ident, $name:ident { $($arg:ident: $t:ty),* }),)*) => {$(
 296         fn $short(&self, $($arg: $t),*) -> Step<'a> {
 297             Step {
 298                 src: Source::$name { $($arg: $arg),* },
 299                 target: self.target,
 300             }
 301         }
 302     )*}
 303 }
 304
 305 impl<'a> Step<'a> {
 306     fn compiler(&self, stage: u32) -> Compiler<'a> {
 307         Compiler::new(stage, self.target)
 308     }
 309
 310     fn target(&self, target: &'a str) -> Step<'a> {
 311         Step { target: target, src: self.src.clone() }
 312     }
 313
 314     // Define ergonomic constructors for each step defined above so they can be
 315     // easily constructed.
 316     targets!(constructors);
 317
 318     /// Mapping of all dependencies for rustbuild.
 319     ///
 320     /// This function receives a step, the build that we're building for, and
 321     /// then returns a list of all the dependencies of that step.
 322     pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
 323         match self.src {
 324             Source::Rustc { stage: 0 } => {
 325                 Vec::new()
 326             }
 327             Source::Rustc { stage } => {
 328                 let compiler = Compiler::new(stage - 1, &build.config.build);
 329                 vec![self.librustc(compiler)]
 330             }
 331             Source::Librustc { compiler } => {
 332                 vec![self.libtest(compiler), self.llvm(())]
 333             }
 334             Source::Libtest { compiler } => {
 335                 vec![self.libstd(compiler)]
 336             }
 337             Source::Libstd { compiler } => {
 338                 vec![self.compiler_rt(()),
 339                      self.rustc(compiler.stage).target(compiler.host)]
 340             }
 341             Source::LibrustcLink { compiler, host } => {
 342                 vec![self.librustc(compiler),
 343                      self.libtest_link(compiler, host)]
 344             }
 345             Source::LibtestLink { compiler, host } => {
 346                 vec![self.libtest(compiler), self.libstd_link(compiler, host)]
 347             }
 348             Source::LibstdLink { compiler, host } => {
 349                 vec![self.libstd(compiler),
 350                      self.target(host).rustc(compiler.stage)]
 351             }
 352             Source::CompilerRt { _dummy } => Vec::new(),
 353             Source::Llvm { _dummy } => Vec::new(),
 354             Source::TestHelpers { _dummy } => Vec::new(),
 355             Source::DebuggerScripts { stage: _ } => Vec::new(),
 356
 357             // Note that all doc targets depend on artifacts from the build
 358             // architecture, not the target (which is where we're generating
 359             // docs into).
 360             Source::DocStd { stage } => {
 361                 let compiler = self.target(&build.config.build).compiler(stage);
 362                 vec![self.libstd(compiler)]
 363             }
 364             Source::DocTest { stage } => {
 365                 let compiler = self.target(&build.config.build).compiler(stage);
 366                 vec![self.libtest(compiler)]
 367             }
 368             Source::DocBook { stage } |
 369             Source::DocNomicon { stage } |
 370             Source::DocStyle { stage } => {
 371                 vec![self.target(&build.config.build).tool_rustbook(stage)]
 372             }
 373             Source::DocErrorIndex { stage } => {
 374                 vec![self.target(&build.config.build).tool_error_index(stage)]
 375             }
 376             Source::DocStandalone { stage } => {
 377                 vec![self.target(&build.config.build).rustc(stage)]
 378             }
 379             Source::DocRustc { stage } => {
 380                 vec![self.doc_test(stage)]
 381             }
 382             Source::Doc { stage } => {
 383                 let mut deps = vec![
 384                     self.doc_book(stage), self.doc_nomicon(stage),
 385                     self.doc_style(stage), self.doc_standalone(stage),
 386                     self.doc_std(stage),
 387                     self.doc_error_index(stage),
 388                 ];
 389
 390                 if build.config.compiler_docs {
 391                     deps.push(self.doc_rustc(stage));
 392                 }
 393
 394                 deps
 395             }
 396             Source::Check { stage, compiler } => {
 397                 // Check is just a pseudo step which means check all targets,
 398                 // so just depend on checking all targets.
 399                 build.config.target.iter().map(|t| {
 400                     self.target(t).check_target(stage, compiler)
 401                 }).collect()
 402             }
 403             Source::CheckTarget { stage, compiler } => {
 404                 // CheckTarget here means run all possible test suites for this
 405                 // target. Most of the time, however, we can't actually run
 406                 // anything if we're not the build triple as we could be cross
 407                 // compiling.
 408                 //
 409                 // As a result, the base set of targets here is quite stripped
 410                 // down from the standard set of targets. These suites have
 411                 // their own internal logic to run in cross-compiled situations
 412                 // if they'll run at all. For example compiletest knows that
 413                 // when testing Android targets we ship artifacts to the
 414                 // emulator.
 415                 //
 416                 // When in doubt the rule of thumb for adding to this list is
 417                 // "should this test suite run on the android bot?"
 418                 let mut base = vec![
 419                     self.check_rpass(compiler),
 420                     self.check_rfail(compiler),
 421                     self.check_crate_std(compiler),
 422                     self.check_crate_test(compiler),
 423                     self.check_debuginfo(compiler),
 424                     self.dist(stage),
 425                 ];
 426
 427                 // If we're testing the build triple, then we know we can
 428                 // actually run binaries and such, so we run all possible tests
 429                 // that we know about.
 430                 if self.target == build.config.build {
 431                     base.extend(vec![
 432                         // docs-related
 433                         self.check_docs(compiler),
 434                         self.check_error_index(compiler),
 435                         self.check_rustdoc(compiler),
 436
 437                         // UI-related
 438                         self.check_cfail(compiler),
 439                         self.check_pfail(compiler),
 440                         self.check_ui(compiler),
 441
 442                         // codegen-related
 443                         self.check_incremental(compiler),
 444                         self.check_codegen(compiler),
 445                         self.check_codegen_units(compiler),
 446
 447                         // misc compiletest-test suites
 448                         self.check_rpass_full(compiler),
 449                         self.check_rfail_full(compiler),
 450                         self.check_cfail_full(compiler),
 451                         self.check_pretty_rpass_full(compiler),
 452                         self.check_pretty_rfail_full(compiler),
 453                         self.check_rpass_valgrind(compiler),
 454                         self.check_rmake(compiler),
 455                         self.check_mir_opt(compiler),
 456
 457                         // crates
 458                         self.check_crate_rustc(compiler),
 459
 460                         // pretty
 461                         self.check_pretty(compiler),
 462                         self.check_pretty_rpass(compiler),
 463                         self.check_pretty_rfail(compiler),
 464                         self.check_pretty_rpass_valgrind(compiler),
 465
 466                         // misc
 467                         self.check_linkcheck(stage),
 468                         self.check_tidy(stage),
 469                     ]);
 470                 }
 471                 return base
 472             }
 473             Source::CheckLinkcheck { stage } => {
 474                 vec![self.tool_linkchecker(stage), self.doc(stage)]
 475             }
 476             Source::CheckCargoTest { stage } => {
 477                 vec![self.tool_cargotest(stage),
 478                      self.librustc(self.compiler(stage))]
 479             }
 480             Source::CheckTidy { stage } => {
 481                 vec![self.tool_tidy(stage)]
 482             }
 483             Source::CheckMirOpt { compiler} |
 484             Source::CheckPrettyRPass { compiler } |
 485             Source::CheckPrettyRFail { compiler } |
 486             Source::CheckRFail { compiler } |
 487             Source::CheckPFail { compiler } |
 488             Source::CheckCodegen { compiler } |
 489             Source::CheckCodegenUnits { compiler } |
 490             Source::CheckIncremental { compiler } |
 491             Source::CheckUi { compiler } |
 492             Source::CheckRustdoc { compiler } |
 493             Source::CheckPretty { compiler } |
 494             Source::CheckCFail { compiler } |
 495             Source::CheckRPassValgrind { compiler } |
 496             Source::CheckRPass { compiler } => {
 497                 let mut base = vec![
 498                     self.libtest(compiler),
 499                     self.target(compiler.host).tool_compiletest(compiler.stage),
 500                     self.test_helpers(()),
 501                 ];
 502                 if self.target.contains("android") {
 503                     base.push(self.android_copy_libs(compiler));
 504                 }
 505                 base
 506             }
 507             Source::CheckDebuginfo { compiler } => {
 508                 vec![
 509                     self.libtest(compiler),
 510                     self.target(compiler.host).tool_compiletest(compiler.stage),
 511                     self.test_helpers(()),
 512                     self.debugger_scripts(compiler.stage),
 513                 ]
 514             }
 515             Source::CheckRPassFull { compiler } |
 516             Source::CheckRFailFull { compiler } |
 517             Source::CheckCFailFull { compiler } |
 518             Source::CheckPrettyRPassFull { compiler } |
 519             Source::CheckPrettyRFailFull { compiler } |
 520             Source::CheckPrettyRPassValgrind { compiler } |
 521             Source::CheckRMake { compiler } => {
 522                 vec![self.librustc(compiler),
 523                      self.target(compiler.host).tool_compiletest(compiler.stage)]
 524             }
 525             Source::CheckDocs { compiler } => {
 526                 vec![self.libstd(compiler)]
 527             }
 528             Source::CheckErrorIndex { compiler } => {
 529                 vec![self.libstd(compiler),
 530                      self.target(compiler.host).tool_error_index(compiler.stage)]
 531             }
 532             Source::CheckCrateStd { compiler } => {
 533                 vec![self.libtest(compiler)]
 534             }
 535             Source::CheckCrateTest { compiler } => {
 536                 vec![self.libtest(compiler)]
 537             }
 538             Source::CheckCrateRustc { compiler } => {
 539                 vec![self.libtest(compiler)]
 540             }
 541
 542             Source::ToolLinkchecker { stage } |
 543             Source::ToolTidy { stage } => {
 544                 vec![self.libstd(self.compiler(stage))]
 545             }
 546             Source::ToolErrorIndex { stage } |
 547             Source::ToolRustbook { stage } => {
 548                 vec![self.librustc(self.compiler(stage))]
 549             }
 550             Source::ToolCargoTest { stage } => {
 551                 vec![self.libstd(self.compiler(stage))]
 552             }
 553             Source::ToolCompiletest { stage } => {
 554                 vec![self.libtest(self.compiler(stage))]
 555             }
 556
 557             Source::DistDocs { stage } => vec![self.doc(stage)],
 558             Source::DistMingw { _dummy: _ } => Vec::new(),
 559             Source::DistRustc { stage } => {
 560                 vec![self.rustc(stage)]
 561             }
 562             Source::DistStd { compiler } => {
 563                 // We want to package up as many target libraries as possible
 564                 // for the `rust-std` package, so if this is a host target we
 565                 // depend on librustc and otherwise we just depend on libtest.
 566                 if build.config.host.iter().any(|t| t == self.target) {
 567                     vec![self.librustc(compiler)]
 568                 } else {
 569                     vec![self.libtest(compiler)]
 570                 }
 571             }
 572             Source::DistSrc { _dummy: _ } => Vec::new(),
 573
 574             Source::Dist { stage } => {
 575                 let mut base = Vec::new();
 576
 577                 for host in build.config.host.iter() {
 578                     let host = self.target(host);
 579                     base.push(host.dist_src(()));
 580                     base.push(host.dist_rustc(stage));
 581                     if host.target.contains("windows-gnu") {
 582                         base.push(host.dist_mingw(()));
 583                     }
 584
 585                     let compiler = self.compiler(stage);
 586                     for target in build.config.target.iter() {
 587                         let target = self.target(target);
 588                         if build.config.docs {
 589                             base.push(target.dist_docs(stage));
 590                         }
 591                         base.push(target.dist_std(compiler));
 592                     }
 593                 }
 594                 return base
 595             }
 596
 597             Source::AndroidCopyLibs { compiler } => {
 598                 vec![self.libtest(compiler)]
 599             }
 600         }
 601     }
 602 }