src/tools/compiletest/src/runtest.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use common::Config;
  12 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
  13 use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
  14 use common::{Incremental, RunMake, Ui, MirOpt};
  15 use errors::{self, ErrorKind, Error};
  16 use json;
  17 use header::TestProps;
  18 use header;
  19 use procsrv;
  20 use test::TestPaths;
  21 use uidiff;
  22 use util::logv;
  23
  24 use std::env;
  25 use std::collections::HashSet;
  26 use std::fmt;
  27 use std::fs::{self, File};
  28 use std::io::{self, BufReader};
  29 use std::io::prelude::*;
  30 use std::net::TcpStream;
  31 use std::path::{Path, PathBuf};
  32 use std::process::{Command, Output, ExitStatus};
  33 use std::str;
  34
  35 pub fn run(config: Config, testpaths: &TestPaths) {
  36     match &*config.target {
  37
  38         "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
  39             if !config.adb_device_status {
  40                 panic!("android device not available");
  41             }
  42         }
  43
  44         _=> { }
  45     }
  46
  47     if config.verbose {
  48         // We're going to be dumping a lot of info. Start on a new line.
  49         print!("\n\n");
  50     }
  51     debug!("running {:?}", testpaths.file.display());
  52     let base_props = TestProps::from_file(&testpaths.file);
  53
  54     let base_cx = TestCx { config: &config,
  55                            props: &base_props,
  56                            testpaths: testpaths,
  57                            revision: None };
  58     base_cx.init_all();
  59
  60     if base_props.revisions.is_empty() {
  61         base_cx.run_revision()
  62     } else {
  63         for revision in &base_props.revisions {
  64             let mut revision_props = base_props.clone();
  65             revision_props.load_from(&testpaths.file, Some(&revision));
  66             let rev_cx = TestCx {
  67                 config: &config,
  68                 props: &revision_props,
  69                 testpaths: testpaths,
  70                 revision: Some(revision)
  71             };
  72             rev_cx.run_revision();
  73         }
  74     }
  75
  76     base_cx.complete_all();
  77 }
  78
  79 struct TestCx<'test> {
  80     config: &'test Config,
  81     props: &'test TestProps,
  82     testpaths: &'test TestPaths,
  83     revision: Option<&'test str>
  84 }
  85
  86 struct DebuggerCommands {
  87     commands: Vec<String>,
  88     check_lines: Vec<String>,
  89     breakpoint_lines: Vec<usize>,
  90 }
  91
  92 impl<'test> TestCx<'test> {
  93     /// invoked once before any revisions have been processed
  94     fn init_all(&self) {
  95         assert!(self.revision.is_none(), "init_all invoked for a revision");
  96         match self.config.mode {
  97             Incremental => self.init_incremental_test(),
  98             _ => { }
  99         }
 100     }
 101
 102     /// Code executed for each revision in turn (or, if there are no
 103     /// revisions, exactly once, with revision == None).
 104     fn run_revision(&self) {
 105         match self.config.mode {
 106             CompileFail => self.run_cfail_test(),
 107             ParseFail => self.run_cfail_test(),
 108             RunFail => self.run_rfail_test(),
 109             RunPass => self.run_rpass_test(),
 110             RunPassValgrind => self.run_valgrind_test(),
 111             Pretty => self.run_pretty_test(),
 112             DebugInfoGdb => self.run_debuginfo_gdb_test(),
 113             DebugInfoLldb => self.run_debuginfo_lldb_test(),
 114             Codegen => self.run_codegen_test(),
 115             Rustdoc => self.run_rustdoc_test(),
 116             CodegenUnits => self.run_codegen_units_test(),
 117             Incremental => self.run_incremental_test(),
 118             RunMake => self.run_rmake_test(),
 119             Ui => self.run_ui_test(),
 120             MirOpt => self.run_mir_opt_test(),
 121         }
 122     }
 123
 124     /// Invoked after all revisions have executed.
 125     fn complete_all(&self) {
 126         assert!(self.revision.is_none(), "init_all invoked for a revision");
 127     }
 128
 129     fn run_cfail_test(&self) {
 130         let proc_res = self.compile_test();
 131
 132         if proc_res.status.success() {
 133             self.fatal_proc_rec(
 134                 &format!("{} test compiled successfully!", self.config.mode)[..],
 135                 &proc_res);
 136         }
 137
 138         self.check_correct_failure_status(&proc_res);
 139
 140         let output_to_check = self.get_output(&proc_res);
 141         let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
 142         if !expected_errors.is_empty() {
 143             if !self.props.error_patterns.is_empty() {
 144                 self.fatal("both error pattern and expected errors specified");
 145             }
 146             self.check_expected_errors(expected_errors, &proc_res);
 147         } else {
 148             self.check_error_patterns(&output_to_check, &proc_res);
 149         }
 150         self.check_no_compiler_crash(&proc_res);
 151         self.check_forbid_output(&output_to_check, &proc_res);
 152     }
 153
 154     fn run_rfail_test(&self) {
 155         let proc_res = self.compile_test();
 156
 157         if !proc_res.status.success() {
 158             self.fatal_proc_rec("compilation failed!", &proc_res);
 159         }
 160
 161         let proc_res = self.exec_compiled_test();
 162
 163         // The value our Makefile configures valgrind to return on failure
 164         const VALGRIND_ERR: i32 = 100;
 165         if proc_res.status.code() == Some(VALGRIND_ERR) {
 166             self.fatal_proc_rec("run-fail test isn't valgrind-clean!", &proc_res);
 167         }
 168
 169         let output_to_check = self.get_output(&proc_res);
 170         self.check_correct_failure_status(&proc_res);
 171         self.check_error_patterns(&output_to_check, &proc_res);
 172     }
 173
 174     fn get_output(&self, proc_res: &ProcRes) -> String {
 175         if self.props.check_stdout {
 176             format!("{}{}", proc_res.stdout, proc_res.stderr)
 177         } else {
 178             proc_res.stderr.clone()
 179         }
 180     }
 181
 182     fn check_correct_failure_status(&self, proc_res: &ProcRes) {
 183         // The value the rust runtime returns on failure
 184         const RUST_ERR: i32 = 101;
 185         if proc_res.status.code() != Some(RUST_ERR) {
 186             self.fatal_proc_rec(
 187                 &format!("failure produced the wrong error: {}",
 188                          proc_res.status),
 189                 proc_res);
 190         }
 191     }
 192
 193     fn run_rpass_test(&self) {
 194         let proc_res = self.compile_test();
 195
 196         if !proc_res.status.success() {
 197             self.fatal_proc_rec("compilation failed!", &proc_res);
 198         }
 199
 200         let proc_res = self.exec_compiled_test();
 201
 202         if !proc_res.status.success() {
 203             self.fatal_proc_rec("test run failed!", &proc_res);
 204         }
 205     }
 206
 207     fn run_valgrind_test(&self) {
 208         assert!(self.revision.is_none(), "revisions not relevant here");
 209
 210         if self.config.valgrind_path.is_none() {
 211             assert!(!self.config.force_valgrind);
 212             return self.run_rpass_test();
 213         }
 214
 215         let mut proc_res = self.compile_test();
 216
 217         if !proc_res.status.success() {
 218             self.fatal_proc_rec("compilation failed!", &proc_res);
 219         }
 220
 221         let mut new_config = self.config.clone();
 222         new_config.runtool = new_config.valgrind_path.clone();
 223         let new_cx = TestCx { config: &new_config, ..*self };
 224         proc_res = new_cx.exec_compiled_test();
 225
 226         if !proc_res.status.success() {
 227             self.fatal_proc_rec("test run failed!", &proc_res);
 228         }
 229     }
 230
 231     fn run_pretty_test(&self) {
 232         if self.props.pp_exact.is_some() {
 233             logv(self.config, "testing for exact pretty-printing".to_owned());
 234         } else {
 235             logv(self.config, "testing for converging pretty-printing".to_owned());
 236         }
 237
 238         let rounds = match self.props.pp_exact { Some(_) => 1, None => 2 };
 239
 240         let mut src = String::new();
 241         File::open(&self.testpaths.file).unwrap().read_to_string(&mut src).unwrap();
 242         let mut srcs = vec!(src);
 243
 244         let mut round = 0;
 245         while round < rounds {
 246             logv(self.config, format!("pretty-printing round {} revision {:?}",
 247                                       round, self.revision));
 248             let proc_res = self.print_source(srcs[round].to_owned(), &self.props.pretty_mode);
 249
 250             if !proc_res.status.success() {
 251                 self.fatal_proc_rec(&format!("pretty-printing failed in round {} revision {:?}",
 252                                              round, self.revision),
 253                                     &proc_res);
 254             }
 255
 256             let ProcRes{ stdout, .. } = proc_res;
 257             srcs.push(stdout);
 258             round += 1;
 259         }
 260
 261         let mut expected = match self.props.pp_exact {
 262             Some(ref file) => {
 263                 let filepath = self.testpaths.file.parent().unwrap().join(file);
 264                 let mut s = String::new();
 265                 File::open(&filepath).unwrap().read_to_string(&mut s).unwrap();
 266                 s
 267             }
 268             None => { srcs[srcs.len() - 2].clone() }
 269         };
 270         let mut actual = srcs[srcs.len() - 1].clone();
 271
 272         if self.props.pp_exact.is_some() {
 273             // Now we have to care about line endings
 274             let cr = "\r".to_owned();
 275             actual = actual.replace(&cr, "").to_owned();
 276             expected = expected.replace(&cr, "").to_owned();
 277         }
 278
 279         self.compare_source(&expected, &actual);
 280
 281         // If we're only making sure that the output matches then just stop here
 282         if self.props.pretty_compare_only { return; }
 283
 284         // Finally, let's make sure it actually appears to remain valid code
 285         let proc_res = self.typecheck_source(actual);
 286         if !proc_res.status.success() {
 287             self.fatal_proc_rec("pretty-printed source does not typecheck", &proc_res);
 288         }
 289
 290         if !self.props.pretty_expanded { return }
 291
 292         // additionally, run `--pretty expanded` and try to build it.
 293         let proc_res = self.print_source(srcs[round].clone(), "expanded");
 294         if !proc_res.status.success() {
 295             self.fatal_proc_rec("pretty-printing (expanded) failed", &proc_res);
 296         }
 297
 298         let ProcRes{ stdout: expanded_src, .. } = proc_res;
 299         let proc_res = self.typecheck_source(expanded_src);
 300         if !proc_res.status.success() {
 301             self.fatal_proc_rec(
 302                 "pretty-printed source (expanded) does not typecheck",
 303                 &proc_res);
 304         }
 305     }
 306
 307     fn print_source(&self,
 308                     src: String,
 309                     pretty_type: &str)
 310                     -> ProcRes {
 311         let aux_dir = self.aux_output_dir_name();
 312         self.compose_and_run(self.make_pp_args(pretty_type.to_owned()),
 313                              self.props.exec_env.clone(),
 314                              self.config.compile_lib_path.to_str().unwrap(),
 315                              Some(aux_dir.to_str().unwrap()),
 316                              Some(src))
 317     }
 318
 319     fn make_pp_args(&self,
 320                     pretty_type: String)
 321                     -> ProcArgs {
 322         let aux_dir = self.aux_output_dir_name();
 323         // FIXME (#9639): This needs to handle non-utf8 paths
 324         let mut args = vec!("-".to_owned(),
 325                             "-Zunstable-options".to_owned(),
 326                             "--unpretty".to_owned(),
 327                             pretty_type,
 328                             format!("--target={}", self.config.target),
 329                             "-L".to_owned(),
 330                             aux_dir.to_str().unwrap().to_owned());
 331         args.extend(self.split_maybe_args(&self.config.target_rustcflags));
 332         args.extend(self.props.compile_flags.iter().cloned());
 333         return ProcArgs {
 334             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
 335             args: args,
 336         };
 337     }
 338
 339     fn compare_source(&self,
 340                       expected: &str,
 341                       actual: &str) {
 342         if expected != actual {
 343             self.error("pretty-printed source does not match expected source");
 344             println!("\n\
 345 expected:\n\
 346 ------------------------------------------\n\
 347 {}\n\
 348 ------------------------------------------\n\
 349 actual:\n\
 350 ------------------------------------------\n\
 351 {}\n\
 352 ------------------------------------------\n\
 353 \n",
 354                      expected, actual);
 355             panic!();
 356         }
 357     }
 358
 359     fn typecheck_source(&self, src: String) -> ProcRes {
 360         let args = self.make_typecheck_args();
 361         self.compose_and_run_compiler(args, Some(src))
 362     }
 363
 364     fn make_typecheck_args(&self) -> ProcArgs {
 365         let aux_dir = self.aux_output_dir_name();
 366         let target = if self.props.force_host {
 367             &*self.config.host
 368         } else {
 369             &*self.config.target
 370         };
 371
 372         let out_dir = self.output_base_name().with_extension("pretty-out");
 373         let _ = fs::remove_dir_all(&out_dir);
 374         self.create_dir_racy(&out_dir);
 375
 376         // FIXME (#9639): This needs to handle non-utf8 paths
 377         let mut args = vec!("-".to_owned(),
 378                             "-Zno-trans".to_owned(),
 379                             "--out-dir".to_owned(),
 380                             out_dir.to_str().unwrap().to_owned(),
 381                             format!("--target={}", target),
 382                             "-L".to_owned(),
 383                             self.config.build_base.to_str().unwrap().to_owned(),
 384                             "-L".to_owned(),
 385                             aux_dir.to_str().unwrap().to_owned());
 386         if let Some(revision) = self.revision {
 387             args.extend(vec![
 388                 format!("--cfg"),
 389                 format!("{}", revision),
 390             ]);
 391         }
 392         args.extend(self.split_maybe_args(&self.config.target_rustcflags));
 393         args.extend(self.props.compile_flags.iter().cloned());
 394         // FIXME (#9639): This needs to handle non-utf8 paths
 395         return ProcArgs {
 396             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
 397             args: args,
 398         };
 399     }
 400
 401     fn run_debuginfo_gdb_test(&self) {
 402         assert!(self.revision.is_none(), "revisions not relevant here");
 403
 404         let config = Config {
 405             target_rustcflags: self.cleanup_debug_info_options(&self.config.target_rustcflags),
 406             host_rustcflags: self.cleanup_debug_info_options(&self.config.host_rustcflags),
 407             .. self.config.clone()
 408         };
 409
 410         let test_cx = TestCx {
 411             config: &config,
 412             ..*self
 413         };
 414
 415         test_cx.run_debuginfo_gdb_test_no_opt();
 416     }
 417
 418     fn run_debuginfo_gdb_test_no_opt(&self) {
 419         let DebuggerCommands {
 420             commands,
 421             check_lines,
 422             breakpoint_lines
 423         } = self.parse_debugger_commands("gdb");
 424         let mut cmds = commands.join("\n");
 425
 426         // compile test file (it should have 'compile-flags:-g' in the header)
 427         let compiler_run_result = self.compile_test();
 428         if !compiler_run_result.status.success() {
 429             self.fatal_proc_rec("compilation failed!", &compiler_run_result);
 430         }
 431
 432         let exe_file = self.make_exe_name();
 433
 434         let debugger_run_result;
 435         match &*self.config.target {
 436             "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
 437
 438                 cmds = cmds.replace("run", "continue");
 439
 440                 let tool_path = match self.config.android_cross_path.to_str() {
 441                     Some(x) => x.to_owned(),
 442                     None => self.fatal("cannot find android cross path")
 443                 };
 444
 445                 // write debugger script
 446                 let mut script_str = String::with_capacity(2048);
 447                 script_str.push_str(&format!("set charset {}\n", Self::charset()));
 448                 script_str.push_str(&format!("set sysroot {}\n", tool_path));
 449                 script_str.push_str(&format!("file {}\n", exe_file.to_str().unwrap()));
 450                 script_str.push_str("target remote :5039\n");
 451                 script_str.push_str(&format!("set solib-search-path \
 452                                               ./{}/stage2/lib/rustlib/{}/lib/\n",
 453                                              self.config.host, self.config.target));
 454                 for line in &breakpoint_lines {
 455                     script_str.push_str(&format!("break {:?}:{}\n",
 456                                                  self.testpaths.file.file_name()
 457                                                  .unwrap()
 458                                                  .to_string_lossy(),
 459                                                  *line)[..]);
 460                 }
 461                 script_str.push_str(&cmds);
 462                 script_str.push_str("\nquit\n");
 463
 464                 debug!("script_str = {}", script_str);
 465                 self.dump_output_file(&script_str, "debugger.script");
 466
 467
 468                 procsrv::run("",
 469                              &self.config.adb_path,
 470                              None,
 471                              &[
 472                                  "push".to_owned(),
 473                                  exe_file.to_str().unwrap().to_owned(),
 474                                  self.config.adb_test_dir.clone()
 475                              ],
 476                              vec!(("".to_owned(), "".to_owned())),
 477                              Some("".to_owned()))
 478                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
 479
 480                 procsrv::run("",
 481                              &self.config.adb_path,
 482                              None,
 483                              &[
 484                                  "forward".to_owned(),
 485                                  "tcp:5039".to_owned(),
 486                                  "tcp:5039".to_owned()
 487                              ],
 488                              vec!(("".to_owned(), "".to_owned())),
 489                              Some("".to_owned()))
 490                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
 491
 492                 let adb_arg = format!("export LD_LIBRARY_PATH={}; \
 493                                        gdbserver{} :5039 {}/{}",
 494                                       self.config.adb_test_dir.clone(),
 495                                       if self.config.target.contains("aarch64")
 496                                       {"64"} else {""},
 497                                       self.config.adb_test_dir.clone(),
 498                                       exe_file.file_name().unwrap().to_str()
 499                                       .unwrap());
 500
 501                 let mut process = procsrv::run_background("",
 502                                                           &self.config.adb_path
 503                                                           ,
 504                                                           None,
 505                                                           &[
 506                                                               "shell".to_owned(),
 507                                                               adb_arg.clone()
 508                                                           ],
 509                                                           vec!(("".to_owned(),
 510                                                                 "".to_owned())),
 511                                                           Some("".to_owned()))
 512                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
 513                 loop {
 514                     //waiting 1 second for gdbserver start
 515                     ::std::thread::sleep(::std::time::Duration::new(1,0));
 516                     if TcpStream::connect("127.0.0.1:5039").is_ok() {
 517                         break
 518                     }
 519                 }
 520
 521                 let debugger_script = self.make_out_name("debugger.script");
 522                 // FIXME (#9639): This needs to handle non-utf8 paths
 523                 let debugger_opts =
 524                     vec!("-quiet".to_owned(),
 525                          "-batch".to_owned(),
 526                          "-nx".to_owned(),
 527                          format!("-command={}", debugger_script.to_str().unwrap()));
 528
 529                 let mut gdb_path = tool_path;
 530                 gdb_path.push_str(&format!("/bin/{}-gdb", self.config.target));
 531                 let procsrv::Result {
 532                     out,
 533                     err,
 534                     status
 535                 } = procsrv::run("",
 536                                  &gdb_path,
 537                                  None,
 538                                  &debugger_opts,
 539                                  vec!(("".to_owned(), "".to_owned())),
 540                                  None)
 541                     .expect(&format!("failed to exec `{:?}`", gdb_path));
 542                 let cmdline = {
 543                     let cmdline = self.make_cmdline("",
 544                                                     &format!("{}-gdb", self.config.target),
 545                                                     &debugger_opts);
 546                     logv(self.config, format!("executing {}", cmdline));
 547                     cmdline
 548                 };
 549
 550                 debugger_run_result = ProcRes {
 551                     status: Status::Normal(status),
 552                     stdout: out,
 553                     stderr: err,
 554                     cmdline: cmdline
 555                 };
 556                 if process.kill().is_err() {
 557                     println!("Adb process is already finished.");
 558                 }
 559             }
 560
 561             _=> {
 562                 let rust_src_root = self.find_rust_src_root()
 563                                         .expect("Could not find Rust source root");
 564                 let rust_pp_module_rel_path = Path::new("./src/etc");
 565                 let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
 566                                                            .to_str()
 567                                                            .unwrap()
 568                                                            .to_owned();
 569                 // write debugger script
 570                 let mut script_str = String::with_capacity(2048);
 571                 script_str.push_str(&format!("set charset {}\n", Self::charset()));
 572                 script_str.push_str("show version\n");
 573
 574                 match self.config.gdb_version {
 575                     Some(ref version) => {
 576                         println!("NOTE: compiletest thinks it is using GDB version {}",
 577                                  version);
 578
 579                         if header::gdb_version_to_int(version) >
 580                             header::gdb_version_to_int("7.4") {
 581                                 // Add the directory containing the pretty printers to
 582                                 // GDB's script auto loading safe path
 583                                 script_str.push_str(
 584                                     &format!("add-auto-load-safe-path {}\n",
 585                                              rust_pp_module_abs_path.replace(r"\", r"\\"))
 586                                 );
 587                             }
 588                     }
 589                     _ => {
 590                         println!("NOTE: compiletest does not know which version of \
 591                                   GDB it is using");
 592                     }
 593                 }
 594
 595                 // The following line actually doesn't have to do anything with
 596                 // pretty printing, it just tells GDB to print values on one line:
 597                 script_str.push_str("set print pretty off\n");
 598
 599                 // Add the pretty printer directory to GDB's source-file search path
 600                 script_str.push_str(&format!("directory {}\n",
 601                                              rust_pp_module_abs_path));
 602
 603                 // Load the target executable
 604                 script_str.push_str(&format!("file {}\n",
 605                                              exe_file.to_str().unwrap()
 606                                              .replace(r"\", r"\\")));
 607
 608                 // Add line breakpoints
 609                 for line in &breakpoint_lines {
 610                     script_str.push_str(&format!("break '{}':{}\n",
 611                                                  self.testpaths.file.file_name().unwrap()
 612                                                  .to_string_lossy(),
 613                                                  *line));
 614                 }
 615
 616                 script_str.push_str(&cmds);
 617                 script_str.push_str("\nquit\n");
 618
 619                 debug!("script_str = {}", script_str);
 620                 self.dump_output_file(&script_str, "debugger.script");
 621
 622                 // run debugger script with gdb
 623                 fn debugger() -> &'static str {
 624                     if cfg!(windows) {"gdb.exe"} else {"gdb"}
 625                 }
 626
 627                 let debugger_script = self.make_out_name("debugger.script");
 628
 629                 // FIXME (#9639): This needs to handle non-utf8 paths
 630                 let debugger_opts =
 631                     vec!("-quiet".to_owned(),
 632                          "-batch".to_owned(),
 633                          "-nx".to_owned(),
 634                          format!("-command={}", debugger_script.to_str().unwrap()));
 635
 636                 let proc_args = ProcArgs {
 637                     prog: debugger().to_owned(),
 638                     args: debugger_opts,
 639                 };
 640
 641                 let environment = vec![("PYTHONPATH".to_owned(), rust_pp_module_abs_path)];
 642
 643                 debugger_run_result =
 644                     self.compose_and_run(proc_args,
 645                                          environment,
 646                                          self.config.run_lib_path.to_str().unwrap(),
 647                                          None,
 648                                          None);
 649             }
 650         }
 651
 652         if !debugger_run_result.status.success() {
 653             self.fatal("gdb failed to execute");
 654         }
 655
 656         self.check_debugger_output(&debugger_run_result, &check_lines);
 657     }
 658
 659     fn find_rust_src_root(&self) -> Option<PathBuf> {
 660         let mut path = self.config.src_base.clone();
 661         let path_postfix = Path::new("src/etc/lldb_batchmode.py");
 662
 663         while path.pop() {
 664             if path.join(&path_postfix).is_file() {
 665                 return Some(path);
 666             }
 667         }
 668
 669         return None;
 670     }
 671
 672     fn run_debuginfo_lldb_test(&self) {
 673         assert!(self.revision.is_none(), "revisions not relevant here");
 674
 675         if self.config.lldb_python_dir.is_none() {
 676             self.fatal("Can't run LLDB test because LLDB's python path is not set.");
 677         }
 678
 679         let config = Config {
 680             target_rustcflags: self.cleanup_debug_info_options(&self.config.target_rustcflags),
 681             host_rustcflags: self.cleanup_debug_info_options(&self.config.host_rustcflags),
 682             .. self.config.clone()
 683         };
 684
 685
 686         let test_cx = TestCx {
 687             config: &config,
 688             ..*self
 689         };
 690
 691         test_cx.run_debuginfo_lldb_test_no_opt();
 692     }
 693
 694     fn run_debuginfo_lldb_test_no_opt(&self) {
 695         // compile test file (it should have 'compile-flags:-g' in the header)
 696         let compile_result = self.compile_test();
 697         if !compile_result.status.success() {
 698             self.fatal_proc_rec("compilation failed!", &compile_result);
 699         }
 700
 701         let exe_file = self.make_exe_name();
 702
 703         match self.config.lldb_version {
 704             Some(ref version) => {
 705                 println!("NOTE: compiletest thinks it is using LLDB version {}",
 706                          version);
 707             }
 708             _ => {
 709                 println!("NOTE: compiletest does not know which version of \
 710                           LLDB it is using");
 711             }
 712         }
 713
 714         // Parse debugger commands etc from test files
 715         let DebuggerCommands {
 716             commands,
 717             check_lines,
 718             breakpoint_lines,
 719             ..
 720         } = self.parse_debugger_commands("lldb");
 721
 722         // Write debugger script:
 723         // We don't want to hang when calling `quit` while the process is still running
 724         let mut script_str = String::from("settings set auto-confirm true\n");
 725
 726         // Make LLDB emit its version, so we have it documented in the test output
 727         script_str.push_str("version\n");
 728
 729         // Switch LLDB into "Rust mode"
 730         let rust_src_root = self.find_rust_src_root().expect("Could not find Rust source root");
 731         let rust_pp_module_rel_path = Path::new("./src/etc/lldb_rust_formatters.py");
 732         let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
 733                                                    .to_str()
 734                                                    .unwrap()
 735                                                    .to_owned();
 736
 737         script_str.push_str(&format!("command script import {}\n",
 738                                      &rust_pp_module_abs_path[..])[..]);
 739         script_str.push_str("type summary add --no-value ");
 740         script_str.push_str("--python-function lldb_rust_formatters.print_val ");
 741         script_str.push_str("-x \".*\" --category Rust\n");
 742         script_str.push_str("type category enable Rust\n");
 743
 744         // Set breakpoints on every line that contains the string "#break"
 745         let source_file_name = self.testpaths.file.file_name().unwrap().to_string_lossy();
 746         for line in &breakpoint_lines {
 747             script_str.push_str(&format!("breakpoint set --file '{}' --line {}\n",
 748                                          source_file_name,
 749                                          line));
 750         }
 751
 752         // Append the other commands
 753         for line in &commands {
 754             script_str.push_str(line);
 755             script_str.push_str("\n");
 756         }
 757
 758         // Finally, quit the debugger
 759         script_str.push_str("\nquit\n");
 760
 761         // Write the script into a file
 762         debug!("script_str = {}", script_str);
 763         self.dump_output_file(&script_str, "debugger.script");
 764         let debugger_script = self.make_out_name("debugger.script");
 765
 766         // Let LLDB execute the script via lldb_batchmode.py
 767         let debugger_run_result = self.run_lldb(&exe_file,
 768                                                 &debugger_script,
 769                                                 &rust_src_root);
 770
 771         if !debugger_run_result.status.success() {
 772             self.fatal_proc_rec("Error while running LLDB", &debugger_run_result);
 773         }
 774
 775         self.check_debugger_output(&debugger_run_result, &check_lines);
 776     }
 777
 778     fn run_lldb(&self,
 779                 test_executable: &Path,
 780                 debugger_script: &Path,
 781                 rust_src_root: &Path)
 782                 -> ProcRes {
 783         // Prepare the lldb_batchmode which executes the debugger script
 784         let lldb_script_path = rust_src_root.join("src/etc/lldb_batchmode.py");
 785         self.cmd2procres(Command::new(&self.config.lldb_python)
 786                          .arg(&lldb_script_path)
 787                          .arg(test_executable)
 788                          .arg(debugger_script)
 789                          .env("PYTHONPATH",
 790                               self.config.lldb_python_dir.as_ref().unwrap()))
 791     }
 792
 793     fn cmd2procres(&self, cmd: &mut Command) -> ProcRes {
 794         let (status, out, err) = match cmd.output() {
 795             Ok(Output { status, stdout, stderr }) => {
 796                 (status,
 797                  String::from_utf8(stdout).unwrap(),
 798                  String::from_utf8(stderr).unwrap())
 799             },
 800             Err(e) => {
 801                 self.fatal(&format!("Failed to setup Python process for \
 802                                       LLDB script: {}", e))
 803             }
 804         };
 805
 806         self.dump_output(&out, &err);
 807         ProcRes {
 808             status: Status::Normal(status),
 809             stdout: out,
 810             stderr: err,
 811             cmdline: format!("{:?}", cmd)
 812         }
 813     }
 814
 815     fn parse_debugger_commands(&self, debugger_prefix: &str) -> DebuggerCommands {
 816         let command_directive = format!("{}-command", debugger_prefix);
 817         let check_directive = format!("{}-check", debugger_prefix);
 818
 819         let mut breakpoint_lines = vec!();
 820         let mut commands = vec!();
 821         let mut check_lines = vec!();
 822         let mut counter = 1;
 823         let reader = BufReader::new(File::open(&self.testpaths.file).unwrap());
 824         for line in reader.lines() {
 825             match line {
 826                 Ok(line) => {
 827                     if line.contains("#break") {
 828                         breakpoint_lines.push(counter);
 829                     }
 830
 831                     header::parse_name_value_directive(
 832                         &line,
 833                         &command_directive).map(|cmd| {
 834                             commands.push(cmd)
 835                         });
 836
 837                     header::parse_name_value_directive(
 838                         &line,
 839                         &check_directive).map(|cmd| {
 840                             check_lines.push(cmd)
 841                         });
 842                 }
 843                 Err(e) => {
 844                     self.fatal(&format!("Error while parsing debugger commands: {}", e))
 845                 }
 846             }
 847             counter += 1;
 848         }
 849
 850         DebuggerCommands {
 851             commands: commands,
 852             check_lines: check_lines,
 853             breakpoint_lines: breakpoint_lines,
 854         }
 855     }
 856
 857     fn cleanup_debug_info_options(&self, options: &Option<String>) -> Option<String> {
 858         if options.is_none() {
 859             return None;
 860         }
 861
 862         // Remove options that are either unwanted (-O) or may lead to duplicates due to RUSTFLAGS.
 863         let options_to_remove = [
 864             "-O".to_owned(),
 865             "-g".to_owned(),
 866             "--debuginfo".to_owned()
 867         ];
 868         let new_options =
 869             self.split_maybe_args(options).into_iter()
 870                                           .filter(|x| !options_to_remove.contains(x))
 871                                           .collect::<Vec<String>>();
 872
 873         Some(new_options.join(" "))
 874     }
 875
 876     fn check_debugger_output(&self, debugger_run_result: &ProcRes, check_lines: &[String]) {
 877         let num_check_lines = check_lines.len();
 878
 879         let mut check_line_index = 0;
 880         for line in debugger_run_result.stdout.lines() {
 881             if check_line_index >= num_check_lines {
 882                 break;
 883             }
 884
 885             if check_single_line(line, &(check_lines[check_line_index])[..]) {
 886                 check_line_index += 1;
 887             }
 888         }
 889         if check_line_index != num_check_lines && num_check_lines > 0 {
 890             self.fatal_proc_rec(&format!("line not found in debugger output: {}",
 891                                          check_lines[check_line_index]),
 892                                 debugger_run_result);
 893         }
 894
 895         fn check_single_line(line: &str, check_line: &str) -> bool {
 896             // Allow check lines to leave parts unspecified (e.g., uninitialized
 897             // bits in the  wrong case of an enum) with the notation "[...]".
 898             let line = line.trim();
 899             let check_line = check_line.trim();
 900             let can_start_anywhere = check_line.starts_with("[...]");
 901             let can_end_anywhere = check_line.ends_with("[...]");
 902
 903             let check_fragments: Vec<&str> = check_line.split("[...]")
 904                                                        .filter(|frag| !frag.is_empty())
 905                                                        .collect();
 906             if check_fragments.is_empty() {
 907                 return true;
 908             }
 909
 910             let (mut rest, first_fragment) = if can_start_anywhere {
 911                 match line.find(check_fragments[0]) {
 912                     Some(pos) => (&line[pos + check_fragments[0].len() ..], 1),
 913                     None => return false
 914                 }
 915             } else {
 916                 (line, 0)
 917             };
 918
 919             for fragment_index in first_fragment .. check_fragments.len() {
 920                 let current_fragment = check_fragments[fragment_index];
 921                 match rest.find(current_fragment) {
 922                     Some(pos) => {
 923                         rest = &rest[pos + current_fragment.len() .. ];
 924                     }
 925                     None => return false
 926                 }
 927             }
 928
 929             if !can_end_anywhere && !rest.is_empty() {
 930                 return false;
 931             }
 932
 933             return true;
 934         }
 935     }
 936
 937     fn check_error_patterns(&self,
 938                             output_to_check: &str,
 939                             proc_res: &ProcRes) {
 940         if self.props.error_patterns.is_empty() {
 941             self.fatal(&format!("no error pattern specified in {:?}",
 942                                 self.testpaths.file.display()));
 943         }
 944         let mut next_err_idx = 0;
 945         let mut next_err_pat = self.props.error_patterns[next_err_idx].trim();
 946         let mut done = false;
 947         for line in output_to_check.lines() {
 948             if line.contains(next_err_pat) {
 949                 debug!("found error pattern {}", next_err_pat);
 950                 next_err_idx += 1;
 951                 if next_err_idx == self.props.error_patterns.len() {
 952                     debug!("found all error patterns");
 953                     done = true;
 954                     break;
 955                 }
 956                 next_err_pat = self.props.error_patterns[next_err_idx].trim();
 957             }
 958         }
 959         if done { return; }
 960
 961         let missing_patterns = &self.props.error_patterns[next_err_idx..];
 962         if missing_patterns.len() == 1 {
 963             self.fatal_proc_rec(
 964                 &format!("error pattern '{}' not found!", missing_patterns[0]),
 965                 proc_res);
 966         } else {
 967             for pattern in missing_patterns {
 968                 self.error(&format!("error pattern '{}' not found!", *pattern));
 969             }
 970             self.fatal_proc_rec("multiple error patterns not found", proc_res);
 971         }
 972     }
 973
 974     fn check_no_compiler_crash(&self, proc_res: &ProcRes) {
 975         for line in proc_res.stderr.lines() {
 976             if line.starts_with("error: internal compiler error:") {
 977                 self.fatal_proc_rec("compiler encountered internal error", proc_res);
 978             }
 979         }
 980     }
 981
 982     fn check_forbid_output(&self,
 983                            output_to_check: &str,
 984                            proc_res: &ProcRes) {
 985         for pat in &self.props.forbid_output {
 986             if output_to_check.contains(pat) {
 987                 self.fatal_proc_rec("forbidden pattern found in compiler output", proc_res);
 988             }
 989         }
 990     }
 991
 992     fn check_expected_errors(&self,
 993                              expected_errors: Vec<errors::Error>,
 994                              proc_res: &ProcRes) {
 995         if proc_res.status.success() {
 996             self.fatal_proc_rec("process did not return an error status", proc_res);
 997         }
 998
 999         let file_name =
1000             format!("{}", self.testpaths.file.display())
1001             .replace(r"\", "/"); // on windows, translate all '\' path separators to '/'
1002
1003         // If the testcase being checked contains at least one expected "help"
1004         // message, then we'll ensure that all "help" messages are expected.
1005         // Otherwise, all "help" messages reported by the compiler will be ignored.
1006         // This logic also applies to "note" messages.
1007         let expect_help = expected_errors.iter().any(|ee| ee.kind == Some(ErrorKind::Help));
1008         let expect_note = expected_errors.iter().any(|ee| ee.kind == Some(ErrorKind::Note));
1009
1010         // Parse the JSON output from the compiler and extract out the messages.
1011         let actual_errors = json::parse_output(&file_name, &proc_res.stderr, &proc_res);
1012         let mut unexpected = Vec::new();
1013         let mut found = vec![false; expected_errors.len()];
1014         for actual_error in &actual_errors {
1015             let opt_index =
1016                 expected_errors
1017                 .iter()
1018                 .enumerate()
1019                 .position(|(index, expected_error)| {
1020                     !found[index] &&
1021                         actual_error.line_num == expected_error.line_num &&
1022                         (expected_error.kind.is_none() ||
1023                          actual_error.kind == expected_error.kind) &&
1024                         actual_error.msg.contains(&expected_error.msg)
1025                 });
1026
1027             match opt_index {
1028                 Some(index) => {
1029                     // found a match, everybody is happy
1030                     assert!(!found[index]);
1031                     found[index] = true;
1032                 }
1033
1034                 None => {
1035                     if self.is_unexpected_compiler_message(actual_error, expect_help, expect_note) {
1036                         self.error(
1037                             &format!("{}:{}: unexpected {:?}: '{}'",
1038                                      file_name,
1039                                      actual_error.line_num,
1040                                      actual_error.kind.as_ref()
1041                                      .map_or(String::from("message"),
1042                                              |k| k.to_string()),
1043                                      actual_error.msg));
1044                         unexpected.push(actual_error.clone());
1045                     }
1046                 }
1047             }
1048         }
1049
1050         let mut not_found = Vec::new();
1051         // anything not yet found is a problem
1052         for (index, expected_error) in expected_errors.iter().enumerate() {
1053             if !found[index] {
1054                 self.error(
1055                     &format!("{}:{}: expected {} not found: {}",
1056                              file_name,
1057                              expected_error.line_num,
1058                              expected_error.kind.as_ref()
1059                              .map_or("message".into(),
1060                                      |k| k.to_string()),
1061                              expected_error.msg));
1062                 not_found.push(expected_error.clone());
1063             }
1064         }
1065
1066         if unexpected.len() > 0 || not_found.len() > 0 {
1067             self.error(
1068                 &format!("{} unexpected errors found, {} expected errors not found",
1069                          unexpected.len(), not_found.len()));
1070             print!("status: {}\ncommand: {}\n",
1071                    proc_res.status, proc_res.cmdline);
1072             if unexpected.len() > 0 {
1073                 println!("unexpected errors (from JSON output): {:#?}\n", unexpected);
1074             }
1075             if not_found.len() > 0 {
1076                 println!("not found errors (from test file): {:#?}\n", not_found);
1077             }
1078             panic!();
1079         }
1080     }
1081
1082     /// Returns true if we should report an error about `actual_error`,
1083     /// which did not match any of the expected error. We always require
1084     /// errors/warnings to be explicitly listed, but only require
1085     /// helps/notes if there are explicit helps/notes given.
1086     fn is_unexpected_compiler_message(&self,
1087                                       actual_error: &Error,
1088                                       expect_help: bool,
1089                                       expect_note: bool)
1090                                       -> bool {
1091         match actual_error.kind {
1092             Some(ErrorKind::Help) => expect_help,
1093             Some(ErrorKind::Note) => expect_note,
1094             Some(ErrorKind::Error) => true,
1095             Some(ErrorKind::Warning) => true,
1096             Some(ErrorKind::Suggestion) => false,
1097             None => false
1098         }
1099     }
1100
1101     fn compile_test(&self) -> ProcRes {
1102         let aux_dir = self.aux_output_dir_name();
1103         // FIXME (#9639): This needs to handle non-utf8 paths
1104         let link_args = vec!("-L".to_owned(),
1105                              aux_dir.to_str().unwrap().to_owned());
1106         let args = self.make_compile_args(link_args,
1107                                           &self.testpaths.file,
1108                                           TargetLocation::ThisFile(self.make_exe_name()));
1109         self.compose_and_run_compiler(args, None)
1110     }
1111
1112     fn document(&self, out_dir: &Path) -> ProcRes {
1113         if self.props.build_aux_docs {
1114             for rel_ab in &self.props.aux_builds {
1115                 let aux_testpaths = self.compute_aux_test_paths(rel_ab);
1116                 let aux_props = self.props.from_aux_file(&aux_testpaths.file, self.revision);
1117                 let aux_cx = TestCx {
1118                     config: self.config,
1119                     props: &aux_props,
1120                     testpaths: &aux_testpaths,
1121                     revision: self.revision
1122                 };
1123                 let auxres = aux_cx.document(out_dir);
1124                 if !auxres.status.success() {
1125                     return auxres;
1126                 }
1127             }
1128         }
1129
1130         let aux_dir = self.aux_output_dir_name();
1131         let mut args = vec!["-L".to_owned(),
1132                             aux_dir.to_str().unwrap().to_owned(),
1133                             "-o".to_owned(),
1134                             out_dir.to_str().unwrap().to_owned(),
1135                             self.testpaths.file.to_str().unwrap().to_owned()];
1136         args.extend(self.props.compile_flags.iter().cloned());
1137         let args = ProcArgs {
1138             prog: self.config.rustdoc_path.to_str().unwrap().to_owned(),
1139             args: args,
1140         };
1141         self.compose_and_run_compiler(args, None)
1142     }
1143
1144     fn exec_compiled_test(&self) -> ProcRes {
1145         let env = self.props.exec_env.clone();
1146
1147         match &*self.config.target {
1148
1149             "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
1150                 self._arm_exec_compiled_test(env)
1151             }
1152
1153             _=> {
1154                 let aux_dir = self.aux_output_dir_name();
1155                 self.compose_and_run(self.make_run_args(),
1156                                      env,
1157                                      self.config.run_lib_path.to_str().unwrap(),
1158                                      Some(aux_dir.to_str().unwrap()),
1159                                      None)
1160             }
1161         }
1162     }
1163
1164     /// For each `aux-build: foo/bar` annotation, we check to find the
1165     /// file in a `aux` directory relative to the test itself.
1166     fn compute_aux_test_paths(&self, rel_ab: &str) -> TestPaths {
1167         let test_ab = self.testpaths.file
1168                                     .parent()
1169                                     .expect("test file path has no parent")
1170                                     .join("auxiliary")
1171                                     .join(rel_ab);
1172         if !test_ab.exists() {
1173             self.fatal(&format!("aux-build `{}` source not found", test_ab.display()))
1174         }
1175
1176         TestPaths {
1177             file: test_ab,
1178             base: self.testpaths.base.clone(),
1179             relative_dir: self.testpaths.relative_dir
1180                                         .join("auxiliary")
1181                                         .join(rel_ab)
1182                                         .parent()
1183                                         .expect("aux-build path has no parent")
1184                                         .to_path_buf()
1185         }
1186     }
1187
1188     fn compose_and_run_compiler(&self, args: ProcArgs, input: Option<String>) -> ProcRes {
1189         if !self.props.aux_builds.is_empty() {
1190             self.create_dir_racy(&self.aux_output_dir_name());
1191         }
1192
1193         let aux_dir = self.aux_output_dir_name();
1194         // FIXME (#9639): This needs to handle non-utf8 paths
1195         let extra_link_args = vec!["-L".to_owned(),
1196                                    aux_dir.to_str().unwrap().to_owned()];
1197
1198         for rel_ab in &self.props.aux_builds {
1199             let aux_testpaths = self.compute_aux_test_paths(rel_ab);
1200             let aux_props = self.props.from_aux_file(&aux_testpaths.file, self.revision);
1201             let mut crate_type = if aux_props.no_prefer_dynamic {
1202                 Vec::new()
1203             } else {
1204                 // We primarily compile all auxiliary libraries as dynamic libraries
1205                 // to avoid code size bloat and large binaries as much as possible
1206                 // for the test suite (otherwise including libstd statically in all
1207                 // executables takes up quite a bit of space).
1208                 //
1209                 // For targets like MUSL or Emscripten, however, there is no support for
1210                 // dynamic libraries so we just go back to building a normal library. Note,
1211                 // however, that for MUSL if the library is built with `force_host` then
1212                 // it's ok to be a dylib as the host should always support dylibs.
1213                 if (self.config.target.contains("musl") && !aux_props.force_host) ||
1214                     self.config.target.contains("emscripten")
1215                 {
1216                     vec!("--crate-type=lib".to_owned())
1217                 } else {
1218                     vec!("--crate-type=dylib".to_owned())
1219                 }
1220             };
1221             crate_type.extend(extra_link_args.clone());
1222             let aux_output = {
1223                 let f = self.make_lib_name(&self.testpaths.file);
1224                 let parent = f.parent().unwrap();
1225                 TargetLocation::ThisDirectory(parent.to_path_buf())
1226             };
1227             let aux_cx = TestCx {
1228                 config: self.config,
1229                 props: &aux_props,
1230                 testpaths: &aux_testpaths,
1231                 revision: self.revision
1232             };
1233             let aux_args = aux_cx.make_compile_args(crate_type, &aux_testpaths.file, aux_output);
1234             let auxres = aux_cx.compose_and_run(aux_args,
1235                                                 Vec::new(),
1236                                                 aux_cx.config.compile_lib_path.to_str().unwrap(),
1237                                                 Some(aux_dir.to_str().unwrap()),
1238                                                 None);
1239             if !auxres.status.success() {
1240                 self.fatal_proc_rec(
1241                     &format!("auxiliary build of {:?} failed to compile: ",
1242                              aux_testpaths.file.display()),
1243                     &auxres);
1244             }
1245
1246             match &*self.config.target {
1247                 "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
1248                     self._arm_push_aux_shared_library();
1249                 }
1250                 _ => {}
1251             }
1252         }
1253
1254         self.compose_and_run(args,
1255                              self.props.rustc_env.clone(),
1256                              self.config.compile_lib_path.to_str().unwrap(),
1257                              Some(aux_dir.to_str().unwrap()),
1258                              input)
1259     }
1260
1261     // Like std::fs::create_dir_all, except handles concurrent calls among multiple
1262     // threads or processes.
1263     fn create_dir_racy(&self, path: &Path) {
1264         match fs::create_dir(path) {
1265             Ok(()) => return,
1266             Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => return,
1267             Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
1268             Err(e) => panic!("failed to create dir {:?}: {}", path, e),
1269         }
1270         self.create_dir_racy(path.parent().unwrap());
1271         match fs::create_dir(path) {
1272             Ok(()) => {}
1273             Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => {}
1274             Err(e) => panic!("failed to create dir {:?}: {}", path, e),
1275         }
1276     }
1277
1278     fn compose_and_run(&self,
1279                        ProcArgs{ args, prog }: ProcArgs,
1280                        procenv: Vec<(String, String)> ,
1281                        lib_path: &str,
1282                        aux_path: Option<&str>,
1283                        input: Option<String>) -> ProcRes {
1284         return self.program_output(lib_path, prog, aux_path, args, procenv, input);
1285     }
1286
1287     fn make_compile_args(&self,
1288                          extras: Vec<String> ,
1289                          input_file: &Path,
1290                          output_file: TargetLocation)
1291                          -> ProcArgs
1292     {
1293         let target = if self.props.force_host {
1294             &*self.config.host
1295         } else {
1296             &*self.config.target
1297         };
1298
1299         // FIXME (#9639): This needs to handle non-utf8 paths
1300         let mut args = vec!(input_file.to_str().unwrap().to_owned(),
1301                             "-L".to_owned(),
1302                             self.config.build_base.to_str().unwrap().to_owned(),
1303                             format!("--target={}", target));
1304
1305         if let Some(revision) = self.revision {
1306             args.extend(vec![
1307                 format!("--cfg"),
1308                 format!("{}", revision),
1309             ]);
1310         }
1311
1312         if let Some(ref incremental_dir) = self.props.incremental_dir {
1313             args.extend(vec![
1314                 format!("-Z"),
1315                 format!("incremental={}", incremental_dir.display()),
1316             ]);
1317         }
1318
1319
1320         match self.config.mode {
1321             CompileFail |
1322             ParseFail |
1323             Incremental => {
1324                 // If we are extracting and matching errors in the new
1325                 // fashion, then you want JSON mode. Old-skool error
1326                 // patterns still match the raw compiler output.
1327                 if self.props.error_patterns.is_empty() {
1328                     args.extend(["--error-format",
1329                                  "json"]
1330                                 .iter()
1331                                 .map(|s| s.to_string()));
1332                 }
1333             }
1334             MirOpt => {
1335                 args.extend(["-Z",
1336                              "dump-mir=all",
1337                              "-Z",
1338                              "mir-opt-level=3",
1339                              "-Z"]
1340                             .iter()
1341                             .map(|s| s.to_string()));
1342
1343
1344                 let mir_dump_dir = self.get_mir_dump_dir();
1345                 self.create_dir_racy(mir_dump_dir.as_path());
1346                 let mut dir_opt = "dump-mir-dir=".to_string();
1347                 dir_opt.push_str(mir_dump_dir.to_str().unwrap());
1348                 debug!("dir_opt: {:?}", dir_opt);
1349
1350                 args.push(dir_opt);
1351             }
1352             RunFail |
1353             RunPass |
1354             RunPassValgrind |
1355             Pretty |
1356             DebugInfoGdb |
1357             DebugInfoLldb |
1358             Codegen |
1359             Rustdoc |
1360             RunMake |
1361             Ui |
1362             CodegenUnits => {
1363                 // do not use JSON output
1364             }
1365         }
1366
1367         args.extend_from_slice(&extras);
1368         if !self.props.no_prefer_dynamic {
1369             args.push("-C".to_owned());
1370             args.push("prefer-dynamic".to_owned());
1371         }
1372         let path = match output_file {
1373             TargetLocation::ThisFile(path) => {
1374                 args.push("-o".to_owned());
1375                 path
1376             }
1377             TargetLocation::ThisDirectory(path) => {
1378                 args.push("--out-dir".to_owned());
1379                 path
1380             }
1381         };
1382         args.push(path.to_str().unwrap().to_owned());
1383         if self.props.force_host {
1384             args.extend(self.split_maybe_args(&self.config.host_rustcflags));
1385         } else {
1386             args.extend(self.split_maybe_args(&self.config.target_rustcflags));
1387         }
1388         args.extend(self.props.compile_flags.iter().cloned());
1389         return ProcArgs {
1390             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
1391             args: args,
1392         };
1393     }
1394
1395     fn make_lib_name(&self, auxfile: &Path) -> PathBuf {
1396         // what we return here is not particularly important, as it
1397         // happens; rustc ignores everything except for the directory.
1398         let auxname = self.output_testname(auxfile);
1399         self.aux_output_dir_name().join(&auxname)
1400     }
1401
1402     fn make_exe_name(&self) -> PathBuf {
1403         let mut f = self.output_base_name();
1404         // FIXME: This is using the host architecture exe suffix, not target!
1405         if self.config.target == "asmjs-unknown-emscripten" {
1406             let mut fname = f.file_name().unwrap().to_os_string();
1407             fname.push(".js");
1408             f.set_file_name(&fname);
1409         } else if !env::consts::EXE_SUFFIX.is_empty() {
1410             let mut fname = f.file_name().unwrap().to_os_string();
1411             fname.push(env::consts::EXE_SUFFIX);
1412             f.set_file_name(&fname);
1413         }
1414         f
1415     }
1416
1417     fn make_run_args(&self) -> ProcArgs {
1418         // If we've got another tool to run under (valgrind),
1419         // then split apart its command
1420         let mut args = self.split_maybe_args(&self.config.runtool);
1421
1422         // If this is emscripten, then run tests under nodejs
1423         if self.config.target == "asmjs-unknown-emscripten" {
1424             args.push("nodejs".to_owned());
1425         }
1426
1427         let exe_file = self.make_exe_name();
1428
1429         // FIXME (#9639): This needs to handle non-utf8 paths
1430         args.push(exe_file.to_str().unwrap().to_owned());
1431
1432         // Add the arguments in the run_flags directive
1433         args.extend(self.split_maybe_args(&self.props.run_flags));
1434
1435         let prog = args.remove(0);
1436         return ProcArgs {
1437             prog: prog,
1438             args: args,
1439         };
1440     }
1441
1442     fn split_maybe_args(&self, argstr: &Option<String>) -> Vec<String> {
1443         match *argstr {
1444             Some(ref s) => {
1445                 s
1446                     .split(' ')
1447                     .filter_map(|s| {
1448                         if s.chars().all(|c| c.is_whitespace()) {
1449                             None
1450                         } else {
1451                             Some(s.to_owned())
1452                         }
1453                     }).collect()
1454             }
1455             None => Vec::new()
1456         }
1457     }
1458
1459     fn program_output(&self,
1460                       lib_path: &str,
1461                       prog: String,
1462                       aux_path: Option<&str>,
1463                       args: Vec<String>,
1464                       env: Vec<(String, String)>,
1465                       input: Option<String>)
1466                       -> ProcRes {
1467         let cmdline =
1468         {
1469             let cmdline = self.make_cmdline(lib_path,
1470                                             &prog,
1471                                             &args);
1472             logv(self.config, format!("executing {}", cmdline));
1473             cmdline
1474         };
1475         let procsrv::Result {
1476             out,
1477             err,
1478             status
1479         } = procsrv::run(lib_path,
1480                          &prog,
1481                          aux_path,
1482                          &args,
1483                          env,
1484                          input).expect(&format!("failed to exec `{}`", prog));
1485         self.dump_output(&out, &err);
1486         return ProcRes {
1487             status: Status::Normal(status),
1488             stdout: out,
1489             stderr: err,
1490             cmdline: cmdline,
1491         };
1492     }
1493
1494     fn make_cmdline(&self, libpath: &str, prog: &str, args: &[String]) -> String {
1495         use util;
1496
1497         // Linux and mac don't require adjusting the library search path
1498         if cfg!(unix) {
1499             format!("{} {}", prog, args.join(" "))
1500         } else {
1501             // Build the LD_LIBRARY_PATH variable as it would be seen on the command line
1502             // for diagnostic purposes
1503             fn lib_path_cmd_prefix(path: &str) -> String {
1504                 format!("{}=\"{}\"", util::lib_path_env_var(), util::make_new_path(path))
1505             }
1506
1507             format!("{} {} {}", lib_path_cmd_prefix(libpath), prog, args.join(" "))
1508         }
1509     }
1510
1511     fn dump_output(&self, out: &str, err: &str) {
1512         self.dump_output_file(out, "out");
1513         self.dump_output_file(err, "err");
1514         self.maybe_dump_to_stdout(out, err);
1515     }
1516
1517     fn dump_output_file(&self,
1518                         out: &str,
1519                         extension: &str) {
1520         let outfile = self.make_out_name(extension);
1521         File::create(&outfile).unwrap().write_all(out.as_bytes()).unwrap();
1522     }
1523
1524     fn make_out_name(&self, extension: &str) -> PathBuf {
1525         self.output_base_name().with_extension(extension)
1526     }
1527
1528     fn aux_output_dir_name(&self) -> PathBuf {
1529         let f = self.output_base_name();
1530         let mut fname = f.file_name().unwrap().to_os_string();
1531         fname.push(&format!(".{}.libaux", self.config.mode));
1532         f.with_file_name(&fname)
1533     }
1534
1535     fn output_testname(&self, filepath: &Path) -> PathBuf {
1536         PathBuf::from(filepath.file_stem().unwrap())
1537     }
1538
1539     /// Given a test path like `compile-fail/foo/bar.rs` Returns a name like
1540     ///
1541     ///     <output>/foo/bar-stage1
1542     fn output_base_name(&self) -> PathBuf {
1543         let dir = self.config.build_base.join(&self.testpaths.relative_dir);
1544
1545         // Note: The directory `dir` is created during `collect_tests_from_dir`
1546         dir
1547             .join(&self.output_testname(&self.testpaths.file))
1548             .with_extension(&self.config.stage_id)
1549     }
1550
1551     fn maybe_dump_to_stdout(&self, out: &str, err: &str) {
1552         if self.config.verbose {
1553             println!("------{}------------------------------", "stdout");
1554             println!("{}", out);
1555             println!("------{}------------------------------", "stderr");
1556             println!("{}", err);
1557             println!("------------------------------------------");
1558         }
1559     }
1560
1561     fn error(&self, err: &str) {
1562         match self.revision {
1563             Some(rev) => println!("\nerror in revision `{}`: {}", rev, err),
1564             None => println!("\nerror: {}", err)
1565         }
1566     }
1567
1568     fn fatal(&self, err: &str) -> ! {
1569         self.error(err); panic!();
1570     }
1571
1572     fn fatal_proc_rec(&self, err: &str, proc_res: &ProcRes) -> ! {
1573         self.error(err);
1574         proc_res.fatal(None);
1575     }
1576
1577     fn _arm_exec_compiled_test(&self, env: Vec<(String, String)>) -> ProcRes {
1578         let args = self.make_run_args();
1579         let cmdline = self.make_cmdline("", &args.prog, &args.args);
1580
1581         // get bare program string
1582         let mut tvec: Vec<String> = args.prog
1583                                         .split('/')
1584                                         .map(str::to_owned)
1585                                         .collect();
1586         let prog_short = tvec.pop().unwrap();
1587
1588         // copy to target
1589         let copy_result = procsrv::run("",
1590                                        &self.config.adb_path,
1591                                        None,
1592                                        &[
1593                                            "push".to_owned(),
1594                                            args.prog.clone(),
1595                                            self.config.adb_test_dir.clone()
1596                                        ],
1597                                        vec!(("".to_owned(), "".to_owned())),
1598                                        Some("".to_owned()))
1599             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1600
1601         if self.config.verbose {
1602             println!("push ({}) {} {} {}",
1603                      self.config.target,
1604                      args.prog,
1605                      copy_result.out,
1606                      copy_result.err);
1607         }
1608
1609         logv(self.config, format!("executing ({}) {}", self.config.target, cmdline));
1610
1611         let mut runargs = Vec::new();
1612
1613         // run test via adb_run_wrapper
1614         runargs.push("shell".to_owned());
1615         for (key, val) in env {
1616             runargs.push(format!("{}={}", key, val));
1617         }
1618         runargs.push(format!("{}/../adb_run_wrapper.sh", self.config.adb_test_dir));
1619         runargs.push(format!("{}", self.config.adb_test_dir));
1620         runargs.push(format!("{}", prog_short));
1621
1622         for tv in &args.args {
1623             runargs.push(tv.to_owned());
1624         }
1625         procsrv::run("",
1626                      &self.config.adb_path,
1627                      None,
1628                      &runargs,
1629                      vec!(("".to_owned(), "".to_owned())), Some("".to_owned()))
1630             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1631
1632         // get exitcode of result
1633         runargs = Vec::new();
1634         runargs.push("shell".to_owned());
1635         runargs.push("cat".to_owned());
1636         runargs.push(format!("{}/{}.exitcode", self.config.adb_test_dir, prog_short));
1637
1638         let procsrv::Result{ out: exitcode_out, err: _, status: _ } =
1639             procsrv::run("",
1640                          &self.config.adb_path,
1641                          None,
1642                          &runargs,
1643                          vec!(("".to_owned(), "".to_owned())),
1644                          Some("".to_owned()))
1645             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1646
1647         let mut exitcode: i32 = 0;
1648         for c in exitcode_out.chars() {
1649             if !c.is_numeric() { break; }
1650             exitcode = exitcode * 10 + match c {
1651                 '0' ... '9' => c as i32 - ('0' as i32),
1652                 _ => 101,
1653             }
1654         }
1655
1656         // get stdout of result
1657         runargs = Vec::new();
1658         runargs.push("shell".to_owned());
1659         runargs.push("cat".to_owned());
1660         runargs.push(format!("{}/{}.stdout", self.config.adb_test_dir, prog_short));
1661
1662         let procsrv::Result{ out: stdout_out, err: _, status: _ } =
1663             procsrv::run("",
1664                          &self.config.adb_path,
1665                          None,
1666                          &runargs,
1667                          vec!(("".to_owned(), "".to_owned())),
1668                          Some("".to_owned()))
1669             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1670
1671         // get stderr of result
1672         runargs = Vec::new();
1673         runargs.push("shell".to_owned());
1674         runargs.push("cat".to_owned());
1675         runargs.push(format!("{}/{}.stderr", self.config.adb_test_dir, prog_short));
1676
1677         let procsrv::Result{ out: stderr_out, err: _, status: _ } =
1678             procsrv::run("",
1679                          &self.config.adb_path,
1680                          None,
1681                          &runargs,
1682                          vec!(("".to_owned(), "".to_owned())),
1683                          Some("".to_owned()))
1684             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1685
1686         self.dump_output(&stdout_out, &stderr_out);
1687
1688         ProcRes {
1689             status: Status::Parsed(exitcode),
1690             stdout: stdout_out,
1691             stderr: stderr_out,
1692             cmdline: cmdline
1693         }
1694     }
1695
1696     fn _arm_push_aux_shared_library(&self) {
1697         let tdir = self.aux_output_dir_name();
1698
1699         let dirs = fs::read_dir(&tdir).unwrap();
1700         for file in dirs {
1701             let file = file.unwrap().path();
1702             if file.extension().and_then(|s| s.to_str()) == Some("so") {
1703                 // FIXME (#9639): This needs to handle non-utf8 paths
1704                 let copy_result = procsrv::run("",
1705                                                &self.config.adb_path,
1706                                                None,
1707                                                &[
1708                                                    "push".to_owned(),
1709                                                    file.to_str()
1710                                                        .unwrap()
1711                                                        .to_owned(),
1712                                                    self.config.adb_test_dir.to_owned(),
1713                                                ],
1714                                                vec!(("".to_owned(),
1715                                                      "".to_owned())),
1716                                                Some("".to_owned()))
1717                     .expect(&format!("failed to exec `{}`", self.config.adb_path));
1718
1719                 if self.config.verbose {
1720                     println!("push ({}) {:?} {} {}",
1721                              self.config.target, file.display(),
1722                              copy_result.out, copy_result.err);
1723                 }
1724             }
1725         }
1726     }
1727
1728     // codegen tests (using FileCheck)
1729
1730     fn compile_test_and_save_ir(&self) -> ProcRes {
1731         let aux_dir = self.aux_output_dir_name();
1732         // FIXME (#9639): This needs to handle non-utf8 paths
1733         let mut link_args = vec!("-L".to_owned(),
1734                                  aux_dir.to_str().unwrap().to_owned());
1735         let llvm_args = vec!("--emit=llvm-ir".to_owned(),);
1736         link_args.extend(llvm_args);
1737         let args = self.make_compile_args(link_args,
1738                                           &self.testpaths.file,
1739                                           TargetLocation::ThisDirectory(
1740                                               self.output_base_name().parent()
1741                                                                      .unwrap()
1742                                                                      .to_path_buf()));
1743         self.compose_and_run_compiler(args, None)
1744     }
1745
1746     fn check_ir_with_filecheck(&self) -> ProcRes {
1747         let irfile = self.output_base_name().with_extension("ll");
1748         let prog = self.config.llvm_filecheck.as_ref().unwrap();
1749         let proc_args = ProcArgs {
1750             // FIXME (#9639): This needs to handle non-utf8 paths
1751             prog: prog.to_str().unwrap().to_owned(),
1752             args: vec!(format!("-input-file={}", irfile.to_str().unwrap()),
1753                        self.testpaths.file.to_str().unwrap().to_owned())
1754         };
1755         self.compose_and_run(proc_args, Vec::new(), "", None, None)
1756     }
1757
1758     fn run_codegen_test(&self) {
1759         assert!(self.revision.is_none(), "revisions not relevant here");
1760
1761         if self.config.llvm_filecheck.is_none() {
1762             self.fatal("missing --llvm-filecheck");
1763         }
1764
1765         let mut proc_res = self.compile_test_and_save_ir();
1766         if !proc_res.status.success() {
1767             self.fatal_proc_rec("compilation failed!", &proc_res);
1768         }
1769
1770         proc_res = self.check_ir_with_filecheck();
1771         if !proc_res.status.success() {
1772             self.fatal_proc_rec("verification with 'FileCheck' failed", &proc_res);
1773         }
1774     }
1775
1776     fn charset() -> &'static str {
1777         // FreeBSD 10.1 defaults to GDB 6.1.1 which doesn't support "auto" charset
1778         if cfg!(target_os = "bitrig") {
1779             "auto"
1780         } else if cfg!(target_os = "freebsd") {
1781             "ISO-8859-1"
1782         } else {
1783             "UTF-8"
1784         }
1785     }
1786
1787     fn run_rustdoc_test(&self) {
1788         assert!(self.revision.is_none(), "revisions not relevant here");
1789
1790         let out_dir = self.output_base_name();
1791         let _ = fs::remove_dir_all(&out_dir);
1792         self.create_dir_racy(&out_dir);
1793
1794         let proc_res = self.document(&out_dir);
1795         if !proc_res.status.success() {
1796             self.fatal_proc_rec("rustdoc failed!", &proc_res);
1797         }
1798         let root = self.find_rust_src_root().unwrap();
1799
1800         let res = self.cmd2procres(Command::new(&self.config.docck_python)
1801                                    .arg(root.join("src/etc/htmldocck.py"))
1802                                    .arg(out_dir)
1803                                    .arg(&self.testpaths.file));
1804         if !res.status.success() {
1805             self.fatal_proc_rec("htmldocck failed!", &res);
1806         }
1807     }
1808
1809     fn run_codegen_units_test(&self) {
1810         assert!(self.revision.is_none(), "revisions not relevant here");
1811
1812         let proc_res = self.compile_test();
1813
1814         if !proc_res.status.success() {
1815             self.fatal_proc_rec("compilation failed!", &proc_res);
1816         }
1817
1818         self.check_no_compiler_crash(&proc_res);
1819
1820         const PREFIX: &'static str = "TRANS_ITEM ";
1821         const CGU_MARKER: &'static str = "@@";
1822
1823         let actual: Vec<TransItem> = proc_res
1824             .stdout
1825             .lines()
1826             .filter(|line| line.starts_with(PREFIX))
1827             .map(str_to_trans_item)
1828             .collect();
1829
1830         let expected: Vec<TransItem> = errors::load_errors(&self.testpaths.file, None)
1831             .iter()
1832             .map(|e| str_to_trans_item(&e.msg[..]))
1833             .collect();
1834
1835         let mut missing = Vec::new();
1836         let mut wrong_cgus = Vec::new();
1837
1838         for expected_item in &expected {
1839             let actual_item_with_same_name = actual.iter()
1840                                                    .find(|ti| ti.name == expected_item.name);
1841
1842             if let Some(actual_item) = actual_item_with_same_name {
1843                 if !expected_item.codegen_units.is_empty() {
1844                     // Also check for codegen units
1845                     if expected_item.codegen_units != actual_item.codegen_units {
1846                         wrong_cgus.push((expected_item.clone(), actual_item.clone()));
1847                     }
1848                 }
1849             } else {
1850                 missing.push(expected_item.string.clone());
1851             }
1852         }
1853
1854         let unexpected: Vec<_> =
1855             actual.iter()
1856                   .filter(|acgu| !expected.iter().any(|ecgu| acgu.name == ecgu.name))
1857                   .map(|acgu| acgu.string.clone())
1858                   .collect();
1859
1860         if !missing.is_empty() {
1861             missing.sort();
1862
1863             println!("\nThese items should have been contained but were not:\n");
1864
1865             for item in &missing {
1866                 println!("{}", item);
1867             }
1868
1869             println!("\n");
1870         }
1871
1872         if !unexpected.is_empty() {
1873             let sorted = {
1874                 let mut sorted = unexpected.clone();
1875                 sorted.sort();
1876                 sorted
1877             };
1878
1879             println!("\nThese items were contained but should not have been:\n");
1880
1881             for item in sorted {
1882                 println!("{}", item);
1883             }
1884
1885             println!("\n");
1886         }
1887
1888         if !wrong_cgus.is_empty() {
1889             wrong_cgus.sort_by_key(|pair| pair.0.name.clone());
1890             println!("\nThe following items were assigned to wrong codegen units:\n");
1891
1892             for &(ref expected_item, ref actual_item) in &wrong_cgus {
1893                 println!("{}", expected_item.name);
1894                 println!("  expected: {}", codegen_units_to_str(&expected_item.codegen_units));
1895                 println!("  actual:   {}", codegen_units_to_str(&actual_item.codegen_units));
1896                 println!("");
1897             }
1898         }
1899
1900         if !(missing.is_empty() && unexpected.is_empty() && wrong_cgus.is_empty())
1901         {
1902             panic!();
1903         }
1904
1905         #[derive(Clone, Eq, PartialEq)]
1906         struct TransItem {
1907             name: String,
1908             codegen_units: HashSet<String>,
1909             string: String,
1910         }
1911
1912         // [TRANS_ITEM] name [@@ (cgu)+]
1913         fn str_to_trans_item(s: &str) -> TransItem {
1914             let s = if s.starts_with(PREFIX) {
1915                 (&s[PREFIX.len()..]).trim()
1916             } else {
1917                 s.trim()
1918             };
1919
1920             let full_string = format!("{}{}", PREFIX, s.trim().to_owned());
1921
1922             let parts: Vec<&str> = s.split(CGU_MARKER)
1923                                     .map(str::trim)
1924                                     .filter(|s| !s.is_empty())
1925                                     .collect();
1926
1927             let name = parts[0].trim();
1928
1929             let cgus = if parts.len() > 1 {
1930                 let cgus_str = parts[1];
1931
1932                 cgus_str.split(" ")
1933                         .map(str::trim)
1934                         .filter(|s| !s.is_empty())
1935                         .map(str::to_owned)
1936                         .collect()
1937             }
1938             else {
1939                 HashSet::new()
1940             };
1941
1942             TransItem {
1943                 name: name.to_owned(),
1944                 codegen_units: cgus,
1945                 string: full_string,
1946             }
1947         }
1948
1949         fn codegen_units_to_str(cgus: &HashSet<String>) -> String
1950         {
1951             let mut cgus: Vec<_> = cgus.iter().collect();
1952             cgus.sort();
1953
1954             let mut string = String::new();
1955             for cgu in cgus {
1956                 string.push_str(&cgu[..]);
1957                 string.push_str(" ");
1958             }
1959
1960             string
1961         }
1962     }
1963
1964     fn init_incremental_test(&self) {
1965         // (See `run_incremental_test` for an overview of how incremental tests work.)
1966
1967         // Before any of the revisions have executed, create the
1968         // incremental workproduct directory.  Delete any old
1969         // incremental work products that may be there from prior
1970         // runs.
1971         let incremental_dir = self.incremental_dir();
1972         if incremental_dir.exists() {
1973             fs::remove_dir_all(&incremental_dir).unwrap();
1974         }
1975         fs::create_dir_all(&incremental_dir).unwrap();
1976
1977         if self.config.verbose {
1978             print!("init_incremental_test: incremental_dir={}", incremental_dir.display());
1979         }
1980     }
1981
1982     fn run_incremental_test(&self) {
1983         // Basic plan for a test incremental/foo/bar.rs:
1984         // - load list of revisions rpass1, cfail2, rpass3
1985         //   - each should begin with `rpass`, `cfail`, or `cfail`
1986         //   - if `rpass`, expect compile and execution to succeed
1987         //   - if `cfail`, expect compilation to fail
1988         //   - if `rfail`, expect execution to fail
1989         // - create a directory build/foo/bar.incremental
1990         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C rpass1
1991         //   - because name of revision starts with "rpass", expect success
1992         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C cfail2
1993         //   - because name of revision starts with "cfail", expect an error
1994         //   - load expected errors as usual, but filter for those that end in `[rfail2]`
1995         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C rpass3
1996         //   - because name of revision starts with "rpass", expect success
1997         // - execute build/foo/bar.exe and save output
1998         //
1999         // FIXME -- use non-incremental mode as an oracle? That doesn't apply
2000         // to #[rustc_dirty] and clean tests I guess
2001
2002         let revision = self.revision.expect("incremental tests require a list of revisions");
2003
2004         // Incremental workproduct directory should have already been created.
2005         let incremental_dir = self.incremental_dir();
2006         assert!(incremental_dir.exists(), "init_incremental_test failed to create incremental dir");
2007
2008         // Add an extra flag pointing at the incremental directory.
2009         let mut revision_props = self.props.clone();
2010         revision_props.incremental_dir = Some(incremental_dir);
2011         revision_props.compile_flags.push(String::from("-Zincremental-info"));
2012
2013         let revision_cx = TestCx {
2014             config: self.config,
2015             props: &revision_props,
2016             testpaths: self.testpaths,
2017             revision: self.revision,
2018         };
2019
2020         if self.config.verbose {
2021             print!("revision={:?} revision_props={:#?}", revision, revision_props);
2022         }
2023
2024         if revision.starts_with("rpass") {
2025             revision_cx.run_rpass_test();
2026         } else if revision.starts_with("rfail") {
2027             revision_cx.run_rfail_test();
2028         } else if revision.starts_with("cfail") {
2029             revision_cx.run_cfail_test();
2030         } else {
2031             revision_cx.fatal(
2032                 "revision name must begin with rpass, rfail, or cfail");
2033         }
2034     }
2035
2036     /// Directory where incremental work products are stored.
2037     fn incremental_dir(&self) -> PathBuf {
2038         self.output_base_name().with_extension("incremental")
2039     }
2040
2041     fn run_rmake_test(&self) {
2042         // FIXME(#11094): we should fix these tests
2043         if self.config.host != self.config.target {
2044             return
2045         }
2046
2047         let cwd = env::current_dir().unwrap();
2048         let src_root = self.config.src_base.parent().unwrap()
2049                                            .parent().unwrap()
2050                                            .parent().unwrap();
2051         let src_root = cwd.join(&src_root);
2052
2053         let tmpdir = cwd.join(self.output_base_name());
2054         if tmpdir.exists() {
2055             self.aggressive_rm_rf(&tmpdir).unwrap();
2056         }
2057         self.create_dir_racy(&tmpdir);
2058
2059         let mut cmd = Command::new("make");
2060         cmd.current_dir(&self.testpaths.file)
2061            .env("TARGET", &self.config.target)
2062            .env("PYTHON", &self.config.docck_python)
2063            .env("S", src_root)
2064            .env("RUST_BUILD_STAGE", &self.config.stage_id)
2065            .env("RUSTC", cwd.join(&self.config.rustc_path))
2066            .env("RUSTDOC", cwd.join(&self.config.rustdoc_path))
2067            .env("TMPDIR", &tmpdir)
2068            .env("LD_LIB_PATH_ENVVAR", procsrv::dylib_env_var())
2069            .env("HOST_RPATH_DIR", cwd.join(&self.config.compile_lib_path))
2070            .env("TARGET_RPATH_DIR", cwd.join(&self.config.run_lib_path))
2071            .env("LLVM_COMPONENTS", &self.config.llvm_components)
2072            .env("LLVM_CXXFLAGS", &self.config.llvm_cxxflags);
2073
2074         if self.config.target.contains("msvc") {
2075             // We need to pass a path to `lib.exe`, so assume that `cc` is `cl.exe`
2076             // and that `lib.exe` lives next to it.
2077             let lib = Path::new(&self.config.cc).parent().unwrap().join("lib.exe");
2078
2079             // MSYS doesn't like passing flags of the form `/foo` as it thinks it's
2080             // a path and instead passes `C:\msys64\foo`, so convert all
2081             // `/`-arguments to MSVC here to `-` arguments.
2082             let cflags = self.config.cflags.split(' ').map(|s| s.replace("/", "-"))
2083                                                  .collect::<Vec<_>>().join(" ");
2084
2085             cmd.env("IS_MSVC", "1")
2086                .env("MSVC_LIB", format!("'{}' -nologo", lib.display()))
2087                .env("CC", format!("'{}' {}", self.config.cc, cflags))
2088                .env("CXX", &self.config.cxx);
2089         } else {
2090             cmd.env("CC", format!("{} {}", self.config.cc, self.config.cflags))
2091                .env("CXX", format!("{} {}", self.config.cxx, self.config.cflags));
2092         }
2093
2094         let output = cmd.output().expect("failed to spawn `make`");
2095         if !output.status.success() {
2096             let res = ProcRes {
2097                 status: Status::Normal(output.status),
2098                 stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
2099                 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
2100                 cmdline: format!("{:?}", cmd),
2101             };
2102             self.fatal_proc_rec("make failed", &res);
2103         }
2104     }
2105
2106     fn aggressive_rm_rf(&self, path: &Path) -> io::Result<()> {
2107         for e in try!(path.read_dir()) {
2108             let entry = try!(e);
2109             let path = entry.path();
2110             if try!(entry.file_type()).is_dir() {
2111                 try!(self.aggressive_rm_rf(&path));
2112             } else {
2113                 // Remove readonly files as well on windows (by default we can't)
2114                 try!(fs::remove_file(&path).or_else(|e| {
2115                     if cfg!(windows) && e.kind() == io::ErrorKind::PermissionDenied {
2116                         let mut meta = try!(entry.metadata()).permissions();
2117                         meta.set_readonly(false);
2118                         try!(fs::set_permissions(&path, meta));
2119                         fs::remove_file(&path)
2120                     } else {
2121                         Err(e)
2122                     }
2123                 }))
2124             }
2125         }
2126         fs::remove_dir(path)
2127     }
2128
2129     fn run_ui_test(&self) {
2130         println!("ui: {}", self.testpaths.file.display());
2131
2132         let proc_res = self.compile_test();
2133
2134         let expected_stderr_path = self.expected_output_path("stderr");
2135         let expected_stderr = self.load_expected_output(&expected_stderr_path);
2136
2137         let expected_stdout_path = self.expected_output_path("stdout");
2138         let expected_stdout = self.load_expected_output(&expected_stdout_path);
2139
2140         let normalized_stdout = self.normalize_output(&proc_res.stdout);
2141         let normalized_stderr = self.normalize_output(&proc_res.stderr);
2142
2143         let mut errors = 0;
2144         errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
2145         errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
2146
2147         if errors > 0 {
2148             println!("To update references, run this command from build directory:");
2149             let relative_path_to_file =
2150                 self.testpaths.relative_dir
2151                               .join(self.testpaths.file.file_name().unwrap());
2152             println!("{}/update-references.sh '{}' '{}'",
2153                      self.config.src_base.display(),
2154                      self.config.build_base.display(),
2155                      relative_path_to_file.display());
2156             self.fatal_proc_rec(&format!("{} errors occurred comparing output.", errors),
2157                                 &proc_res);
2158         }
2159     }
2160
2161     fn run_mir_opt_test(&self) {
2162         let proc_res = self.compile_test();
2163
2164         if !proc_res.status.success() {
2165             self.fatal_proc_rec("compilation failed!", &proc_res);
2166         }
2167
2168         let proc_res = self.exec_compiled_test();
2169
2170         if !proc_res.status.success() {
2171             self.fatal_proc_rec("test run failed!", &proc_res);
2172         }
2173         self.check_mir_dump();
2174     }
2175
2176     fn check_mir_dump(&self) {
2177         let mut test_file_contents = String::new();
2178         fs::File::open(self.testpaths.file.clone()).unwrap()
2179                                                    .read_to_string(&mut test_file_contents)
2180                                                    .unwrap();
2181         if let Some(idx) =  test_file_contents.find("// END RUST SOURCE") {
2182             let (_, tests_text) = test_file_contents.split_at(idx + "// END_RUST SOURCE".len());
2183             let tests_text_str = String::from(tests_text);
2184             let mut curr_test : Option<&str> = None;
2185             let mut curr_test_contents = Vec::new();
2186             for l in tests_text_str.lines() {
2187                 debug!("line: {:?}", l);
2188                 if l.starts_with("// START ") {
2189                     let (_, t) = l.split_at("// START ".len());
2190                     curr_test = Some(t);
2191                 } else if l.starts_with("// END") {
2192                     let (_, t) = l.split_at("// END ".len());
2193                     if Some(t) != curr_test {
2194                         panic!("mismatched START END test name");
2195                     }
2196                     self.compare_mir_test_output(curr_test.unwrap(), &curr_test_contents);
2197                     curr_test = None;
2198                     curr_test_contents.clear();
2199                 } else if l.is_empty() {
2200                     // ignore
2201                 } else if l.starts_with("// ") {
2202                     let (_, test_content) = l.split_at("// ".len());
2203                     curr_test_contents.push(test_content);
2204                 }
2205             }
2206         }
2207     }
2208
2209     fn compare_mir_test_output(&self, test_name: &str, expected_content: &Vec<&str>) {
2210         let mut output_file = PathBuf::new();
2211         output_file.push(self.get_mir_dump_dir());
2212         output_file.push(test_name);
2213         debug!("comparing the contests of: {:?}", output_file);
2214         debug!("with: {:?}", expected_content);
2215
2216         let mut dumped_file = fs::File::open(output_file.clone()).unwrap();
2217         let mut dumped_string = String::new();
2218         dumped_file.read_to_string(&mut dumped_string).unwrap();
2219         let mut dumped_lines = dumped_string.lines().filter(|l| !l.is_empty());
2220         let mut expected_lines = expected_content.iter().filter(|l| !l.is_empty());
2221
2222         // We expect each non-empty line from expected_content to appear
2223         // in the dump in order, but there may be extra lines interleaved
2224         while let Some(expected_line) = expected_lines.next() {
2225             let e_norm = normalize_mir_line(expected_line);
2226             if e_norm.is_empty() {
2227                 continue;
2228             };
2229             let mut found = false;
2230             while let Some(dumped_line) = dumped_lines.next() {
2231                 let d_norm = normalize_mir_line(dumped_line);
2232                 debug!("found: {:?}", d_norm);
2233                 debug!("expected: {:?}", e_norm);
2234                 if e_norm == d_norm {
2235                     found = true;
2236                     break;
2237                 };
2238             }
2239             if !found {
2240                 panic!("ran out of mir dump output to match against");
2241             }
2242         }
2243     }
2244
2245     fn get_mir_dump_dir(&self) -> PathBuf {
2246         let mut mir_dump_dir = PathBuf::from(self.config.build_base
2247                                                     .as_path()
2248                                                     .to_str()
2249                                                     .unwrap());
2250         debug!("input_file: {:?}", self.testpaths.file);
2251         mir_dump_dir.push(self.testpaths.file.file_stem().unwrap().to_str().unwrap());
2252         mir_dump_dir
2253     }
2254
2255     fn normalize_output(&self, output: &str) -> String {
2256         let parent_dir = self.testpaths.file.parent().unwrap();
2257         let parent_dir_str = parent_dir.display().to_string();
2258         output.replace(&parent_dir_str, "$DIR")
2259               .replace("\\", "/") // normalize for paths on windows
2260               .replace("\r\n", "\n") // normalize for linebreaks on windows
2261               .replace("\t", "\\t") // makes tabs visible
2262     }
2263
2264     fn expected_output_path(&self, kind: &str) -> PathBuf {
2265         let extension = match self.revision {
2266             Some(r) => format!("{}.{}", r, kind),
2267             None => kind.to_string(),
2268         };
2269         self.testpaths.file.with_extension(extension)
2270     }
2271
2272     fn load_expected_output(&self, path: &Path) -> String {
2273         if !path.exists() {
2274             return String::new();
2275         }
2276
2277         let mut result = String::new();
2278         match File::open(path).and_then(|mut f| f.read_to_string(&mut result)) {
2279             Ok(_) => result,
2280             Err(e) => {
2281                 self.fatal(&format!("failed to load expected output from `{}`: {}",
2282                                     path.display(), e))
2283             }
2284         }
2285     }
2286
2287     fn compare_output(&self, kind: &str, actual: &str, expected: &str) -> usize {
2288         if actual == expected {
2289             return 0;
2290         }
2291
2292         println!("normalized {}:\n{}\n", kind, actual);
2293         println!("expected {}:\n{}\n", kind, expected);
2294         println!("diff of {}:\n", kind);
2295         for line in uidiff::diff_lines(actual, expected) {
2296             println!("{}", line);
2297         }
2298
2299         let output_file = self.output_base_name().with_extension(kind);
2300         match File::create(&output_file).and_then(|mut f| f.write_all(actual.as_bytes())) {
2301             Ok(()) => { }
2302             Err(e) => {
2303                 self.fatal(&format!("failed to write {} to `{}`: {}",
2304                                     kind, output_file.display(), e))
2305             }
2306         }
2307
2308         println!("\nThe actual {0} differed from the expected {0}.", kind);
2309         println!("Actual {} saved to {}", kind, output_file.display());
2310         1
2311     }
2312 }
2313
2314 struct ProcArgs {
2315     prog: String,
2316     args: Vec<String>,
2317 }
2318
2319 pub struct ProcRes {
2320     status: Status,
2321     stdout: String,
2322     stderr: String,
2323     cmdline: String,
2324 }
2325
2326 enum Status {
2327     Parsed(i32),
2328     Normal(ExitStatus),
2329 }
2330
2331 impl ProcRes {
2332     pub fn fatal(&self, err: Option<&str>) -> ! {
2333         if let Some(e) = err {
2334             println!("\nerror: {}", e);
2335         }
2336         print!("\
2337             status: {}\n\
2338             command: {}\n\
2339             stdout:\n\
2340             ------------------------------------------\n\
2341             {}\n\
2342             ------------------------------------------\n\
2343             stderr:\n\
2344             ------------------------------------------\n\
2345             {}\n\
2346             ------------------------------------------\n\
2347             \n",
2348                self.status, self.cmdline, self.stdout,
2349                self.stderr);
2350         panic!();
2351     }
2352 }
2353
2354 impl Status {
2355     fn code(&self) -> Option<i32> {
2356         match *self {
2357             Status::Parsed(i) => Some(i),
2358             Status::Normal(ref e) => e.code(),
2359         }
2360     }
2361
2362     fn success(&self) -> bool {
2363         match *self {
2364             Status::Parsed(i) => i == 0,
2365             Status::Normal(ref e) => e.success(),
2366         }
2367     }
2368 }
2369
2370 impl fmt::Display for Status {
2371     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2372         match *self {
2373             Status::Parsed(i) => write!(f, "exit code: {}", i),
2374             Status::Normal(ref e) => e.fmt(f),
2375         }
2376     }
2377 }
2378
2379 enum TargetLocation {
2380     ThisFile(PathBuf),
2381     ThisDirectory(PathBuf),
2382 }
2383
2384 fn normalize_mir_line(line: &str) -> String {
2385     let no_comments = if let Some(idx) = line.find("//") {
2386         let (l, _) = line.split_at(idx);
2387         l
2388     } else {
2389         line
2390     };
2391     no_comments.replace(char::is_whitespace, "")
2392 }