]> git.lizzy.rs Git - rust.git/blob - src/tools/compiletest/src/runtest.rs
add mir optimization tests, dump-mir-dir option
[rust.git] / src / tools / compiletest / src / runtest.rs
1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 use common::Config;
12 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
13 use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
14 use common::{Incremental, RunMake, Ui, MirOpt};
15 use errors::{self, ErrorKind, Error};
16 use json;
17 use header::TestProps;
18 use header;
19 use procsrv;
20 use test::TestPaths;
21 use uidiff;
22 use util::logv;
23
24 use std::env;
25 use std::collections::HashSet;
26 use std::fmt;
27 use std::fs::{self, File};
28 use std::io::{self, BufReader};
29 use std::io::prelude::*;
30 use std::net::TcpStream;
31 use std::path::{Path, PathBuf};
32 use std::process::{Command, Output, ExitStatus};
33 use std::str;
34
35 pub fn run(config: Config, testpaths: &TestPaths) {
36     match &*config.target {
37
38         "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
39             if !config.adb_device_status {
40                 panic!("android device not available");
41             }
42         }
43
44         _=> { }
45     }
46
47     if config.verbose {
48         // We're going to be dumping a lot of info. Start on a new line.
49         print!("\n\n");
50     }
51     debug!("running {:?}", testpaths.file.display());
52     let base_props = TestProps::from_file(&testpaths.file);
53
54     let base_cx = TestCx { config: &config,
55                            props: &base_props,
56                            testpaths: testpaths,
57                            revision: None };
58     base_cx.init_all();
59
60     if base_props.revisions.is_empty() {
61         base_cx.run_revision()
62     } else {
63         for revision in &base_props.revisions {
64             let mut revision_props = base_props.clone();
65             revision_props.load_from(&testpaths.file, Some(&revision));
66             let rev_cx = TestCx {
67                 config: &config,
68                 props: &revision_props,
69                 testpaths: testpaths,
70                 revision: Some(revision)
71             };
72             rev_cx.run_revision();
73         }
74     }
75
76     base_cx.complete_all();
77 }
78
79 struct TestCx<'test> {
80     config: &'test Config,
81     props: &'test TestProps,
82     testpaths: &'test TestPaths,
83     revision: Option<&'test str>
84 }
85
86 struct DebuggerCommands {
87     commands: Vec<String>,
88     check_lines: Vec<String>,
89     breakpoint_lines: Vec<usize>,
90 }
91
92 impl<'test> TestCx<'test> {
93     /// invoked once before any revisions have been processed
94     fn init_all(&self) {
95         assert!(self.revision.is_none(), "init_all invoked for a revision");
96         match self.config.mode {
97             Incremental => self.init_incremental_test(),
98             _ => { }
99         }
100     }
101
102     /// Code executed for each revision in turn (or, if there are no
103     /// revisions, exactly once, with revision == None).
104     fn run_revision(&self) {
105         match self.config.mode {
106             CompileFail => self.run_cfail_test(),
107             ParseFail => self.run_cfail_test(),
108             RunFail => self.run_rfail_test(),
109             RunPass => self.run_rpass_test(),
110             RunPassValgrind => self.run_valgrind_test(),
111             Pretty => self.run_pretty_test(),
112             DebugInfoGdb => self.run_debuginfo_gdb_test(),
113             DebugInfoLldb => self.run_debuginfo_lldb_test(),
114             Codegen => self.run_codegen_test(),
115             Rustdoc => self.run_rustdoc_test(),
116             CodegenUnits => self.run_codegen_units_test(),
117             Incremental => self.run_incremental_test(),
118             RunMake => self.run_rmake_test(),
119             Ui => self.run_ui_test(),
120             MirOpt => self.run_mir_opt_test(),
121         }
122     }
123
124     /// Invoked after all revisions have executed.
125     fn complete_all(&self) {
126         assert!(self.revision.is_none(), "init_all invoked for a revision");
127     }
128
129     fn run_cfail_test(&self) {
130         let proc_res = self.compile_test();
131
132         if proc_res.status.success() {
133             self.fatal_proc_rec(
134                 &format!("{} test compiled successfully!", self.config.mode)[..],
135                 &proc_res);
136         }
137
138         self.check_correct_failure_status(&proc_res);
139
140         if proc_res.status.success() {
141             self.fatal("process did not return an error status");
142         }
143
144         let output_to_check = self.get_output(&proc_res);
145         let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
146         if !expected_errors.is_empty() {
147             if !self.props.error_patterns.is_empty() {
148                 self.fatal("both error pattern and expected errors specified");
149             }
150             self.check_expected_errors(expected_errors, &proc_res);
151         } else {
152             self.check_error_patterns(&output_to_check, &proc_res);
153         }
154         self.check_no_compiler_crash(&proc_res);
155         self.check_forbid_output(&output_to_check, &proc_res);
156     }
157
158     fn run_rfail_test(&self) {
159         let proc_res = self.compile_test();
160
161         if !proc_res.status.success() {
162             self.fatal_proc_rec("compilation failed!", &proc_res);
163         }
164
165         let proc_res = self.exec_compiled_test();
166
167         // The value our Makefile configures valgrind to return on failure
168         const VALGRIND_ERR: i32 = 100;
169         if proc_res.status.code() == Some(VALGRIND_ERR) {
170             self.fatal_proc_rec("run-fail test isn't valgrind-clean!", &proc_res);
171         }
172
173         let output_to_check = self.get_output(&proc_res);
174         self.check_correct_failure_status(&proc_res);
175         self.check_error_patterns(&output_to_check, &proc_res);
176     }
177
178     fn get_output(&self, proc_res: &ProcRes) -> String {
179         if self.props.check_stdout {
180             format!("{}{}", proc_res.stdout, proc_res.stderr)
181         } else {
182             proc_res.stderr.clone()
183         }
184     }
185
186     fn check_correct_failure_status(&self, proc_res: &ProcRes) {
187         // The value the rust runtime returns on failure
188         const RUST_ERR: i32 = 101;
189         if proc_res.status.code() != Some(RUST_ERR) {
190             self.fatal_proc_rec(
191                 &format!("failure produced the wrong error: {}",
192                          proc_res.status),
193                 proc_res);
194         }
195     }
196
197     fn run_rpass_test(&self) {
198         let proc_res = self.compile_test();
199
200         if !proc_res.status.success() {
201             self.fatal_proc_rec("compilation failed!", &proc_res);
202         }
203
204         let proc_res = self.exec_compiled_test();
205
206         if !proc_res.status.success() {
207             self.fatal_proc_rec("test run failed!", &proc_res);
208         }
209     }
210
211     fn run_valgrind_test(&self) {
212         assert!(self.revision.is_none(), "revisions not relevant here");
213
214         if self.config.valgrind_path.is_none() {
215             assert!(!self.config.force_valgrind);
216             return self.run_rpass_test();
217         }
218
219         let mut proc_res = self.compile_test();
220
221         if !proc_res.status.success() {
222             self.fatal_proc_rec("compilation failed!", &proc_res);
223         }
224
225         let mut new_config = self.config.clone();
226         new_config.runtool = new_config.valgrind_path.clone();
227         let new_cx = TestCx { config: &new_config, ..*self };
228         proc_res = new_cx.exec_compiled_test();
229
230         if !proc_res.status.success() {
231             self.fatal_proc_rec("test run failed!", &proc_res);
232         }
233     }
234
235     fn run_pretty_test(&self) {
236         if self.props.pp_exact.is_some() {
237             logv(self.config, "testing for exact pretty-printing".to_owned());
238         } else {
239             logv(self.config, "testing for converging pretty-printing".to_owned());
240         }
241
242         let rounds = match self.props.pp_exact { Some(_) => 1, None => 2 };
243
244         let mut src = String::new();
245         File::open(&self.testpaths.file).unwrap().read_to_string(&mut src).unwrap();
246         let mut srcs = vec!(src);
247
248         let mut round = 0;
249         while round < rounds {
250             logv(self.config, format!("pretty-printing round {} revision {:?}",
251                                       round, self.revision));
252             let proc_res = self.print_source(srcs[round].to_owned(), &self.props.pretty_mode);
253
254             if !proc_res.status.success() {
255                 self.fatal_proc_rec(&format!("pretty-printing failed in round {} revision {:?}",
256                                              round, self.revision),
257                                     &proc_res);
258             }
259
260             let ProcRes{ stdout, .. } = proc_res;
261             srcs.push(stdout);
262             round += 1;
263         }
264
265         let mut expected = match self.props.pp_exact {
266             Some(ref file) => {
267                 let filepath = self.testpaths.file.parent().unwrap().join(file);
268                 let mut s = String::new();
269                 File::open(&filepath).unwrap().read_to_string(&mut s).unwrap();
270                 s
271             }
272             None => { srcs[srcs.len() - 2].clone() }
273         };
274         let mut actual = srcs[srcs.len() - 1].clone();
275
276         if self.props.pp_exact.is_some() {
277             // Now we have to care about line endings
278             let cr = "\r".to_owned();
279             actual = actual.replace(&cr, "").to_owned();
280             expected = expected.replace(&cr, "").to_owned();
281         }
282
283         self.compare_source(&expected, &actual);
284
285         // If we're only making sure that the output matches then just stop here
286         if self.props.pretty_compare_only { return; }
287
288         // Finally, let's make sure it actually appears to remain valid code
289         let proc_res = self.typecheck_source(actual);
290         if !proc_res.status.success() {
291             self.fatal_proc_rec("pretty-printed source does not typecheck", &proc_res);
292         }
293
294         if !self.props.pretty_expanded { return }
295
296         // additionally, run `--pretty expanded` and try to build it.
297         let proc_res = self.print_source(srcs[round].clone(), "expanded");
298         if !proc_res.status.success() {
299             self.fatal_proc_rec("pretty-printing (expanded) failed", &proc_res);
300         }
301
302         let ProcRes{ stdout: expanded_src, .. } = proc_res;
303         let proc_res = self.typecheck_source(expanded_src);
304         if !proc_res.status.success() {
305             self.fatal_proc_rec(
306                 "pretty-printed source (expanded) does not typecheck",
307                 &proc_res);
308         }
309     }
310
311     fn print_source(&self,
312                     src: String,
313                     pretty_type: &str)
314                     -> ProcRes {
315         let aux_dir = self.aux_output_dir_name();
316         self.compose_and_run(self.make_pp_args(pretty_type.to_owned()),
317                              self.props.exec_env.clone(),
318                              self.config.compile_lib_path.to_str().unwrap(),
319                              Some(aux_dir.to_str().unwrap()),
320                              Some(src))
321     }
322
323     fn make_pp_args(&self,
324                     pretty_type: String)
325                     -> ProcArgs {
326         let aux_dir = self.aux_output_dir_name();
327         // FIXME (#9639): This needs to handle non-utf8 paths
328         let mut args = vec!("-".to_owned(),
329                             "-Zunstable-options".to_owned(),
330                             "--unpretty".to_owned(),
331                             pretty_type,
332                             format!("--target={}", self.config.target),
333                             "-L".to_owned(),
334                             aux_dir.to_str().unwrap().to_owned());
335         args.extend(self.split_maybe_args(&self.config.target_rustcflags));
336         args.extend(self.props.compile_flags.iter().cloned());
337         return ProcArgs {
338             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
339             args: args,
340         };
341     }
342
343     fn compare_source(&self,
344                       expected: &str,
345                       actual: &str) {
346         if expected != actual {
347             self.error("pretty-printed source does not match expected source");
348             println!("\n\
349 expected:\n\
350 ------------------------------------------\n\
351 {}\n\
352 ------------------------------------------\n\
353 actual:\n\
354 ------------------------------------------\n\
355 {}\n\
356 ------------------------------------------\n\
357 \n",
358                      expected, actual);
359             panic!();
360         }
361     }
362
363     fn typecheck_source(&self, src: String) -> ProcRes {
364         let args = self.make_typecheck_args();
365         self.compose_and_run_compiler(args, Some(src))
366     }
367
368     fn make_typecheck_args(&self) -> ProcArgs {
369         let aux_dir = self.aux_output_dir_name();
370         let target = if self.props.force_host {
371             &*self.config.host
372         } else {
373             &*self.config.target
374         };
375
376         let out_dir = self.output_base_name().with_extension("pretty-out");
377         let _ = fs::remove_dir_all(&out_dir);
378         self.create_dir_racy(&out_dir);
379
380         // FIXME (#9639): This needs to handle non-utf8 paths
381         let mut args = vec!("-".to_owned(),
382                             "-Zno-trans".to_owned(),
383                             "--out-dir".to_owned(),
384                             out_dir.to_str().unwrap().to_owned(),
385                             format!("--target={}", target),
386                             "-L".to_owned(),
387                             self.config.build_base.to_str().unwrap().to_owned(),
388                             "-L".to_owned(),
389                             aux_dir.to_str().unwrap().to_owned());
390         if let Some(revision) = self.revision {
391             args.extend(vec![
392                 format!("--cfg"),
393                 format!("{}", revision),
394             ]);
395         }
396         args.extend(self.split_maybe_args(&self.config.target_rustcflags));
397         args.extend(self.props.compile_flags.iter().cloned());
398         // FIXME (#9639): This needs to handle non-utf8 paths
399         return ProcArgs {
400             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
401             args: args,
402         };
403     }
404
405     fn run_debuginfo_gdb_test(&self) {
406         assert!(self.revision.is_none(), "revisions not relevant here");
407
408         let config = Config {
409             target_rustcflags: self.cleanup_debug_info_options(&self.config.target_rustcflags),
410             host_rustcflags: self.cleanup_debug_info_options(&self.config.host_rustcflags),
411             .. self.config.clone()
412         };
413
414         let test_cx = TestCx {
415             config: &config,
416             ..*self
417         };
418
419         test_cx.run_debuginfo_gdb_test_no_opt();
420     }
421
422     fn run_debuginfo_gdb_test_no_opt(&self) {
423         let DebuggerCommands {
424             commands,
425             check_lines,
426             breakpoint_lines
427         } = self.parse_debugger_commands("gdb");
428         let mut cmds = commands.join("\n");
429
430         // compile test file (it should have 'compile-flags:-g' in the header)
431         let compiler_run_result = self.compile_test();
432         if !compiler_run_result.status.success() {
433             self.fatal_proc_rec("compilation failed!", &compiler_run_result);
434         }
435
436         let exe_file = self.make_exe_name();
437
438         let debugger_run_result;
439         match &*self.config.target {
440             "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
441
442                 cmds = cmds.replace("run", "continue");
443
444                 let tool_path = match self.config.android_cross_path.to_str() {
445                     Some(x) => x.to_owned(),
446                     None => self.fatal("cannot find android cross path")
447                 };
448
449                 // write debugger script
450                 let mut script_str = String::with_capacity(2048);
451                 script_str.push_str(&format!("set charset {}\n", Self::charset()));
452                 script_str.push_str(&format!("set sysroot {}\n", tool_path));
453                 script_str.push_str(&format!("file {}\n", exe_file.to_str().unwrap()));
454                 script_str.push_str("target remote :5039\n");
455                 script_str.push_str(&format!("set solib-search-path \
456                                               ./{}/stage2/lib/rustlib/{}/lib/\n",
457                                              self.config.host, self.config.target));
458                 for line in &breakpoint_lines {
459                     script_str.push_str(&format!("break {:?}:{}\n",
460                                                  self.testpaths.file.file_name()
461                                                  .unwrap()
462                                                  .to_string_lossy(),
463                                                  *line)[..]);
464                 }
465                 script_str.push_str(&cmds);
466                 script_str.push_str("\nquit\n");
467
468                 debug!("script_str = {}", script_str);
469                 self.dump_output_file(&script_str, "debugger.script");
470
471
472                 procsrv::run("",
473                              &self.config.adb_path,
474                              None,
475                              &[
476                                  "push".to_owned(),
477                                  exe_file.to_str().unwrap().to_owned(),
478                                  self.config.adb_test_dir.clone()
479                              ],
480                              vec!(("".to_owned(), "".to_owned())),
481                              Some("".to_owned()))
482                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
483
484                 procsrv::run("",
485                              &self.config.adb_path,
486                              None,
487                              &[
488                                  "forward".to_owned(),
489                                  "tcp:5039".to_owned(),
490                                  "tcp:5039".to_owned()
491                              ],
492                              vec!(("".to_owned(), "".to_owned())),
493                              Some("".to_owned()))
494                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
495
496                 let adb_arg = format!("export LD_LIBRARY_PATH={}; \
497                                        gdbserver{} :5039 {}/{}",
498                                       self.config.adb_test_dir.clone(),
499                                       if self.config.target.contains("aarch64")
500                                       {"64"} else {""},
501                                       self.config.adb_test_dir.clone(),
502                                       exe_file.file_name().unwrap().to_str()
503                                       .unwrap());
504
505                 let mut process = procsrv::run_background("",
506                                                           &self.config.adb_path
507                                                           ,
508                                                           None,
509                                                           &[
510                                                               "shell".to_owned(),
511                                                               adb_arg.clone()
512                                                           ],
513                                                           vec!(("".to_owned(),
514                                                                 "".to_owned())),
515                                                           Some("".to_owned()))
516                     .expect(&format!("failed to exec `{:?}`", self.config.adb_path));
517                 loop {
518                     //waiting 1 second for gdbserver start
519                     ::std::thread::sleep(::std::time::Duration::new(1,0));
520                     if TcpStream::connect("127.0.0.1:5039").is_ok() {
521                         break
522                     }
523                 }
524
525                 let debugger_script = self.make_out_name("debugger.script");
526                 // FIXME (#9639): This needs to handle non-utf8 paths
527                 let debugger_opts =
528                     vec!("-quiet".to_owned(),
529                          "-batch".to_owned(),
530                          "-nx".to_owned(),
531                          format!("-command={}", debugger_script.to_str().unwrap()));
532
533                 let mut gdb_path = tool_path;
534                 gdb_path.push_str(&format!("/bin/{}-gdb", self.config.target));
535                 let procsrv::Result {
536                     out,
537                     err,
538                     status
539                 } = procsrv::run("",
540                                  &gdb_path,
541                                  None,
542                                  &debugger_opts,
543                                  vec!(("".to_owned(), "".to_owned())),
544                                  None)
545                     .expect(&format!("failed to exec `{:?}`", gdb_path));
546                 let cmdline = {
547                     let cmdline = self.make_cmdline("",
548                                                     &format!("{}-gdb", self.config.target),
549                                                     &debugger_opts);
550                     logv(self.config, format!("executing {}", cmdline));
551                     cmdline
552                 };
553
554                 debugger_run_result = ProcRes {
555                     status: Status::Normal(status),
556                     stdout: out,
557                     stderr: err,
558                     cmdline: cmdline
559                 };
560                 if process.kill().is_err() {
561                     println!("Adb process is already finished.");
562                 }
563             }
564
565             _=> {
566                 let rust_src_root = self.find_rust_src_root()
567                                         .expect("Could not find Rust source root");
568                 let rust_pp_module_rel_path = Path::new("./src/etc");
569                 let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
570                                                            .to_str()
571                                                            .unwrap()
572                                                            .to_owned();
573                 // write debugger script
574                 let mut script_str = String::with_capacity(2048);
575                 script_str.push_str(&format!("set charset {}\n", Self::charset()));
576                 script_str.push_str("show version\n");
577
578                 match self.config.gdb_version {
579                     Some(ref version) => {
580                         println!("NOTE: compiletest thinks it is using GDB version {}",
581                                  version);
582
583                         if header::gdb_version_to_int(version) >
584                             header::gdb_version_to_int("7.4") {
585                                 // Add the directory containing the pretty printers to
586                                 // GDB's script auto loading safe path
587                                 script_str.push_str(
588                                     &format!("add-auto-load-safe-path {}\n",
589                                              rust_pp_module_abs_path.replace(r"\", r"\\"))
590                                 );
591                             }
592                     }
593                     _ => {
594                         println!("NOTE: compiletest does not know which version of \
595                                   GDB it is using");
596                     }
597                 }
598
599                 // The following line actually doesn't have to do anything with
600                 // pretty printing, it just tells GDB to print values on one line:
601                 script_str.push_str("set print pretty off\n");
602
603                 // Add the pretty printer directory to GDB's source-file search path
604                 script_str.push_str(&format!("directory {}\n",
605                                              rust_pp_module_abs_path));
606
607                 // Load the target executable
608                 script_str.push_str(&format!("file {}\n",
609                                              exe_file.to_str().unwrap()
610                                              .replace(r"\", r"\\")));
611
612                 // Add line breakpoints
613                 for line in &breakpoint_lines {
614                     script_str.push_str(&format!("break '{}':{}\n",
615                                                  self.testpaths.file.file_name().unwrap()
616                                                  .to_string_lossy(),
617                                                  *line));
618                 }
619
620                 script_str.push_str(&cmds);
621                 script_str.push_str("\nquit\n");
622
623                 debug!("script_str = {}", script_str);
624                 self.dump_output_file(&script_str, "debugger.script");
625
626                 // run debugger script with gdb
627                 fn debugger() -> &'static str {
628                     if cfg!(windows) {"gdb.exe"} else {"gdb"}
629                 }
630
631                 let debugger_script = self.make_out_name("debugger.script");
632
633                 // FIXME (#9639): This needs to handle non-utf8 paths
634                 let debugger_opts =
635                     vec!("-quiet".to_owned(),
636                          "-batch".to_owned(),
637                          "-nx".to_owned(),
638                          format!("-command={}", debugger_script.to_str().unwrap()));
639
640                 let proc_args = ProcArgs {
641                     prog: debugger().to_owned(),
642                     args: debugger_opts,
643                 };
644
645                 let environment = vec![("PYTHONPATH".to_owned(), rust_pp_module_abs_path)];
646
647                 debugger_run_result =
648                     self.compose_and_run(proc_args,
649                                          environment,
650                                          self.config.run_lib_path.to_str().unwrap(),
651                                          None,
652                                          None);
653             }
654         }
655
656         if !debugger_run_result.status.success() {
657             self.fatal("gdb failed to execute");
658         }
659
660         self.check_debugger_output(&debugger_run_result, &check_lines);
661     }
662
663     fn find_rust_src_root(&self) -> Option<PathBuf> {
664         let mut path = self.config.src_base.clone();
665         let path_postfix = Path::new("src/etc/lldb_batchmode.py");
666
667         while path.pop() {
668             if path.join(&path_postfix).is_file() {
669                 return Some(path);
670             }
671         }
672
673         return None;
674     }
675
676     fn run_debuginfo_lldb_test(&self) {
677         assert!(self.revision.is_none(), "revisions not relevant here");
678
679         if self.config.lldb_python_dir.is_none() {
680             self.fatal("Can't run LLDB test because LLDB's python path is not set.");
681         }
682
683         let config = Config {
684             target_rustcflags: self.cleanup_debug_info_options(&self.config.target_rustcflags),
685             host_rustcflags: self.cleanup_debug_info_options(&self.config.host_rustcflags),
686             .. self.config.clone()
687         };
688
689
690         let test_cx = TestCx {
691             config: &config,
692             ..*self
693         };
694
695         test_cx.run_debuginfo_lldb_test_no_opt();
696     }
697
698     fn run_debuginfo_lldb_test_no_opt(&self) {
699         // compile test file (it should have 'compile-flags:-g' in the header)
700         let compile_result = self.compile_test();
701         if !compile_result.status.success() {
702             self.fatal_proc_rec("compilation failed!", &compile_result);
703         }
704
705         let exe_file = self.make_exe_name();
706
707         match self.config.lldb_version {
708             Some(ref version) => {
709                 println!("NOTE: compiletest thinks it is using LLDB version {}",
710                          version);
711             }
712             _ => {
713                 println!("NOTE: compiletest does not know which version of \
714                           LLDB it is using");
715             }
716         }
717
718         // Parse debugger commands etc from test files
719         let DebuggerCommands {
720             commands,
721             check_lines,
722             breakpoint_lines,
723             ..
724         } = self.parse_debugger_commands("lldb");
725
726         // Write debugger script:
727         // We don't want to hang when calling `quit` while the process is still running
728         let mut script_str = String::from("settings set auto-confirm true\n");
729
730         // Make LLDB emit its version, so we have it documented in the test output
731         script_str.push_str("version\n");
732
733         // Switch LLDB into "Rust mode"
734         let rust_src_root = self.find_rust_src_root().expect("Could not find Rust source root");
735         let rust_pp_module_rel_path = Path::new("./src/etc/lldb_rust_formatters.py");
736         let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
737                                                    .to_str()
738                                                    .unwrap()
739                                                    .to_owned();
740
741         script_str.push_str(&format!("command script import {}\n",
742                                      &rust_pp_module_abs_path[..])[..]);
743         script_str.push_str("type summary add --no-value ");
744         script_str.push_str("--python-function lldb_rust_formatters.print_val ");
745         script_str.push_str("-x \".*\" --category Rust\n");
746         script_str.push_str("type category enable Rust\n");
747
748         // Set breakpoints on every line that contains the string "#break"
749         let source_file_name = self.testpaths.file.file_name().unwrap().to_string_lossy();
750         for line in &breakpoint_lines {
751             script_str.push_str(&format!("breakpoint set --file '{}' --line {}\n",
752                                          source_file_name,
753                                          line));
754         }
755
756         // Append the other commands
757         for line in &commands {
758             script_str.push_str(line);
759             script_str.push_str("\n");
760         }
761
762         // Finally, quit the debugger
763         script_str.push_str("\nquit\n");
764
765         // Write the script into a file
766         debug!("script_str = {}", script_str);
767         self.dump_output_file(&script_str, "debugger.script");
768         let debugger_script = self.make_out_name("debugger.script");
769
770         // Let LLDB execute the script via lldb_batchmode.py
771         let debugger_run_result = self.run_lldb(&exe_file,
772                                                 &debugger_script,
773                                                 &rust_src_root);
774
775         if !debugger_run_result.status.success() {
776             self.fatal_proc_rec("Error while running LLDB", &debugger_run_result);
777         }
778
779         self.check_debugger_output(&debugger_run_result, &check_lines);
780     }
781
782     fn run_lldb(&self,
783                 test_executable: &Path,
784                 debugger_script: &Path,
785                 rust_src_root: &Path)
786                 -> ProcRes {
787         // Prepare the lldb_batchmode which executes the debugger script
788         let lldb_script_path = rust_src_root.join("src/etc/lldb_batchmode.py");
789         self.cmd2procres(Command::new(&self.config.lldb_python)
790                          .arg(&lldb_script_path)
791                          .arg(test_executable)
792                          .arg(debugger_script)
793                          .env("PYTHONPATH",
794                               self.config.lldb_python_dir.as_ref().unwrap()))
795     }
796
797     fn cmd2procres(&self, cmd: &mut Command) -> ProcRes {
798         let (status, out, err) = match cmd.output() {
799             Ok(Output { status, stdout, stderr }) => {
800                 (status,
801                  String::from_utf8(stdout).unwrap(),
802                  String::from_utf8(stderr).unwrap())
803             },
804             Err(e) => {
805                 self.fatal(&format!("Failed to setup Python process for \
806                                       LLDB script: {}", e))
807             }
808         };
809
810         self.dump_output(&out, &err);
811         ProcRes {
812             status: Status::Normal(status),
813             stdout: out,
814             stderr: err,
815             cmdline: format!("{:?}", cmd)
816         }
817     }
818
819     fn parse_debugger_commands(&self, debugger_prefix: &str) -> DebuggerCommands {
820         let command_directive = format!("{}-command", debugger_prefix);
821         let check_directive = format!("{}-check", debugger_prefix);
822
823         let mut breakpoint_lines = vec!();
824         let mut commands = vec!();
825         let mut check_lines = vec!();
826         let mut counter = 1;
827         let reader = BufReader::new(File::open(&self.testpaths.file).unwrap());
828         for line in reader.lines() {
829             match line {
830                 Ok(line) => {
831                     if line.contains("#break") {
832                         breakpoint_lines.push(counter);
833                     }
834
835                     header::parse_name_value_directive(
836                         &line,
837                         &command_directive).map(|cmd| {
838                             commands.push(cmd)
839                         });
840
841                     header::parse_name_value_directive(
842                         &line,
843                         &check_directive).map(|cmd| {
844                             check_lines.push(cmd)
845                         });
846                 }
847                 Err(e) => {
848                     self.fatal(&format!("Error while parsing debugger commands: {}", e))
849                 }
850             }
851             counter += 1;
852         }
853
854         DebuggerCommands {
855             commands: commands,
856             check_lines: check_lines,
857             breakpoint_lines: breakpoint_lines,
858         }
859     }
860
861     fn cleanup_debug_info_options(&self, options: &Option<String>) -> Option<String> {
862         if options.is_none() {
863             return None;
864         }
865
866         // Remove options that are either unwanted (-O) or may lead to duplicates due to RUSTFLAGS.
867         let options_to_remove = [
868             "-O".to_owned(),
869             "-g".to_owned(),
870             "--debuginfo".to_owned()
871         ];
872         let new_options =
873             self.split_maybe_args(options).into_iter()
874                                           .filter(|x| !options_to_remove.contains(x))
875                                           .collect::<Vec<String>>();
876
877         Some(new_options.join(" "))
878     }
879
880     fn check_debugger_output(&self, debugger_run_result: &ProcRes, check_lines: &[String]) {
881         let num_check_lines = check_lines.len();
882
883         let mut check_line_index = 0;
884         for line in debugger_run_result.stdout.lines() {
885             if check_line_index >= num_check_lines {
886                 break;
887             }
888
889             if check_single_line(line, &(check_lines[check_line_index])[..]) {
890                 check_line_index += 1;
891             }
892         }
893         if check_line_index != num_check_lines && num_check_lines > 0 {
894             self.fatal_proc_rec(&format!("line not found in debugger output: {}",
895                                          check_lines[check_line_index]),
896                                 debugger_run_result);
897         }
898
899         fn check_single_line(line: &str, check_line: &str) -> bool {
900             // Allow check lines to leave parts unspecified (e.g., uninitialized
901             // bits in the  wrong case of an enum) with the notation "[...]".
902             let line = line.trim();
903             let check_line = check_line.trim();
904             let can_start_anywhere = check_line.starts_with("[...]");
905             let can_end_anywhere = check_line.ends_with("[...]");
906
907             let check_fragments: Vec<&str> = check_line.split("[...]")
908                                                        .filter(|frag| !frag.is_empty())
909                                                        .collect();
910             if check_fragments.is_empty() {
911                 return true;
912             }
913
914             let (mut rest, first_fragment) = if can_start_anywhere {
915                 match line.find(check_fragments[0]) {
916                     Some(pos) => (&line[pos + check_fragments[0].len() ..], 1),
917                     None => return false
918                 }
919             } else {
920                 (line, 0)
921             };
922
923             for fragment_index in first_fragment .. check_fragments.len() {
924                 let current_fragment = check_fragments[fragment_index];
925                 match rest.find(current_fragment) {
926                     Some(pos) => {
927                         rest = &rest[pos + current_fragment.len() .. ];
928                     }
929                     None => return false
930                 }
931             }
932
933             if !can_end_anywhere && !rest.is_empty() {
934                 return false;
935             }
936
937             return true;
938         }
939     }
940
941     fn check_error_patterns(&self,
942                             output_to_check: &str,
943                             proc_res: &ProcRes) {
944         if self.props.error_patterns.is_empty() {
945             self.fatal(&format!("no error pattern specified in {:?}",
946                                 self.testpaths.file.display()));
947         }
948         let mut next_err_idx = 0;
949         let mut next_err_pat = self.props.error_patterns[next_err_idx].trim();
950         let mut done = false;
951         for line in output_to_check.lines() {
952             if line.contains(next_err_pat) {
953                 debug!("found error pattern {}", next_err_pat);
954                 next_err_idx += 1;
955                 if next_err_idx == self.props.error_patterns.len() {
956                     debug!("found all error patterns");
957                     done = true;
958                     break;
959                 }
960                 next_err_pat = self.props.error_patterns[next_err_idx].trim();
961             }
962         }
963         if done { return; }
964
965         let missing_patterns = &self.props.error_patterns[next_err_idx..];
966         if missing_patterns.len() == 1 {
967             self.fatal_proc_rec(
968                 &format!("error pattern '{}' not found!", missing_patterns[0]),
969                 proc_res);
970         } else {
971             for pattern in missing_patterns {
972                 self.error(&format!("error pattern '{}' not found!", *pattern));
973             }
974             self.fatal_proc_rec("multiple error patterns not found", proc_res);
975         }
976     }
977
978     fn check_no_compiler_crash(&self, proc_res: &ProcRes) {
979         for line in proc_res.stderr.lines() {
980             if line.starts_with("error: internal compiler error:") {
981                 self.fatal_proc_rec("compiler encountered internal error", proc_res);
982             }
983         }
984     }
985
986     fn check_forbid_output(&self,
987                            output_to_check: &str,
988                            proc_res: &ProcRes) {
989         for pat in &self.props.forbid_output {
990             if output_to_check.contains(pat) {
991                 self.fatal_proc_rec("forbidden pattern found in compiler output", proc_res);
992             }
993         }
994     }
995
996     fn check_expected_errors(&self,
997                              expected_errors: Vec<errors::Error>,
998                              proc_res: &ProcRes) {
999         if proc_res.status.success() {
1000             self.fatal_proc_rec("process did not return an error status", proc_res);
1001         }
1002
1003         let file_name =
1004             format!("{}", self.testpaths.file.display())
1005             .replace(r"\", "/"); // on windows, translate all '\' path separators to '/'
1006
1007         // If the testcase being checked contains at least one expected "help"
1008         // message, then we'll ensure that all "help" messages are expected.
1009         // Otherwise, all "help" messages reported by the compiler will be ignored.
1010         // This logic also applies to "note" messages.
1011         let expect_help = expected_errors.iter().any(|ee| ee.kind == Some(ErrorKind::Help));
1012         let expect_note = expected_errors.iter().any(|ee| ee.kind == Some(ErrorKind::Note));
1013
1014         // Parse the JSON output from the compiler and extract out the messages.
1015         let actual_errors = json::parse_output(&file_name, &proc_res.stderr, &proc_res);
1016         let mut unexpected = Vec::new();
1017         let mut found = vec![false; expected_errors.len()];
1018         for actual_error in &actual_errors {
1019             let opt_index =
1020                 expected_errors
1021                 .iter()
1022                 .enumerate()
1023                 .position(|(index, expected_error)| {
1024                     !found[index] &&
1025                         actual_error.line_num == expected_error.line_num &&
1026                         (expected_error.kind.is_none() ||
1027                          actual_error.kind == expected_error.kind) &&
1028                         actual_error.msg.contains(&expected_error.msg)
1029                 });
1030
1031             match opt_index {
1032                 Some(index) => {
1033                     // found a match, everybody is happy
1034                     assert!(!found[index]);
1035                     found[index] = true;
1036                 }
1037
1038                 None => {
1039                     if self.is_unexpected_compiler_message(actual_error, expect_help, expect_note) {
1040                         self.error(
1041                             &format!("{}:{}: unexpected {:?}: '{}'",
1042                                      file_name,
1043                                      actual_error.line_num,
1044                                      actual_error.kind.as_ref()
1045                                      .map_or(String::from("message"),
1046                                              |k| k.to_string()),
1047                                      actual_error.msg));
1048                         unexpected.push(actual_error.clone());
1049                     }
1050                 }
1051             }
1052         }
1053
1054         let mut not_found = Vec::new();
1055         // anything not yet found is a problem
1056         for (index, expected_error) in expected_errors.iter().enumerate() {
1057             if !found[index] {
1058                 self.error(
1059                     &format!("{}:{}: expected {} not found: {}",
1060                              file_name,
1061                              expected_error.line_num,
1062                              expected_error.kind.as_ref()
1063                              .map_or("message".into(),
1064                                      |k| k.to_string()),
1065                              expected_error.msg));
1066                 not_found.push(expected_error.clone());
1067             }
1068         }
1069
1070         if unexpected.len() > 0 || not_found.len() > 0 {
1071             self.error(
1072                 &format!("{} unexpected errors found, {} expected errors not found",
1073                          unexpected.len(), not_found.len()));
1074             print!("status: {}\ncommand: {}\n",
1075                    proc_res.status, proc_res.cmdline);
1076             if unexpected.len() > 0 {
1077                 println!("unexpected errors (from JSON output): {:#?}\n", unexpected);
1078             }
1079             if not_found.len() > 0 {
1080                 println!("not found errors (from test file): {:#?}\n", not_found);
1081             }
1082             panic!();
1083         }
1084     }
1085
1086     /// Returns true if we should report an error about `actual_error`,
1087     /// which did not match any of the expected error. We always require
1088     /// errors/warnings to be explicitly listed, but only require
1089     /// helps/notes if there are explicit helps/notes given.
1090     fn is_unexpected_compiler_message(&self,
1091                                       actual_error: &Error,
1092                                       expect_help: bool,
1093                                       expect_note: bool)
1094                                       -> bool {
1095         match actual_error.kind {
1096             Some(ErrorKind::Help) => expect_help,
1097             Some(ErrorKind::Note) => expect_note,
1098             Some(ErrorKind::Error) => true,
1099             Some(ErrorKind::Warning) => true,
1100             Some(ErrorKind::Suggestion) => false,
1101             None => false
1102         }
1103     }
1104
1105     fn compile_test(&self) -> ProcRes {
1106         let aux_dir = self.aux_output_dir_name();
1107         // FIXME (#9639): This needs to handle non-utf8 paths
1108         let link_args = vec!("-L".to_owned(),
1109                              aux_dir.to_str().unwrap().to_owned());
1110         let args = self.make_compile_args(link_args,
1111                                           &self.testpaths.file,
1112                                           TargetLocation::ThisFile(self.make_exe_name()));
1113         self.compose_and_run_compiler(args, None)
1114     }
1115
1116     fn document(&self, out_dir: &Path) -> ProcRes {
1117         if self.props.build_aux_docs {
1118             for rel_ab in &self.props.aux_builds {
1119                 let aux_testpaths = self.compute_aux_test_paths(rel_ab);
1120                 let aux_props = self.props.from_aux_file(&aux_testpaths.file, self.revision);
1121                 let aux_cx = TestCx {
1122                     config: self.config,
1123                     props: &aux_props,
1124                     testpaths: &aux_testpaths,
1125                     revision: self.revision
1126                 };
1127                 let auxres = aux_cx.document(out_dir);
1128                 if !auxres.status.success() {
1129                     return auxres;
1130                 }
1131             }
1132         }
1133
1134         let aux_dir = self.aux_output_dir_name();
1135         let mut args = vec!["-L".to_owned(),
1136                             aux_dir.to_str().unwrap().to_owned(),
1137                             "-o".to_owned(),
1138                             out_dir.to_str().unwrap().to_owned(),
1139                             self.testpaths.file.to_str().unwrap().to_owned()];
1140         args.extend(self.props.compile_flags.iter().cloned());
1141         let args = ProcArgs {
1142             prog: self.config.rustdoc_path.to_str().unwrap().to_owned(),
1143             args: args,
1144         };
1145         self.compose_and_run_compiler(args, None)
1146     }
1147
1148     fn exec_compiled_test(&self) -> ProcRes {
1149         let env = self.props.exec_env.clone();
1150
1151         match &*self.config.target {
1152
1153             "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
1154                 self._arm_exec_compiled_test(env)
1155             }
1156
1157             _=> {
1158                 let aux_dir = self.aux_output_dir_name();
1159                 self.compose_and_run(self.make_run_args(),
1160                                      env,
1161                                      self.config.run_lib_path.to_str().unwrap(),
1162                                      Some(aux_dir.to_str().unwrap()),
1163                                      None)
1164             }
1165         }
1166     }
1167
1168     /// For each `aux-build: foo/bar` annotation, we check to find the
1169     /// file in a `aux` directory relative to the test itself.
1170     fn compute_aux_test_paths(&self, rel_ab: &str) -> TestPaths {
1171         let test_ab = self.testpaths.file
1172                                     .parent()
1173                                     .expect("test file path has no parent")
1174                                     .join("auxiliary")
1175                                     .join(rel_ab);
1176         if !test_ab.exists() {
1177             self.fatal(&format!("aux-build `{}` source not found", test_ab.display()))
1178         }
1179
1180         TestPaths {
1181             file: test_ab,
1182             base: self.testpaths.base.clone(),
1183             relative_dir: self.testpaths.relative_dir
1184                                         .join("auxiliary")
1185                                         .join(rel_ab)
1186                                         .parent()
1187                                         .expect("aux-build path has no parent")
1188                                         .to_path_buf()
1189         }
1190     }
1191
1192     fn compose_and_run_compiler(&self, args: ProcArgs, input: Option<String>) -> ProcRes {
1193         if !self.props.aux_builds.is_empty() {
1194             self.create_dir_racy(&self.aux_output_dir_name());
1195         }
1196
1197         let aux_dir = self.aux_output_dir_name();
1198         // FIXME (#9639): This needs to handle non-utf8 paths
1199         let extra_link_args = vec!["-L".to_owned(),
1200                                    aux_dir.to_str().unwrap().to_owned()];
1201
1202         for rel_ab in &self.props.aux_builds {
1203             let aux_testpaths = self.compute_aux_test_paths(rel_ab);
1204             let aux_props = self.props.from_aux_file(&aux_testpaths.file, self.revision);
1205             let mut crate_type = if aux_props.no_prefer_dynamic {
1206                 Vec::new()
1207             } else {
1208                 // We primarily compile all auxiliary libraries as dynamic libraries
1209                 // to avoid code size bloat and large binaries as much as possible
1210                 // for the test suite (otherwise including libstd statically in all
1211                 // executables takes up quite a bit of space).
1212                 //
1213                 // For targets like MUSL or Emscripten, however, there is no support for
1214                 // dynamic libraries so we just go back to building a normal library. Note,
1215                 // however, that for MUSL if the library is built with `force_host` then
1216                 // it's ok to be a dylib as the host should always support dylibs.
1217                 if (self.config.target.contains("musl") && !aux_props.force_host) ||
1218                     self.config.target.contains("emscripten")
1219                 {
1220                     vec!("--crate-type=lib".to_owned())
1221                 } else {
1222                     vec!("--crate-type=dylib".to_owned())
1223                 }
1224             };
1225             crate_type.extend(extra_link_args.clone());
1226             let aux_output = {
1227                 let f = self.make_lib_name(&self.testpaths.file);
1228                 let parent = f.parent().unwrap();
1229                 TargetLocation::ThisDirectory(parent.to_path_buf())
1230             };
1231             let aux_cx = TestCx {
1232                 config: self.config,
1233                 props: &aux_props,
1234                 testpaths: &aux_testpaths,
1235                 revision: self.revision
1236             };
1237             let aux_args = aux_cx.make_compile_args(crate_type, &aux_testpaths.file, aux_output);
1238             let auxres = aux_cx.compose_and_run(aux_args,
1239                                                 Vec::new(),
1240                                                 aux_cx.config.compile_lib_path.to_str().unwrap(),
1241                                                 Some(aux_dir.to_str().unwrap()),
1242                                                 None);
1243             if !auxres.status.success() {
1244                 self.fatal_proc_rec(
1245                     &format!("auxiliary build of {:?} failed to compile: ",
1246                              aux_testpaths.file.display()),
1247                     &auxres);
1248             }
1249
1250             match &*self.config.target {
1251                 "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
1252                     self._arm_push_aux_shared_library();
1253                 }
1254                 _ => {}
1255             }
1256         }
1257
1258         self.compose_and_run(args,
1259                              self.props.rustc_env.clone(),
1260                              self.config.compile_lib_path.to_str().unwrap(),
1261                              Some(aux_dir.to_str().unwrap()),
1262                              input)
1263     }
1264
1265     // Like std::fs::create_dir_all, except handles concurrent calls among multiple
1266     // threads or processes.
1267     fn create_dir_racy(&self, path: &Path) {
1268         match fs::create_dir(path) {
1269             Ok(()) => return,
1270             Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => return,
1271             Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
1272             Err(e) => panic!("failed to create dir {:?}: {}", path, e),
1273         }
1274         self.create_dir_racy(path.parent().unwrap());
1275         match fs::create_dir(path) {
1276             Ok(()) => {}
1277             Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => {}
1278             Err(e) => panic!("failed to create dir {:?}: {}", path, e),
1279         }
1280     }
1281
1282     fn compose_and_run(&self,
1283                        ProcArgs{ args, prog }: ProcArgs,
1284                        procenv: Vec<(String, String)> ,
1285                        lib_path: &str,
1286                        aux_path: Option<&str>,
1287                        input: Option<String>) -> ProcRes {
1288         return self.program_output(lib_path, prog, aux_path, args, procenv, input);
1289     }
1290
1291     fn make_compile_args(&self,
1292                          extras: Vec<String> ,
1293                          input_file: &Path,
1294                          output_file: TargetLocation)
1295                          -> ProcArgs
1296     {
1297         let target = if self.props.force_host {
1298             &*self.config.host
1299         } else {
1300             &*self.config.target
1301         };
1302
1303         // FIXME (#9639): This needs to handle non-utf8 paths
1304         let mut args = vec!(input_file.to_str().unwrap().to_owned(),
1305                             "-L".to_owned(),
1306                             self.config.build_base.to_str().unwrap().to_owned(),
1307                             format!("--target={}", target));
1308
1309         if let Some(revision) = self.revision {
1310             args.extend(vec![
1311                 format!("--cfg"),
1312                 format!("{}", revision),
1313             ]);
1314         }
1315
1316         if let Some(ref incremental_dir) = self.props.incremental_dir {
1317             args.extend(vec![
1318                 format!("-Z"),
1319                 format!("incremental={}", incremental_dir.display()),
1320             ]);
1321         }
1322
1323
1324         match self.config.mode {
1325             CompileFail |
1326             ParseFail |
1327             Incremental => {
1328                 // If we are extracting and matching errors in the new
1329                 // fashion, then you want JSON mode. Old-skool error
1330                 // patterns still match the raw compiler output.
1331                 if self.props.error_patterns.is_empty() {
1332                     args.extend(["--error-format",
1333                                  "json",
1334                                  "-Z",
1335                                  "unstable-options"]
1336                                 .iter()
1337                                 .map(|s| s.to_string()));
1338                 }
1339             }
1340             MirOpt => {
1341                 args.extend(["-Z",
1342                              "dump-mir=all",
1343                              "-Z"]
1344                             .iter()
1345                             .map(|s| s.to_string()));
1346
1347
1348                 let mir_dump_dir = self.get_mir_dump_dir();
1349                 self.create_dir_racy(mir_dump_dir.as_path());
1350                 let mut dir_opt = "dump-mir-dir=".to_string();
1351                 dir_opt.push_str(mir_dump_dir.to_str().unwrap());
1352                 debug!("dir_opt: {:?}", dir_opt);
1353
1354                 args.push(dir_opt);
1355             }
1356             RunFail |
1357             RunPass |
1358             RunPassValgrind |
1359             Pretty |
1360             DebugInfoGdb |
1361             DebugInfoLldb |
1362             Codegen |
1363             Rustdoc |
1364             RunMake |
1365             Ui |
1366             CodegenUnits => {
1367                 // do not use JSON output
1368             }
1369         }
1370
1371         args.extend_from_slice(&extras);
1372         if !self.props.no_prefer_dynamic {
1373             args.push("-C".to_owned());
1374             args.push("prefer-dynamic".to_owned());
1375         }
1376         let path = match output_file {
1377             TargetLocation::ThisFile(path) => {
1378                 args.push("-o".to_owned());
1379                 path
1380             }
1381             TargetLocation::ThisDirectory(path) => {
1382                 args.push("--out-dir".to_owned());
1383                 path
1384             }
1385         };
1386         args.push(path.to_str().unwrap().to_owned());
1387         if self.props.force_host {
1388             args.extend(self.split_maybe_args(&self.config.host_rustcflags));
1389         } else {
1390             args.extend(self.split_maybe_args(&self.config.target_rustcflags));
1391         }
1392         args.extend(self.props.compile_flags.iter().cloned());
1393         return ProcArgs {
1394             prog: self.config.rustc_path.to_str().unwrap().to_owned(),
1395             args: args,
1396         };
1397     }
1398
1399     fn make_lib_name(&self, auxfile: &Path) -> PathBuf {
1400         // what we return here is not particularly important, as it
1401         // happens; rustc ignores everything except for the directory.
1402         let auxname = self.output_testname(auxfile);
1403         self.aux_output_dir_name().join(&auxname)
1404     }
1405
1406     fn make_exe_name(&self) -> PathBuf {
1407         let mut f = self.output_base_name();
1408         // FIXME: This is using the host architecture exe suffix, not target!
1409         if self.config.target == "asmjs-unknown-emscripten" {
1410             let mut fname = f.file_name().unwrap().to_os_string();
1411             fname.push(".js");
1412             f.set_file_name(&fname);
1413         } else if !env::consts::EXE_SUFFIX.is_empty() {
1414             let mut fname = f.file_name().unwrap().to_os_string();
1415             fname.push(env::consts::EXE_SUFFIX);
1416             f.set_file_name(&fname);
1417         }
1418         f
1419     }
1420
1421     fn make_run_args(&self) -> ProcArgs {
1422         // If we've got another tool to run under (valgrind),
1423         // then split apart its command
1424         let mut args = self.split_maybe_args(&self.config.runtool);
1425
1426         // If this is emscripten, then run tests under nodejs
1427         if self.config.target == "asmjs-unknown-emscripten" {
1428             args.push("nodejs".to_owned());
1429         }
1430
1431         let exe_file = self.make_exe_name();
1432
1433         // FIXME (#9639): This needs to handle non-utf8 paths
1434         args.push(exe_file.to_str().unwrap().to_owned());
1435
1436         // Add the arguments in the run_flags directive
1437         args.extend(self.split_maybe_args(&self.props.run_flags));
1438
1439         let prog = args.remove(0);
1440         return ProcArgs {
1441             prog: prog,
1442             args: args,
1443         };
1444     }
1445
1446     fn split_maybe_args(&self, argstr: &Option<String>) -> Vec<String> {
1447         match *argstr {
1448             Some(ref s) => {
1449                 s
1450                     .split(' ')
1451                     .filter_map(|s| {
1452                         if s.chars().all(|c| c.is_whitespace()) {
1453                             None
1454                         } else {
1455                             Some(s.to_owned())
1456                         }
1457                     }).collect()
1458             }
1459             None => Vec::new()
1460         }
1461     }
1462
1463     fn program_output(&self,
1464                       lib_path: &str,
1465                       prog: String,
1466                       aux_path: Option<&str>,
1467                       args: Vec<String>,
1468                       env: Vec<(String, String)>,
1469                       input: Option<String>)
1470                       -> ProcRes {
1471         let cmdline =
1472         {
1473             let cmdline = self.make_cmdline(lib_path,
1474                                             &prog,
1475                                             &args);
1476             logv(self.config, format!("executing {}", cmdline));
1477             cmdline
1478         };
1479         let procsrv::Result {
1480             out,
1481             err,
1482             status
1483         } = procsrv::run(lib_path,
1484                          &prog,
1485                          aux_path,
1486                          &args,
1487                          env,
1488                          input).expect(&format!("failed to exec `{}`", prog));
1489         self.dump_output(&out, &err);
1490         return ProcRes {
1491             status: Status::Normal(status),
1492             stdout: out,
1493             stderr: err,
1494             cmdline: cmdline,
1495         };
1496     }
1497
1498     fn make_cmdline(&self, libpath: &str, prog: &str, args: &[String]) -> String {
1499         use util;
1500
1501         // Linux and mac don't require adjusting the library search path
1502         if cfg!(unix) {
1503             format!("{} {}", prog, args.join(" "))
1504         } else {
1505             // Build the LD_LIBRARY_PATH variable as it would be seen on the command line
1506             // for diagnostic purposes
1507             fn lib_path_cmd_prefix(path: &str) -> String {
1508                 format!("{}=\"{}\"", util::lib_path_env_var(), util::make_new_path(path))
1509             }
1510
1511             format!("{} {} {}", lib_path_cmd_prefix(libpath), prog, args.join(" "))
1512         }
1513     }
1514
1515     fn dump_output(&self, out: &str, err: &str) {
1516         self.dump_output_file(out, "out");
1517         self.dump_output_file(err, "err");
1518         self.maybe_dump_to_stdout(out, err);
1519     }
1520
1521     fn dump_output_file(&self,
1522                         out: &str,
1523                         extension: &str) {
1524         let outfile = self.make_out_name(extension);
1525         File::create(&outfile).unwrap().write_all(out.as_bytes()).unwrap();
1526     }
1527
1528     fn make_out_name(&self, extension: &str) -> PathBuf {
1529         self.output_base_name().with_extension(extension)
1530     }
1531
1532     fn aux_output_dir_name(&self) -> PathBuf {
1533         let f = self.output_base_name();
1534         let mut fname = f.file_name().unwrap().to_os_string();
1535         fname.push(&format!(".{}.libaux", self.config.mode));
1536         f.with_file_name(&fname)
1537     }
1538
1539     fn output_testname(&self, filepath: &Path) -> PathBuf {
1540         PathBuf::from(filepath.file_stem().unwrap())
1541     }
1542
1543     /// Given a test path like `compile-fail/foo/bar.rs` Returns a name like
1544     ///
1545     ///     <output>/foo/bar-stage1
1546     fn output_base_name(&self) -> PathBuf {
1547         let dir = self.config.build_base.join(&self.testpaths.relative_dir);
1548
1549         // Note: The directory `dir` is created during `collect_tests_from_dir`
1550         dir
1551             .join(&self.output_testname(&self.testpaths.file))
1552             .with_extension(&self.config.stage_id)
1553     }
1554
1555     fn maybe_dump_to_stdout(&self, out: &str, err: &str) {
1556         if self.config.verbose {
1557             println!("------{}------------------------------", "stdout");
1558             println!("{}", out);
1559             println!("------{}------------------------------", "stderr");
1560             println!("{}", err);
1561             println!("------------------------------------------");
1562         }
1563     }
1564
1565     fn error(&self, err: &str) {
1566         match self.revision {
1567             Some(rev) => println!("\nerror in revision `{}`: {}", rev, err),
1568             None => println!("\nerror: {}", err)
1569         }
1570     }
1571
1572     fn fatal(&self, err: &str) -> ! {
1573         self.error(err); panic!();
1574     }
1575
1576     fn fatal_proc_rec(&self, err: &str, proc_res: &ProcRes) -> ! {
1577         self.error(err);
1578         proc_res.fatal(None);
1579     }
1580
1581     fn _arm_exec_compiled_test(&self, env: Vec<(String, String)>) -> ProcRes {
1582         let args = self.make_run_args();
1583         let cmdline = self.make_cmdline("", &args.prog, &args.args);
1584
1585         // get bare program string
1586         let mut tvec: Vec<String> = args.prog
1587                                         .split('/')
1588                                         .map(str::to_owned)
1589                                         .collect();
1590         let prog_short = tvec.pop().unwrap();
1591
1592         // copy to target
1593         let copy_result = procsrv::run("",
1594                                        &self.config.adb_path,
1595                                        None,
1596                                        &[
1597                                            "push".to_owned(),
1598                                            args.prog.clone(),
1599                                            self.config.adb_test_dir.clone()
1600                                        ],
1601                                        vec!(("".to_owned(), "".to_owned())),
1602                                        Some("".to_owned()))
1603             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1604
1605         if self.config.verbose {
1606             println!("push ({}) {} {} {}",
1607                      self.config.target,
1608                      args.prog,
1609                      copy_result.out,
1610                      copy_result.err);
1611         }
1612
1613         logv(self.config, format!("executing ({}) {}", self.config.target, cmdline));
1614
1615         let mut runargs = Vec::new();
1616
1617         // run test via adb_run_wrapper
1618         runargs.push("shell".to_owned());
1619         for (key, val) in env {
1620             runargs.push(format!("{}={}", key, val));
1621         }
1622         runargs.push(format!("{}/../adb_run_wrapper.sh", self.config.adb_test_dir));
1623         runargs.push(format!("{}", self.config.adb_test_dir));
1624         runargs.push(format!("{}", prog_short));
1625
1626         for tv in &args.args {
1627             runargs.push(tv.to_owned());
1628         }
1629         procsrv::run("",
1630                      &self.config.adb_path,
1631                      None,
1632                      &runargs,
1633                      vec!(("".to_owned(), "".to_owned())), Some("".to_owned()))
1634             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1635
1636         // get exitcode of result
1637         runargs = Vec::new();
1638         runargs.push("shell".to_owned());
1639         runargs.push("cat".to_owned());
1640         runargs.push(format!("{}/{}.exitcode", self.config.adb_test_dir, prog_short));
1641
1642         let procsrv::Result{ out: exitcode_out, err: _, status: _ } =
1643             procsrv::run("",
1644                          &self.config.adb_path,
1645                          None,
1646                          &runargs,
1647                          vec!(("".to_owned(), "".to_owned())),
1648                          Some("".to_owned()))
1649             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1650
1651         let mut exitcode: i32 = 0;
1652         for c in exitcode_out.chars() {
1653             if !c.is_numeric() { break; }
1654             exitcode = exitcode * 10 + match c {
1655                 '0' ... '9' => c as i32 - ('0' as i32),
1656                 _ => 101,
1657             }
1658         }
1659
1660         // get stdout of result
1661         runargs = Vec::new();
1662         runargs.push("shell".to_owned());
1663         runargs.push("cat".to_owned());
1664         runargs.push(format!("{}/{}.stdout", self.config.adb_test_dir, prog_short));
1665
1666         let procsrv::Result{ out: stdout_out, err: _, status: _ } =
1667             procsrv::run("",
1668                          &self.config.adb_path,
1669                          None,
1670                          &runargs,
1671                          vec!(("".to_owned(), "".to_owned())),
1672                          Some("".to_owned()))
1673             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1674
1675         // get stderr of result
1676         runargs = Vec::new();
1677         runargs.push("shell".to_owned());
1678         runargs.push("cat".to_owned());
1679         runargs.push(format!("{}/{}.stderr", self.config.adb_test_dir, prog_short));
1680
1681         let procsrv::Result{ out: stderr_out, err: _, status: _ } =
1682             procsrv::run("",
1683                          &self.config.adb_path,
1684                          None,
1685                          &runargs,
1686                          vec!(("".to_owned(), "".to_owned())),
1687                          Some("".to_owned()))
1688             .expect(&format!("failed to exec `{}`", self.config.adb_path));
1689
1690         self.dump_output(&stdout_out, &stderr_out);
1691
1692         ProcRes {
1693             status: Status::Parsed(exitcode),
1694             stdout: stdout_out,
1695             stderr: stderr_out,
1696             cmdline: cmdline
1697         }
1698     }
1699
1700     fn _arm_push_aux_shared_library(&self) {
1701         let tdir = self.aux_output_dir_name();
1702
1703         let dirs = fs::read_dir(&tdir).unwrap();
1704         for file in dirs {
1705             let file = file.unwrap().path();
1706             if file.extension().and_then(|s| s.to_str()) == Some("so") {
1707                 // FIXME (#9639): This needs to handle non-utf8 paths
1708                 let copy_result = procsrv::run("",
1709                                                &self.config.adb_path,
1710                                                None,
1711                                                &[
1712                                                    "push".to_owned(),
1713                                                    file.to_str()
1714                                                        .unwrap()
1715                                                        .to_owned(),
1716                                                    self.config.adb_test_dir.to_owned(),
1717                                                ],
1718                                                vec!(("".to_owned(),
1719                                                      "".to_owned())),
1720                                                Some("".to_owned()))
1721                     .expect(&format!("failed to exec `{}`", self.config.adb_path));
1722
1723                 if self.config.verbose {
1724                     println!("push ({}) {:?} {} {}",
1725                              self.config.target, file.display(),
1726                              copy_result.out, copy_result.err);
1727                 }
1728             }
1729         }
1730     }
1731
1732     // codegen tests (using FileCheck)
1733
1734     fn compile_test_and_save_ir(&self) -> ProcRes {
1735         let aux_dir = self.aux_output_dir_name();
1736         // FIXME (#9639): This needs to handle non-utf8 paths
1737         let mut link_args = vec!("-L".to_owned(),
1738                                  aux_dir.to_str().unwrap().to_owned());
1739         let llvm_args = vec!("--emit=llvm-ir".to_owned(),);
1740         link_args.extend(llvm_args);
1741         let args = self.make_compile_args(link_args,
1742                                           &self.testpaths.file,
1743                                           TargetLocation::ThisDirectory(
1744                                               self.output_base_name().parent()
1745                                                                      .unwrap()
1746                                                                      .to_path_buf()));
1747         self.compose_and_run_compiler(args, None)
1748     }
1749
1750     fn check_ir_with_filecheck(&self) -> ProcRes {
1751         let irfile = self.output_base_name().with_extension("ll");
1752         let prog = self.config.llvm_filecheck.as_ref().unwrap();
1753         let proc_args = ProcArgs {
1754             // FIXME (#9639): This needs to handle non-utf8 paths
1755             prog: prog.to_str().unwrap().to_owned(),
1756             args: vec!(format!("-input-file={}", irfile.to_str().unwrap()),
1757                        self.testpaths.file.to_str().unwrap().to_owned())
1758         };
1759         self.compose_and_run(proc_args, Vec::new(), "", None, None)
1760     }
1761
1762     fn run_codegen_test(&self) {
1763         assert!(self.revision.is_none(), "revisions not relevant here");
1764
1765         if self.config.llvm_filecheck.is_none() {
1766             self.fatal("missing --llvm-filecheck");
1767         }
1768
1769         let mut proc_res = self.compile_test_and_save_ir();
1770         if !proc_res.status.success() {
1771             self.fatal_proc_rec("compilation failed!", &proc_res);
1772         }
1773
1774         proc_res = self.check_ir_with_filecheck();
1775         if !proc_res.status.success() {
1776             self.fatal_proc_rec("verification with 'FileCheck' failed", &proc_res);
1777         }
1778     }
1779
1780     fn charset() -> &'static str {
1781         // FreeBSD 10.1 defaults to GDB 6.1.1 which doesn't support "auto" charset
1782         if cfg!(target_os = "bitrig") {
1783             "auto"
1784         } else if cfg!(target_os = "freebsd") {
1785             "ISO-8859-1"
1786         } else {
1787             "UTF-8"
1788         }
1789     }
1790
1791     fn run_rustdoc_test(&self) {
1792         assert!(self.revision.is_none(), "revisions not relevant here");
1793
1794         let out_dir = self.output_base_name();
1795         let _ = fs::remove_dir_all(&out_dir);
1796         self.create_dir_racy(&out_dir);
1797
1798         let proc_res = self.document(&out_dir);
1799         if !proc_res.status.success() {
1800             self.fatal_proc_rec("rustdoc failed!", &proc_res);
1801         }
1802         let root = self.find_rust_src_root().unwrap();
1803
1804         let res = self.cmd2procres(Command::new(&self.config.docck_python)
1805                                    .arg(root.join("src/etc/htmldocck.py"))
1806                                    .arg(out_dir)
1807                                    .arg(&self.testpaths.file));
1808         if !res.status.success() {
1809             self.fatal_proc_rec("htmldocck failed!", &res);
1810         }
1811     }
1812
1813     fn run_codegen_units_test(&self) {
1814         assert!(self.revision.is_none(), "revisions not relevant here");
1815
1816         let proc_res = self.compile_test();
1817
1818         if !proc_res.status.success() {
1819             self.fatal_proc_rec("compilation failed!", &proc_res);
1820         }
1821
1822         self.check_no_compiler_crash(&proc_res);
1823
1824         const PREFIX: &'static str = "TRANS_ITEM ";
1825         const CGU_MARKER: &'static str = "@@";
1826
1827         let actual: Vec<TransItem> = proc_res
1828             .stdout
1829             .lines()
1830             .filter(|line| line.starts_with(PREFIX))
1831             .map(str_to_trans_item)
1832             .collect();
1833
1834         let expected: Vec<TransItem> = errors::load_errors(&self.testpaths.file, None)
1835             .iter()
1836             .map(|e| str_to_trans_item(&e.msg[..]))
1837             .collect();
1838
1839         let mut missing = Vec::new();
1840         let mut wrong_cgus = Vec::new();
1841
1842         for expected_item in &expected {
1843             let actual_item_with_same_name = actual.iter()
1844                                                    .find(|ti| ti.name == expected_item.name);
1845
1846             if let Some(actual_item) = actual_item_with_same_name {
1847                 if !expected_item.codegen_units.is_empty() {
1848                     // Also check for codegen units
1849                     if expected_item.codegen_units != actual_item.codegen_units {
1850                         wrong_cgus.push((expected_item.clone(), actual_item.clone()));
1851                     }
1852                 }
1853             } else {
1854                 missing.push(expected_item.string.clone());
1855             }
1856         }
1857
1858         let unexpected: Vec<_> =
1859             actual.iter()
1860                   .filter(|acgu| !expected.iter().any(|ecgu| acgu.name == ecgu.name))
1861                   .map(|acgu| acgu.string.clone())
1862                   .collect();
1863
1864         if !missing.is_empty() {
1865             missing.sort();
1866
1867             println!("\nThese items should have been contained but were not:\n");
1868
1869             for item in &missing {
1870                 println!("{}", item);
1871             }
1872
1873             println!("\n");
1874         }
1875
1876         if !unexpected.is_empty() {
1877             let sorted = {
1878                 let mut sorted = unexpected.clone();
1879                 sorted.sort();
1880                 sorted
1881             };
1882
1883             println!("\nThese items were contained but should not have been:\n");
1884
1885             for item in sorted {
1886                 println!("{}", item);
1887             }
1888
1889             println!("\n");
1890         }
1891
1892         if !wrong_cgus.is_empty() {
1893             wrong_cgus.sort_by_key(|pair| pair.0.name.clone());
1894             println!("\nThe following items were assigned to wrong codegen units:\n");
1895
1896             for &(ref expected_item, ref actual_item) in &wrong_cgus {
1897                 println!("{}", expected_item.name);
1898                 println!("  expected: {}", codegen_units_to_str(&expected_item.codegen_units));
1899                 println!("  actual:   {}", codegen_units_to_str(&actual_item.codegen_units));
1900                 println!("");
1901             }
1902         }
1903
1904         if !(missing.is_empty() && unexpected.is_empty() && wrong_cgus.is_empty())
1905         {
1906             panic!();
1907         }
1908
1909         #[derive(Clone, Eq, PartialEq)]
1910         struct TransItem {
1911             name: String,
1912             codegen_units: HashSet<String>,
1913             string: String,
1914         }
1915
1916         // [TRANS_ITEM] name [@@ (cgu)+]
1917         fn str_to_trans_item(s: &str) -> TransItem {
1918             let s = if s.starts_with(PREFIX) {
1919                 (&s[PREFIX.len()..]).trim()
1920             } else {
1921                 s.trim()
1922             };
1923
1924             let full_string = format!("{}{}", PREFIX, s.trim().to_owned());
1925
1926             let parts: Vec<&str> = s.split(CGU_MARKER)
1927                                     .map(str::trim)
1928                                     .filter(|s| !s.is_empty())
1929                                     .collect();
1930
1931             let name = parts[0].trim();
1932
1933             let cgus = if parts.len() > 1 {
1934                 let cgus_str = parts[1];
1935
1936                 cgus_str.split(" ")
1937                         .map(str::trim)
1938                         .filter(|s| !s.is_empty())
1939                         .map(str::to_owned)
1940                         .collect()
1941             }
1942             else {
1943                 HashSet::new()
1944             };
1945
1946             TransItem {
1947                 name: name.to_owned(),
1948                 codegen_units: cgus,
1949                 string: full_string,
1950             }
1951         }
1952
1953         fn codegen_units_to_str(cgus: &HashSet<String>) -> String
1954         {
1955             let mut cgus: Vec<_> = cgus.iter().collect();
1956             cgus.sort();
1957
1958             let mut string = String::new();
1959             for cgu in cgus {
1960                 string.push_str(&cgu[..]);
1961                 string.push_str(" ");
1962             }
1963
1964             string
1965         }
1966     }
1967
1968     fn init_incremental_test(&self) {
1969         // (See `run_incremental_test` for an overview of how incremental tests work.)
1970
1971         // Before any of the revisions have executed, create the
1972         // incremental workproduct directory.  Delete any old
1973         // incremental work products that may be there from prior
1974         // runs.
1975         let incremental_dir = self.incremental_dir();
1976         if incremental_dir.exists() {
1977             fs::remove_dir_all(&incremental_dir).unwrap();
1978         }
1979         fs::create_dir_all(&incremental_dir).unwrap();
1980
1981         if self.config.verbose {
1982             print!("init_incremental_test: incremental_dir={}", incremental_dir.display());
1983         }
1984     }
1985
1986     fn run_incremental_test(&self) {
1987         // Basic plan for a test incremental/foo/bar.rs:
1988         // - load list of revisions rpass1, cfail2, rpass3
1989         //   - each should begin with `rpass`, `cfail`, or `cfail`
1990         //   - if `rpass`, expect compile and execution to succeed
1991         //   - if `cfail`, expect compilation to fail
1992         //   - if `rfail`, expect execution to fail
1993         // - create a directory build/foo/bar.incremental
1994         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C rpass1
1995         //   - because name of revision starts with "rpass", expect success
1996         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C cfail2
1997         //   - because name of revision starts with "cfail", expect an error
1998         //   - load expected errors as usual, but filter for those that end in `[rfail2]`
1999         // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C rpass3
2000         //   - because name of revision starts with "rpass", expect success
2001         // - execute build/foo/bar.exe and save output
2002         //
2003         // FIXME -- use non-incremental mode as an oracle? That doesn't apply
2004         // to #[rustc_dirty] and clean tests I guess
2005
2006         let revision = self.revision.expect("incremental tests require a list of revisions");
2007
2008         // Incremental workproduct directory should have already been created.
2009         let incremental_dir = self.incremental_dir();
2010         assert!(incremental_dir.exists(), "init_incremental_test failed to create incremental dir");
2011
2012         // Add an extra flag pointing at the incremental directory.
2013         let mut revision_props = self.props.clone();
2014         revision_props.incremental_dir = Some(incremental_dir);
2015
2016         let revision_cx = TestCx {
2017             config: self.config,
2018             props: &revision_props,
2019             testpaths: self.testpaths,
2020             revision: self.revision,
2021         };
2022
2023         if self.config.verbose {
2024             print!("revision={:?} revision_props={:#?}", revision, revision_props);
2025         }
2026
2027         if revision.starts_with("rpass") {
2028             revision_cx.run_rpass_test();
2029         } else if revision.starts_with("rfail") {
2030             revision_cx.run_rfail_test();
2031         } else if revision.starts_with("cfail") {
2032             revision_cx.run_cfail_test();
2033         } else {
2034             revision_cx.fatal(
2035                 "revision name must begin with rpass, rfail, or cfail");
2036         }
2037     }
2038
2039     /// Directory where incremental work products are stored.
2040     fn incremental_dir(&self) -> PathBuf {
2041         self.output_base_name().with_extension("incremental")
2042     }
2043
2044     fn run_rmake_test(&self) {
2045         // FIXME(#11094): we should fix these tests
2046         if self.config.host != self.config.target {
2047             return
2048         }
2049
2050         let cwd = env::current_dir().unwrap();
2051         let src_root = self.config.src_base.parent().unwrap()
2052                                            .parent().unwrap()
2053                                            .parent().unwrap();
2054         let src_root = cwd.join(&src_root);
2055
2056         let tmpdir = cwd.join(self.output_base_name());
2057         if tmpdir.exists() {
2058             self.aggressive_rm_rf(&tmpdir).unwrap();
2059         }
2060         self.create_dir_racy(&tmpdir);
2061
2062         let mut cmd = Command::new("make");
2063         cmd.current_dir(&self.testpaths.file)
2064            .env("TARGET", &self.config.target)
2065            .env("PYTHON", &self.config.docck_python)
2066            .env("S", src_root)
2067            .env("RUST_BUILD_STAGE", &self.config.stage_id)
2068            .env("RUSTC", cwd.join(&self.config.rustc_path))
2069            .env("RUSTDOC", cwd.join(&self.config.rustdoc_path))
2070            .env("TMPDIR", &tmpdir)
2071            .env("LD_LIB_PATH_ENVVAR", procsrv::dylib_env_var())
2072            .env("HOST_RPATH_DIR", cwd.join(&self.config.compile_lib_path))
2073            .env("TARGET_RPATH_DIR", cwd.join(&self.config.run_lib_path))
2074            .env("LLVM_COMPONENTS", &self.config.llvm_components)
2075            .env("LLVM_CXXFLAGS", &self.config.llvm_cxxflags);
2076
2077         if self.config.target.contains("msvc") {
2078             // We need to pass a path to `lib.exe`, so assume that `cc` is `cl.exe`
2079             // and that `lib.exe` lives next to it.
2080             let lib = Path::new(&self.config.cc).parent().unwrap().join("lib.exe");
2081
2082             // MSYS doesn't like passing flags of the form `/foo` as it thinks it's
2083             // a path and instead passes `C:\msys64\foo`, so convert all
2084             // `/`-arguments to MSVC here to `-` arguments.
2085             let cflags = self.config.cflags.split(' ').map(|s| s.replace("/", "-"))
2086                                                  .collect::<Vec<_>>().join(" ");
2087
2088             cmd.env("IS_MSVC", "1")
2089                .env("MSVC_LIB", format!("'{}' -nologo", lib.display()))
2090                .env("CC", format!("'{}' {}", self.config.cc, cflags))
2091                .env("CXX", &self.config.cxx);
2092         } else {
2093             cmd.env("CC", format!("{} {}", self.config.cc, self.config.cflags))
2094                .env("CXX", format!("{} {}", self.config.cxx, self.config.cflags));
2095         }
2096
2097         let output = cmd.output().expect("failed to spawn `make`");
2098         if !output.status.success() {
2099             let res = ProcRes {
2100                 status: Status::Normal(output.status),
2101                 stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
2102                 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
2103                 cmdline: format!("{:?}", cmd),
2104             };
2105             self.fatal_proc_rec("make failed", &res);
2106         }
2107     }
2108
2109     fn aggressive_rm_rf(&self, path: &Path) -> io::Result<()> {
2110         for e in try!(path.read_dir()) {
2111             let entry = try!(e);
2112             let path = entry.path();
2113             if try!(entry.file_type()).is_dir() {
2114                 try!(self.aggressive_rm_rf(&path));
2115             } else {
2116                 // Remove readonly files as well on windows (by default we can't)
2117                 try!(fs::remove_file(&path).or_else(|e| {
2118                     if cfg!(windows) && e.kind() == io::ErrorKind::PermissionDenied {
2119                         let mut meta = try!(entry.metadata()).permissions();
2120                         meta.set_readonly(false);
2121                         try!(fs::set_permissions(&path, meta));
2122                         fs::remove_file(&path)
2123                     } else {
2124                         Err(e)
2125                     }
2126                 }))
2127             }
2128         }
2129         fs::remove_dir(path)
2130     }
2131
2132     fn run_ui_test(&self) {
2133         println!("ui: {}", self.testpaths.file.display());
2134
2135         let proc_res = self.compile_test();
2136
2137         let expected_stderr_path = self.expected_output_path("stderr");
2138         let expected_stderr = self.load_expected_output(&expected_stderr_path);
2139
2140         let expected_stdout_path = self.expected_output_path("stdout");
2141         let expected_stdout = self.load_expected_output(&expected_stdout_path);
2142
2143         let normalized_stdout = self.normalize_output(&proc_res.stdout);
2144         let normalized_stderr = self.normalize_output(&proc_res.stderr);
2145
2146         let mut errors = 0;
2147         errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
2148         errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
2149
2150         if errors > 0 {
2151             println!("To update references, run this command from build directory:");
2152             let relative_path_to_file =
2153                 self.testpaths.relative_dir
2154                               .join(self.testpaths.file.file_name().unwrap());
2155             println!("{}/update-references.sh '{}' '{}'",
2156                      self.config.src_base.display(),
2157                      self.config.build_base.display(),
2158                      relative_path_to_file.display());
2159             self.fatal_proc_rec(&format!("{} errors occurred comparing output.", errors),
2160                                 &proc_res);
2161         }
2162     }
2163
2164     fn run_mir_opt_test(&self) {
2165         let proc_res = self.compile_test();
2166
2167         if !proc_res.status.success() {
2168             self.fatal_proc_rec("compilation failed!", &proc_res);
2169         }
2170
2171         let proc_res = self.exec_compiled_test();
2172
2173         if !proc_res.status.success() {
2174             self.fatal_proc_rec("test run failed!", &proc_res);
2175         }
2176         self.check_mir_dump();
2177     }
2178
2179     fn check_mir_dump(&self) {
2180         let mut test_file_contents = String::new();
2181         fs::File::open(self.testpaths.file.clone()).unwrap()
2182                                                    .read_to_string(&mut test_file_contents)
2183                                                    .unwrap();
2184         if let Some(idx) =  test_file_contents.find("// END RUST SOURCE") {
2185             let (_, tests_text) = test_file_contents.split_at(idx + "// END_RUST SOURCE".len());
2186             let tests_text_str = String::from(tests_text);
2187             let mut curr_test : Option<&str> = None;
2188             let mut curr_test_contents = Vec::new();
2189             for l in tests_text_str.lines() {
2190                 debug!("line: {:?}", l);
2191                 if l.starts_with("// START ") {
2192                     let (_, t) = l.split_at("// START ".len());
2193                     curr_test = Some(t);
2194                 } else if l.starts_with("// END") {
2195                     let (_, t) = l.split_at("// END ".len());
2196                     if Some(t) != curr_test {
2197                         panic!("mismatched START END test name");
2198                     }
2199                     self.compare_mir_test_output(curr_test.unwrap(), &curr_test_contents);
2200                     curr_test = None;
2201                     curr_test_contents.clear();
2202                 } else if l.is_empty() {
2203                     // ignore
2204                 } else if l.starts_with("// ") {
2205                     let (_, test_content) = l.split_at("// ".len());
2206                     curr_test_contents.push(test_content);
2207                 }
2208             }
2209         }
2210     }
2211
2212     fn compare_mir_test_output(&self, test_name: &str, expected_content: &Vec<&str>) {
2213         let mut output_file = PathBuf::new();
2214         output_file.push(self.get_mir_dump_dir());
2215         output_file.push(test_name);
2216         debug!("comparing the contests of: {:?}", output_file);
2217         debug!("with: {:?}", expected_content);
2218
2219         let mut dumped_file = fs::File::open(output_file.clone()).unwrap();
2220         let mut dumped_string = String::new();
2221         dumped_file.read_to_string(&mut dumped_string).unwrap();
2222         let mut dumped_lines = dumped_string.lines().filter(|l| !l.is_empty());
2223         let mut expected_lines = expected_content.iter().filter(|l| !l.is_empty());
2224
2225         // We expect each non-empty line from expected_content to appear
2226         // in the dump in order, but there may be extra lines interleaved
2227         while let Some(expected_line) = expected_lines.next() {
2228             let e_norm = normalize_mir_line(expected_line);
2229             if e_norm.is_empty() {
2230                 continue;
2231             };
2232             let mut found = false;
2233             while let Some(dumped_line) = dumped_lines.next() {
2234                 let d_norm = normalize_mir_line(dumped_line);
2235                 debug!("found: {:?}", d_norm);
2236                 debug!("expected: {:?}", e_norm);
2237                 if e_norm == d_norm {
2238                     found = true;
2239                     break;
2240                 };
2241             }
2242             if !found {
2243                 panic!("ran out of mir dump output to match against");
2244             }
2245         }
2246     }
2247
2248     fn get_mir_dump_dir(&self) -> PathBuf {
2249         let mut mir_dump_dir = PathBuf::from(self.config.build_base
2250                                                     .as_path()
2251                                                     .to_str()
2252                                                     .unwrap());
2253         debug!("input_file: {:?}", self.testpaths.file);
2254         mir_dump_dir.push(self.testpaths.file.file_stem().unwrap().to_str().unwrap());
2255         mir_dump_dir
2256     }
2257
2258     fn normalize_output(&self, output: &str) -> String {
2259         let parent_dir = self.testpaths.file.parent().unwrap();
2260         let parent_dir_str = parent_dir.display().to_string();
2261         output.replace(&parent_dir_str, "$DIR")
2262               .replace("\\", "/") // normalize for paths on windows
2263               .replace("\r\n", "\n") // normalize for linebreaks on windows
2264               .replace("\t", "\\t") // makes tabs visible
2265     }
2266
2267     fn expected_output_path(&self, kind: &str) -> PathBuf {
2268         let extension = match self.revision {
2269             Some(r) => format!("{}.{}", r, kind),
2270             None => kind.to_string(),
2271         };
2272         self.testpaths.file.with_extension(extension)
2273     }
2274
2275     fn load_expected_output(&self, path: &Path) -> String {
2276         if !path.exists() {
2277             return String::new();
2278         }
2279
2280         let mut result = String::new();
2281         match File::open(path).and_then(|mut f| f.read_to_string(&mut result)) {
2282             Ok(_) => result,
2283             Err(e) => {
2284                 self.fatal(&format!("failed to load expected output from `{}`: {}",
2285                                     path.display(), e))
2286             }
2287         }
2288     }
2289
2290     fn compare_output(&self, kind: &str, actual: &str, expected: &str) -> usize {
2291         if actual == expected {
2292             return 0;
2293         }
2294
2295         println!("normalized {}:\n{}\n", kind, actual);
2296         println!("expected {}:\n{}\n", kind, expected);
2297         println!("diff of {}:\n", kind);
2298         for line in uidiff::diff_lines(actual, expected) {
2299             println!("{}", line);
2300         }
2301
2302         let output_file = self.output_base_name().with_extension(kind);
2303         match File::create(&output_file).and_then(|mut f| f.write_all(actual.as_bytes())) {
2304             Ok(()) => { }
2305             Err(e) => {
2306                 self.fatal(&format!("failed to write {} to `{}`: {}",
2307                                     kind, output_file.display(), e))
2308             }
2309         }
2310
2311         println!("\nThe actual {0} differed from the expected {0}.", kind);
2312         println!("Actual {} saved to {}", kind, output_file.display());
2313         1
2314     }
2315 }
2316
2317 struct ProcArgs {
2318     prog: String,
2319     args: Vec<String>,
2320 }
2321
2322 pub struct ProcRes {
2323     status: Status,
2324     stdout: String,
2325     stderr: String,
2326     cmdline: String,
2327 }
2328
2329 enum Status {
2330     Parsed(i32),
2331     Normal(ExitStatus),
2332 }
2333
2334 impl ProcRes {
2335     pub fn fatal(&self, err: Option<&str>) -> ! {
2336         if let Some(e) = err {
2337             println!("\nerror: {}", e);
2338         }
2339         print!("\
2340             status: {}\n\
2341             command: {}\n\
2342             stdout:\n\
2343             ------------------------------------------\n\
2344             {}\n\
2345             ------------------------------------------\n\
2346             stderr:\n\
2347             ------------------------------------------\n\
2348             {}\n\
2349             ------------------------------------------\n\
2350             \n",
2351                self.status, self.cmdline, self.stdout,
2352                self.stderr);
2353         panic!();
2354     }
2355 }
2356
2357 impl Status {
2358     fn code(&self) -> Option<i32> {
2359         match *self {
2360             Status::Parsed(i) => Some(i),
2361             Status::Normal(ref e) => e.code(),
2362         }
2363     }
2364
2365     fn success(&self) -> bool {
2366         match *self {
2367             Status::Parsed(i) => i == 0,
2368             Status::Normal(ref e) => e.success(),
2369         }
2370     }
2371 }
2372
2373 impl fmt::Display for Status {
2374     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2375         match *self {
2376             Status::Parsed(i) => write!(f, "exit code: {}", i),
2377             Status::Normal(ref e) => e.fmt(f),
2378         }
2379     }
2380 }
2381
2382 enum TargetLocation {
2383     ThisFile(PathBuf),
2384     ThisDirectory(PathBuf),
2385 }
2386
2387 fn normalize_mir_line(line: &str) -> String {
2388     let no_comments = if let Some(idx) = line.find("//") {
2389         let (l, _) = line.split_at(idx);
2390         l
2391     } else {
2392         line
2393     };
2394     no_comments.replace(char::is_whitespace, "")
2395 }