1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![crate_name = "compiletest"]
13 #![feature(box_syntax)]
14 #![feature(rustc_private)]
16 #![feature(question_mark)]
24 extern crate serialize as rustc_serialize;
30 extern crate env_logger;
33 use std::ffi::OsString;
36 use std::path::{Path, PathBuf};
37 use getopts::{optopt, optflag, reqopt};
39 use common::{Pretty, DebugInfoGdb, DebugInfoLldb, Mode};
43 use self::header::EarlyProps;
57 fn log_init() { env_logger::init().unwrap(); }
58 #[cfg(not(cargobuild))]
62 let config = parse_config(env::args().collect());
64 if config.valgrind_path.is_none() && config.force_valgrind {
65 panic!("Can't find Valgrind to run Valgrind tests");
72 pub fn parse_config(args: Vec<String> ) -> Config {
74 let groups : Vec<getopts::OptGroup> =
75 vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
76 reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
77 reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
78 reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
79 reqopt("", "lldb-python", "path to python to use for doc tests", "PATH"),
80 reqopt("", "docck-python", "path to python to use for doc tests", "PATH"),
81 optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
82 optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
83 optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR"),
84 reqopt("", "src-base", "directory to scan for test files", "PATH"),
85 reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
86 reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
87 reqopt("", "mode", "which sort of compile tests to run",
88 "(compile-fail|parse-fail|run-fail|run-pass|\
89 run-pass-valgrind|pretty|debug-info|incremental|mir-opt)"),
90 optflag("", "ignored", "run tests marked as ignored"),
91 optopt("", "runtool", "supervisor program to run tests under \
92 (eg. emulator, valgrind)", "PROGRAM"),
93 optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
94 optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
95 optflag("", "verbose", "run tests verbosely, showing all output"),
96 optflag("", "quiet", "print one character per test instead of one line"),
97 optopt("", "logfile", "file to log test execution to", "FILE"),
98 optopt("", "target", "the target to build for", "TARGET"),
99 optopt("", "host", "the host to build for", "HOST"),
100 optopt("", "gdb-version", "the version of GDB used", "VERSION STRING"),
101 optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
102 optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
103 optopt("", "adb-path", "path to the android debugger", "PATH"),
104 optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
105 optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
106 reqopt("", "cc", "path to a C compiler", "PATH"),
107 reqopt("", "cxx", "path to a C++ compiler", "PATH"),
108 reqopt("", "cflags", "flags for the C compiler", "FLAGS"),
109 reqopt("", "llvm-components", "list of LLVM components built in", "LIST"),
110 reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"),
111 optflag("h", "help", "show this message"));
113 let (argv0, args_) = args.split_first().unwrap();
114 if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
115 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
116 println!("{}", getopts::usage(&message, &groups));
122 &match getopts::getopts(args_, &groups) {
124 Err(f) => panic!("{:?}", f)
127 if matches.opt_present("h") || matches.opt_present("help") {
128 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
129 println!("{}", getopts::usage(&message, &groups));
134 fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
135 match m.opt_str(nm) {
136 Some(s) => PathBuf::from(&s),
137 None => panic!("no option (=path) found for {}", nm),
141 fn make_absolute(path: PathBuf) -> PathBuf {
142 if path.is_relative() {
143 env::current_dir().unwrap().join(path)
150 compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
151 run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
152 rustc_path: opt_path(matches, "rustc-path"),
153 rustdoc_path: opt_path(matches, "rustdoc-path"),
154 lldb_python: matches.opt_str("lldb-python").unwrap(),
155 docck_python: matches.opt_str("docck-python").unwrap(),
156 valgrind_path: matches.opt_str("valgrind-path"),
157 force_valgrind: matches.opt_present("force-valgrind"),
158 llvm_filecheck: matches.opt_str("llvm-filecheck").map(|s| PathBuf::from(&s)),
159 src_base: opt_path(matches, "src-base"),
160 build_base: opt_path(matches, "build-base"),
161 stage_id: matches.opt_str("stage-id").unwrap(),
162 mode: matches.opt_str("mode").unwrap().parse().ok().expect("invalid mode"),
163 run_ignored: matches.opt_present("ignored"),
164 filter: matches.free.first().cloned(),
165 logfile: matches.opt_str("logfile").map(|s| PathBuf::from(&s)),
166 runtool: matches.opt_str("runtool"),
167 host_rustcflags: matches.opt_str("host-rustcflags"),
168 target_rustcflags: matches.opt_str("target-rustcflags"),
169 target: opt_str2(matches.opt_str("target")),
170 host: opt_str2(matches.opt_str("host")),
171 gdb_version: extract_gdb_version(matches.opt_str("gdb-version")),
172 lldb_version: extract_lldb_version(matches.opt_str("lldb-version")),
173 android_cross_path: opt_path(matches, "android-cross-path"),
174 adb_path: opt_str2(matches.opt_str("adb-path")),
175 adb_test_dir: format!("{}/{}",
176 opt_str2(matches.opt_str("adb-test-dir")),
177 opt_str2(matches.opt_str("target"))),
179 opt_str2(matches.opt_str("target")).contains("android") &&
180 "(none)" != opt_str2(matches.opt_str("adb-test-dir")) &&
181 !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
182 lldb_python_dir: matches.opt_str("lldb-python-dir"),
183 verbose: matches.opt_present("verbose"),
184 quiet: matches.opt_present("quiet"),
186 cc: matches.opt_str("cc").unwrap(),
187 cxx: matches.opt_str("cxx").unwrap(),
188 cflags: matches.opt_str("cflags").unwrap(),
189 llvm_components: matches.opt_str("llvm-components").unwrap(),
190 llvm_cxxflags: matches.opt_str("llvm-cxxflags").unwrap(),
194 pub fn log_config(config: &Config) {
196 logv(c, format!("configuration:"));
197 logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
198 logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
199 logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
200 logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
201 logv(c, format!("src_base: {:?}", config.src_base.display()));
202 logv(c, format!("build_base: {:?}", config.build_base.display()));
203 logv(c, format!("stage_id: {}", config.stage_id));
204 logv(c, format!("mode: {}", config.mode));
205 logv(c, format!("run_ignored: {}", config.run_ignored));
206 logv(c, format!("filter: {}",
207 opt_str(&config.filter
209 .map(|re| re.to_owned()))));
210 logv(c, format!("runtool: {}", opt_str(&config.runtool)));
211 logv(c, format!("host-rustcflags: {}",
212 opt_str(&config.host_rustcflags)));
213 logv(c, format!("target-rustcflags: {}",
214 opt_str(&config.target_rustcflags)));
215 logv(c, format!("target: {}", config.target));
216 logv(c, format!("host: {}", config.host));
217 logv(c, format!("android-cross-path: {:?}",
218 config.android_cross_path.display()));
219 logv(c, format!("adb_path: {:?}", config.adb_path));
220 logv(c, format!("adb_test_dir: {:?}", config.adb_test_dir));
221 logv(c, format!("adb_device_status: {}",
222 config.adb_device_status));
223 logv(c, format!("verbose: {}", config.verbose));
224 logv(c, format!("quiet: {}", config.quiet));
225 logv(c, format!("\n"));
228 pub fn opt_str<'a>(maybestr: &'a Option<String>) -> &'a str {
235 pub fn opt_str2(maybestr: Option<String>) -> String {
237 None => "(none)".to_owned(),
242 pub fn run_tests(config: &Config) {
243 if config.target.contains("android") {
244 if let DebugInfoGdb = config.mode {
245 println!("{} debug-info test uses tcp 5039 port.\
246 please reserve it", config.target);
249 // android debug-info test uses remote debugger
250 // so, we test 1 thread at once.
251 // also trying to isolate problems with adb_run_wrapper.sh ilooping
252 env::set_var("RUST_TEST_THREADS","1");
257 if let Some(lldb_version) = config.lldb_version.as_ref() {
258 if is_blacklisted_lldb_version(&lldb_version[..]) {
259 println!("WARNING: The used version of LLDB ({}) has a \
260 known issue that breaks debuginfo tests. See \
261 issue #32520 for more information. Skipping all \
268 // Some older versions of LLDB seem to have problems with multiple
269 // instances running in parallel, so only run one test thread at a
271 env::set_var("RUST_TEST_THREADS", "1");
273 _ => { /* proceed */ }
276 // FIXME(#33435) Avoid spurious failures in codegen-units/partitioning tests.
277 if let Mode::CodegenUnits = config.mode {
278 let _ = fs::remove_dir_all("tmp/partitioning-tests");
281 let opts = test_opts(config);
282 let tests = make_tests(config);
283 // sadly osx needs some file descriptor limits raised for running tests in
284 // parallel (especially when we have lots and lots of child processes).
285 // For context, see #8904
286 unsafe { raise_fd_limit::raise_fd_limit(); }
287 // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
288 // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
289 env::set_var("__COMPAT_LAYER", "RunAsInvoker");
290 let res = test::run_tests_console(&opts, tests.into_iter().collect());
293 Ok(false) => panic!("Some tests failed"),
295 println!("I/O failure during tests: {:?}", e);
300 pub fn test_opts(config: &Config) -> test::TestOpts {
302 filter: config.filter.clone(),
303 run_ignored: config.run_ignored,
305 logfile: config.logfile.clone(),
307 bench_benchmarks: true,
308 nocapture: match env::var("RUST_TEST_NOCAPTURE") {
309 Ok(val) => &val != "0",
312 color: test::AutoColor,
317 pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
318 debug!("making tests from {:?}",
319 config.src_base.display());
320 let mut tests = Vec::new();
321 collect_tests_from_dir(config,
330 fn collect_tests_from_dir(config: &Config,
333 relative_dir_path: &Path,
334 tests: &mut Vec<test::TestDescAndFn>)
336 // Ignore directories that contain a file
337 // `compiletest-ignore-dir`.
338 for file in fs::read_dir(dir)? {
340 let name = file.file_name();
341 if name == *"compiletest-ignore-dir" {
344 if name == *"Makefile" && config.mode == Mode::RunMake {
345 let paths = TestPaths {
346 file: dir.to_path_buf(),
347 base: base.to_path_buf(),
348 relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
350 tests.push(make_test(config, &paths));
355 // If we find a test foo/bar.rs, we have to build the
356 // output directory `$build/foo` so we can write
357 // `$build/foo/bar` into it. We do this *now* in this
358 // sequential loop because otherwise, if we do it in the
359 // tests themselves, they race for the privilege of
360 // creating the directories and sometimes fail randomly.
361 let build_dir = config.build_base.join(&relative_dir_path);
362 fs::create_dir_all(&build_dir).unwrap();
364 // Add each `.rs` file as a test, and recurse further on any
365 // subdirectories we find, except for `aux` directories.
366 let dirs = fs::read_dir(dir)?;
369 let file_path = file.path();
370 let file_name = file.file_name();
371 if is_test(&file_name) {
372 debug!("found test file: {:?}", file_path.display());
373 let paths = TestPaths {
375 base: base.to_path_buf(),
376 relative_dir: relative_dir_path.to_path_buf(),
378 tests.push(make_test(config, &paths))
379 } else if file_path.is_dir() {
380 let relative_file_path = relative_dir_path.join(file.file_name());
381 if &file_name == "auxiliary" {
382 // `aux` directories contain other crates used for
383 // cross-crate tests. Don't search them for tests, but
384 // do create a directory in the build dir for them,
385 // since we will dump intermediate output in there
387 let build_dir = config.build_base.join(&relative_file_path);
388 fs::create_dir_all(&build_dir).unwrap();
390 debug!("found directory: {:?}", file_path.display());
391 collect_tests_from_dir(config,
398 debug!("found other file/directory: {:?}", file_path.display());
404 pub fn is_test(file_name: &OsString) -> bool {
405 let file_name = file_name.to_str().unwrap();
407 if !file_name.ends_with(".rs") {
411 // `.`, `#`, and `~` are common temp-file prefixes.
412 let invalid_prefixes = &[".", "#", "~"];
413 !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
416 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
417 let early_props = EarlyProps::from_file(config, &testpaths.file);
419 // The `should-fail` annotation doesn't apply to pretty tests,
420 // since we run the pretty printer across all tests by default.
421 // If desired, we could add a `should-fail-pretty` annotation.
422 let should_panic = match config.mode {
423 Pretty => test::ShouldPanic::No,
424 _ => if early_props.should_fail {
425 test::ShouldPanic::Yes
427 test::ShouldPanic::No
431 test::TestDescAndFn {
432 desc: test::TestDesc {
433 name: make_test_name(config, testpaths),
434 ignore: early_props.ignore,
435 should_panic: should_panic,
437 testfn: make_test_closure(config, testpaths),
441 pub fn make_test_name(config: &Config, testpaths: &TestPaths) -> test::TestName {
442 // Convert a complete path to something like
444 // run-pass/foo/bar/baz.rs
446 PathBuf::from(config.mode.to_string())
447 .join(&testpaths.relative_dir)
448 .join(&testpaths.file.file_name().unwrap());
449 test::DynTestName(format!("[{}] {}", config.mode, path.display()))
452 pub fn make_test_closure(config: &Config, testpaths: &TestPaths) -> test::TestFn {
453 let config = config.clone();
454 let testpaths = testpaths.clone();
455 test::DynTestFn(Box::new(move || {
456 runtest::run(config, &testpaths)
460 fn extract_gdb_version(full_version_line: Option<String>) -> Option<String> {
461 match full_version_line {
462 Some(ref full_version_line)
463 if !full_version_line.trim().is_empty() => {
464 let full_version_line = full_version_line.trim();
466 // used to be a regex "(^|[^0-9])([0-9]\.[0-9]+)"
467 for (pos, c) in full_version_line.char_indices() {
471 if pos + 2 >= full_version_line.len() {
474 if full_version_line[pos + 1..].chars().next().unwrap() != '.' {
477 if !full_version_line[pos + 2..].chars().next().unwrap().is_digit(10) {
480 if pos > 0 && full_version_line[..pos].chars().next_back()
481 .unwrap().is_digit(10) {
484 let mut end = pos + 3;
485 while end < full_version_line.len() &&
486 full_version_line[end..].chars().next()
487 .unwrap().is_digit(10) {
490 return Some(full_version_line[pos..end].to_owned());
492 println!("Could not extract GDB version from line '{}'",
500 fn extract_lldb_version(full_version_line: Option<String>) -> Option<String> {
501 // Extract the major LLDB version from the given version string.
502 // LLDB version strings are different for Apple and non-Apple platforms.
503 // At the moment, this function only supports the Apple variant, which looks
506 // LLDB-179.5 (older versions)
507 // lldb-300.2.51 (new versions)
509 // We are only interested in the major version number, so this function
510 // will return `Some("179")` and `Some("300")` respectively.
512 if let Some(ref full_version_line) = full_version_line {
513 if !full_version_line.trim().is_empty() {
514 let full_version_line = full_version_line.trim();
516 for (pos, l) in full_version_line.char_indices() {
517 if l != 'l' && l != 'L' { continue }
518 if pos + 5 >= full_version_line.len() { continue }
519 let l = full_version_line[pos + 1..].chars().next().unwrap();
520 if l != 'l' && l != 'L' { continue }
521 let d = full_version_line[pos + 2..].chars().next().unwrap();
522 if d != 'd' && d != 'D' { continue }
523 let b = full_version_line[pos + 3..].chars().next().unwrap();
524 if b != 'b' && b != 'B' { continue }
525 let dash = full_version_line[pos + 4..].chars().next().unwrap();
526 if dash != '-' { continue }
528 let vers = full_version_line[pos + 5..].chars().take_while(|c| {
530 }).collect::<String>();
531 if !vers.is_empty() { return Some(vers) }
533 println!("Could not extract LLDB version from line '{}'",
540 fn is_blacklisted_lldb_version(version: &str) -> bool {