1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![crate_name = "compiletest"]
13 #![feature(box_syntax)]
14 #![feature(rustc_private)]
16 #![feature(question_mark)]
24 extern crate serialize as rustc_serialize;
30 extern crate env_logger;
33 use std::ffi::OsString;
36 use std::path::{Path, PathBuf};
37 use getopts::{optopt, optflag, reqopt};
39 use common::{Pretty, DebugInfoGdb, DebugInfoLldb, Mode};
43 use self::header::EarlyProps;
57 fn log_init() { env_logger::init().unwrap(); }
58 #[cfg(not(cargobuild))]
62 let config = parse_config(env::args().collect());
64 if config.valgrind_path.is_none() && config.force_valgrind {
65 panic!("Can't find Valgrind to run Valgrind tests");
72 pub fn parse_config(args: Vec<String> ) -> Config {
74 let groups : Vec<getopts::OptGroup> =
75 vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
76 reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
77 reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
78 reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
79 reqopt("", "lldb-python", "path to python to use for doc tests", "PATH"),
80 reqopt("", "docck-python", "path to python to use for doc tests", "PATH"),
81 optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
82 optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
83 optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR"),
84 reqopt("", "src-base", "directory to scan for test files", "PATH"),
85 reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
86 reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
87 reqopt("", "mode", "which sort of compile tests to run",
88 "(compile-fail|parse-fail|run-fail|run-pass|\
89 run-pass-valgrind|pretty|debug-info|incremental|mir-opt)"),
90 optflag("", "ignored", "run tests marked as ignored"),
91 optopt("", "runtool", "supervisor program to run tests under \
92 (eg. emulator, valgrind)", "PROGRAM"),
93 optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
94 optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
95 optflag("", "verbose", "run tests verbosely, showing all output"),
96 optflag("", "quiet", "print one character per test instead of one line"),
97 optopt("", "logfile", "file to log test execution to", "FILE"),
98 optopt("", "target", "the target to build for", "TARGET"),
99 optopt("", "host", "the host to build for", "HOST"),
100 optopt("", "gdb-version", "the version of GDB used", "VERSION STRING"),
101 optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
102 optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
103 optopt("", "adb-path", "path to the android debugger", "PATH"),
104 optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
105 optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
106 reqopt("", "cc", "path to a C compiler", "PATH"),
107 reqopt("", "cxx", "path to a C++ compiler", "PATH"),
108 reqopt("", "cflags", "flags for the C compiler", "FLAGS"),
109 reqopt("", "llvm-components", "list of LLVM components built in", "LIST"),
110 reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"),
111 optflag("h", "help", "show this message"));
113 let (argv0, args_) = args.split_first().unwrap();
114 if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
115 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
116 println!("{}", getopts::usage(&message, &groups));
122 &match getopts::getopts(args_, &groups) {
124 Err(f) => panic!("{:?}", f)
127 if matches.opt_present("h") || matches.opt_present("help") {
128 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
129 println!("{}", getopts::usage(&message, &groups));
134 fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
135 match m.opt_str(nm) {
136 Some(s) => PathBuf::from(&s),
137 None => panic!("no option (=path) found for {}", nm),
141 fn make_absolute(path: PathBuf) -> PathBuf {
142 if path.is_relative() {
143 env::current_dir().unwrap().join(path)
150 compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
151 run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
152 rustc_path: opt_path(matches, "rustc-path"),
153 rustdoc_path: opt_path(matches, "rustdoc-path"),
154 lldb_python: matches.opt_str("lldb-python").unwrap(),
155 docck_python: matches.opt_str("docck-python").unwrap(),
156 valgrind_path: matches.opt_str("valgrind-path"),
157 force_valgrind: matches.opt_present("force-valgrind"),
158 llvm_filecheck: matches.opt_str("llvm-filecheck").map(|s| PathBuf::from(&s)),
159 src_base: opt_path(matches, "src-base"),
160 build_base: opt_path(matches, "build-base"),
161 stage_id: matches.opt_str("stage-id").unwrap(),
162 mode: matches.opt_str("mode").unwrap().parse().ok().expect("invalid mode"),
163 run_ignored: matches.opt_present("ignored"),
164 filter: matches.free.first().cloned(),
165 logfile: matches.opt_str("logfile").map(|s| PathBuf::from(&s)),
166 runtool: matches.opt_str("runtool"),
167 host_rustcflags: matches.opt_str("host-rustcflags"),
168 target_rustcflags: matches.opt_str("target-rustcflags"),
169 target: opt_str2(matches.opt_str("target")),
170 host: opt_str2(matches.opt_str("host")),
171 gdb_version: extract_gdb_version(matches.opt_str("gdb-version")),
172 lldb_version: extract_lldb_version(matches.opt_str("lldb-version")),
173 android_cross_path: opt_path(matches, "android-cross-path"),
174 adb_path: opt_str2(matches.opt_str("adb-path")),
175 adb_test_dir: format!("{}/{}",
176 opt_str2(matches.opt_str("adb-test-dir")),
177 opt_str2(matches.opt_str("target"))),
179 opt_str2(matches.opt_str("target")).contains("android") &&
180 "(none)" != opt_str2(matches.opt_str("adb-test-dir")) &&
181 !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
182 lldb_python_dir: matches.opt_str("lldb-python-dir"),
183 verbose: matches.opt_present("verbose"),
184 quiet: matches.opt_present("quiet"),
186 cc: matches.opt_str("cc").unwrap(),
187 cxx: matches.opt_str("cxx").unwrap(),
188 cflags: matches.opt_str("cflags").unwrap(),
189 llvm_components: matches.opt_str("llvm-components").unwrap(),
190 llvm_cxxflags: matches.opt_str("llvm-cxxflags").unwrap(),
194 pub fn log_config(config: &Config) {
196 logv(c, format!("configuration:"));
197 logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
198 logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
199 logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
200 logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
201 logv(c, format!("src_base: {:?}", config.src_base.display()));
202 logv(c, format!("build_base: {:?}", config.build_base.display()));
203 logv(c, format!("stage_id: {}", config.stage_id));
204 logv(c, format!("mode: {}", config.mode));
205 logv(c, format!("run_ignored: {}", config.run_ignored));
206 logv(c, format!("filter: {}",
207 opt_str(&config.filter
209 .map(|re| re.to_owned()))));
210 logv(c, format!("runtool: {}", opt_str(&config.runtool)));
211 logv(c, format!("host-rustcflags: {}",
212 opt_str(&config.host_rustcflags)));
213 logv(c, format!("target-rustcflags: {}",
214 opt_str(&config.target_rustcflags)));
215 logv(c, format!("target: {}", config.target));
216 logv(c, format!("host: {}", config.host));
217 logv(c, format!("android-cross-path: {:?}",
218 config.android_cross_path.display()));
219 logv(c, format!("adb_path: {:?}", config.adb_path));
220 logv(c, format!("adb_test_dir: {:?}", config.adb_test_dir));
221 logv(c, format!("adb_device_status: {}",
222 config.adb_device_status));
223 logv(c, format!("verbose: {}", config.verbose));
224 logv(c, format!("quiet: {}", config.quiet));
225 logv(c, format!("\n"));
228 pub fn opt_str<'a>(maybestr: &'a Option<String>) -> &'a str {
235 pub fn opt_str2(maybestr: Option<String>) -> String {
237 None => "(none)".to_owned(),
242 pub fn run_tests(config: &Config) {
243 if config.target.contains("android") {
244 if let DebugInfoGdb = config.mode {
245 println!("{} debug-info test uses tcp 5039 port.\
246 please reserve it", config.target);
249 // android debug-info test uses remote debugger
250 // so, we test 1 thread at once.
251 // also trying to isolate problems with adb_run_wrapper.sh ilooping
252 env::set_var("RUST_TEST_THREADS","1");
257 if let Some(lldb_version) = config.lldb_version.as_ref() {
258 if is_blacklisted_lldb_version(&lldb_version[..]) {
259 println!("WARNING: The used version of LLDB ({}) has a \
260 known issue that breaks debuginfo tests. See \
261 issue #32520 for more information. Skipping all \
268 // Some older versions of LLDB seem to have problems with multiple
269 // instances running in parallel, so only run one test thread at a
271 env::set_var("RUST_TEST_THREADS", "1");
273 _ => { /* proceed */ }
276 // FIXME(#33435) Avoid spurious failures in codegen-units/partitioning tests.
277 if let Mode::CodegenUnits = config.mode {
278 let _ = fs::remove_dir_all("tmp/partitioning-tests");
281 let opts = test_opts(config);
282 let tests = make_tests(config);
283 // sadly osx needs some file descriptor limits raised for running tests in
284 // parallel (especially when we have lots and lots of child processes).
285 // For context, see #8904
286 unsafe { raise_fd_limit::raise_fd_limit(); }
287 // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
288 // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
289 env::set_var("__COMPAT_LAYER", "RunAsInvoker");
290 let res = test::run_tests_console(&opts, tests.into_iter().collect());
293 Ok(false) => panic!("Some tests failed"),
295 println!("I/O failure during tests: {:?}", e);
300 pub fn test_opts(config: &Config) -> test::TestOpts {
302 filter: config.filter.clone(),
303 run_ignored: config.run_ignored,
305 logfile: config.logfile.clone(),
307 bench_benchmarks: true,
308 nocapture: match env::var("RUST_TEST_NOCAPTURE") {
309 Ok(val) => &val != "0",
312 color: test::AutoColor,
316 pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
317 debug!("making tests from {:?}",
318 config.src_base.display());
319 let mut tests = Vec::new();
320 collect_tests_from_dir(config,
329 fn collect_tests_from_dir(config: &Config,
332 relative_dir_path: &Path,
333 tests: &mut Vec<test::TestDescAndFn>)
335 // Ignore directories that contain a file
336 // `compiletest-ignore-dir`.
337 for file in fs::read_dir(dir)? {
339 let name = file.file_name();
340 if name == *"compiletest-ignore-dir" {
343 if name == *"Makefile" && config.mode == Mode::RunMake {
344 let paths = TestPaths {
345 file: dir.to_path_buf(),
346 base: base.to_path_buf(),
347 relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
349 tests.push(make_test(config, &paths));
354 // If we find a test foo/bar.rs, we have to build the
355 // output directory `$build/foo` so we can write
356 // `$build/foo/bar` into it. We do this *now* in this
357 // sequential loop because otherwise, if we do it in the
358 // tests themselves, they race for the privilege of
359 // creating the directories and sometimes fail randomly.
360 let build_dir = config.build_base.join(&relative_dir_path);
361 fs::create_dir_all(&build_dir).unwrap();
363 // Add each `.rs` file as a test, and recurse further on any
364 // subdirectories we find, except for `aux` directories.
365 let dirs = fs::read_dir(dir)?;
368 let file_path = file.path();
369 let file_name = file.file_name();
370 if is_test(&file_name) {
371 debug!("found test file: {:?}", file_path.display());
372 let paths = TestPaths {
374 base: base.to_path_buf(),
375 relative_dir: relative_dir_path.to_path_buf(),
377 tests.push(make_test(config, &paths))
378 } else if file_path.is_dir() {
379 let relative_file_path = relative_dir_path.join(file.file_name());
380 if &file_name == "auxiliary" {
381 // `aux` directories contain other crates used for
382 // cross-crate tests. Don't search them for tests, but
383 // do create a directory in the build dir for them,
384 // since we will dump intermediate output in there
386 let build_dir = config.build_base.join(&relative_file_path);
387 fs::create_dir_all(&build_dir).unwrap();
389 debug!("found directory: {:?}", file_path.display());
390 collect_tests_from_dir(config,
397 debug!("found other file/directory: {:?}", file_path.display());
403 pub fn is_test(file_name: &OsString) -> bool {
404 let file_name = file_name.to_str().unwrap();
406 if !file_name.ends_with(".rs") {
410 // `.`, `#`, and `~` are common temp-file prefixes.
411 let invalid_prefixes = &[".", "#", "~"];
412 !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
415 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
416 let early_props = EarlyProps::from_file(config, &testpaths.file);
418 // The `should-fail` annotation doesn't apply to pretty tests,
419 // since we run the pretty printer across all tests by default.
420 // If desired, we could add a `should-fail-pretty` annotation.
421 let should_panic = match config.mode {
422 Pretty => test::ShouldPanic::No,
423 _ => if early_props.should_fail {
424 test::ShouldPanic::Yes
426 test::ShouldPanic::No
430 test::TestDescAndFn {
431 desc: test::TestDesc {
432 name: make_test_name(config, testpaths),
433 ignore: early_props.ignore,
434 should_panic: should_panic,
436 testfn: make_test_closure(config, testpaths),
440 pub fn make_test_name(config: &Config, testpaths: &TestPaths) -> test::TestName {
441 // Convert a complete path to something like
443 // run-pass/foo/bar/baz.rs
445 PathBuf::from(config.mode.to_string())
446 .join(&testpaths.relative_dir)
447 .join(&testpaths.file.file_name().unwrap());
448 test::DynTestName(format!("[{}] {}", config.mode, path.display()))
451 pub fn make_test_closure(config: &Config, testpaths: &TestPaths) -> test::TestFn {
452 let config = config.clone();
453 let testpaths = testpaths.clone();
454 test::DynTestFn(Box::new(move || {
455 runtest::run(config, &testpaths)
459 fn extract_gdb_version(full_version_line: Option<String>) -> Option<String> {
460 match full_version_line {
461 Some(ref full_version_line)
462 if !full_version_line.trim().is_empty() => {
463 let full_version_line = full_version_line.trim();
465 // used to be a regex "(^|[^0-9])([0-9]\.[0-9]+)"
466 for (pos, c) in full_version_line.char_indices() {
470 if pos + 2 >= full_version_line.len() {
473 if full_version_line[pos + 1..].chars().next().unwrap() != '.' {
476 if !full_version_line[pos + 2..].chars().next().unwrap().is_digit(10) {
479 if pos > 0 && full_version_line[..pos].chars().next_back()
480 .unwrap().is_digit(10) {
483 let mut end = pos + 3;
484 while end < full_version_line.len() &&
485 full_version_line[end..].chars().next()
486 .unwrap().is_digit(10) {
489 return Some(full_version_line[pos..end].to_owned());
491 println!("Could not extract GDB version from line '{}'",
499 fn extract_lldb_version(full_version_line: Option<String>) -> Option<String> {
500 // Extract the major LLDB version from the given version string.
501 // LLDB version strings are different for Apple and non-Apple platforms.
502 // At the moment, this function only supports the Apple variant, which looks
505 // LLDB-179.5 (older versions)
506 // lldb-300.2.51 (new versions)
508 // We are only interested in the major version number, so this function
509 // will return `Some("179")` and `Some("300")` respectively.
511 if let Some(ref full_version_line) = full_version_line {
512 if !full_version_line.trim().is_empty() {
513 let full_version_line = full_version_line.trim();
515 for (pos, l) in full_version_line.char_indices() {
516 if l != 'l' && l != 'L' { continue }
517 if pos + 5 >= full_version_line.len() { continue }
518 let l = full_version_line[pos + 1..].chars().next().unwrap();
519 if l != 'l' && l != 'L' { continue }
520 let d = full_version_line[pos + 2..].chars().next().unwrap();
521 if d != 'd' && d != 'D' { continue }
522 let b = full_version_line[pos + 3..].chars().next().unwrap();
523 if b != 'b' && b != 'B' { continue }
524 let dash = full_version_line[pos + 4..].chars().next().unwrap();
525 if dash != '-' { continue }
527 let vers = full_version_line[pos + 5..].chars().take_while(|c| {
529 }).collect::<String>();
530 if !vers.is_empty() { return Some(vers) }
532 println!("Could not extract LLDB version from line '{}'",
539 fn is_blacklisted_lldb_version(version: &str) -> bool {