1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![crate_name = "compiletest"]
13 #![feature(box_syntax)]
14 #![feature(rustc_private)]
23 extern crate rustc_serialize;
26 extern crate env_logger;
27 extern crate filetime;
31 use std::ffi::OsString;
34 use std::path::{Path, PathBuf};
35 use std::process::Command;
36 use filetime::FileTime;
37 use getopts::{optopt, optflag, reqopt};
39 use common::{Pretty, DebugInfoGdb, DebugInfoLldb, Mode};
43 use self::header::EarlyProps;
55 env_logger::init().unwrap();
57 let config = parse_config(env::args().collect());
59 if config.valgrind_path.is_none() && config.force_valgrind {
60 panic!("Can't find Valgrind to run Valgrind tests");
67 pub fn parse_config(args: Vec<String> ) -> Config {
69 let groups : Vec<getopts::OptGroup> =
70 vec![reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
71 reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
72 reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
73 reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
74 reqopt("", "lldb-python", "path to python to use for doc tests", "PATH"),
75 reqopt("", "docck-python", "path to python to use for doc tests", "PATH"),
76 optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
77 optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
78 optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR"),
79 reqopt("", "src-base", "directory to scan for test files", "PATH"),
80 reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
81 reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
82 reqopt("", "mode", "which sort of compile tests to run",
83 "(compile-fail|parse-fail|run-fail|run-pass|\
84 run-pass-valgrind|pretty|debug-info|incremental|mir-opt)"),
85 optflag("", "ignored", "run tests marked as ignored"),
86 optflag("", "exact", "filters match exactly"),
87 optopt("", "runtool", "supervisor program to run tests under \
88 (eg. emulator, valgrind)", "PROGRAM"),
89 optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
90 optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
91 optflag("", "verbose", "run tests verbosely, showing all output"),
92 optflag("", "quiet", "print one character per test instead of one line"),
93 optopt("", "logfile", "file to log test execution to", "FILE"),
94 optopt("", "target", "the target to build for", "TARGET"),
95 optopt("", "host", "the host to build for", "HOST"),
96 optopt("", "gdb", "path to GDB to use for GDB debuginfo tests", "PATH"),
97 optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
98 optopt("", "llvm-version", "the version of LLVM used", "VERSION STRING"),
99 optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
100 optopt("", "adb-path", "path to the android debugger", "PATH"),
101 optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
102 optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
103 reqopt("", "cc", "path to a C compiler", "PATH"),
104 reqopt("", "cxx", "path to a C++ compiler", "PATH"),
105 reqopt("", "cflags", "flags for the C compiler", "FLAGS"),
106 reqopt("", "llvm-components", "list of LLVM components built in", "LIST"),
107 reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"),
108 optopt("", "nodejs", "the name of nodejs", "PATH"),
109 optopt("", "remote-test-client", "path to the remote test client", "PATH"),
110 optflag("h", "help", "show this message")];
112 let (argv0, args_) = args.split_first().unwrap();
113 if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
114 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
115 println!("{}", getopts::usage(&message, &groups));
121 &match getopts::getopts(args_, &groups) {
123 Err(f) => panic!("{:?}", f)
126 if matches.opt_present("h") || matches.opt_present("help") {
127 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
128 println!("{}", getopts::usage(&message, &groups));
133 fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
134 match m.opt_str(nm) {
135 Some(s) => PathBuf::from(&s),
136 None => panic!("no option (=path) found for {}", nm),
140 fn make_absolute(path: PathBuf) -> PathBuf {
141 if path.is_relative() {
142 env::current_dir().unwrap().join(path)
148 let (gdb, gdb_version, gdb_native_rust) = analyze_gdb(matches.opt_str("gdb"));
151 compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
152 run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
153 rustc_path: opt_path(matches, "rustc-path"),
154 rustdoc_path: opt_path(matches, "rustdoc-path"),
155 lldb_python: matches.opt_str("lldb-python").unwrap(),
156 docck_python: matches.opt_str("docck-python").unwrap(),
157 valgrind_path: matches.opt_str("valgrind-path"),
158 force_valgrind: matches.opt_present("force-valgrind"),
159 llvm_filecheck: matches.opt_str("llvm-filecheck").map(|s| PathBuf::from(&s)),
160 src_base: opt_path(matches, "src-base"),
161 build_base: opt_path(matches, "build-base"),
162 stage_id: matches.opt_str("stage-id").unwrap(),
163 mode: matches.opt_str("mode").unwrap().parse().ok().expect("invalid mode"),
164 run_ignored: matches.opt_present("ignored"),
165 filter: matches.free.first().cloned(),
166 filter_exact: matches.opt_present("exact"),
167 logfile: matches.opt_str("logfile").map(|s| PathBuf::from(&s)),
168 runtool: matches.opt_str("runtool"),
169 host_rustcflags: matches.opt_str("host-rustcflags"),
170 target_rustcflags: matches.opt_str("target-rustcflags"),
171 target: opt_str2(matches.opt_str("target")),
172 host: opt_str2(matches.opt_str("host")),
174 gdb_version: gdb_version,
175 gdb_native_rust: gdb_native_rust,
176 lldb_version: extract_lldb_version(matches.opt_str("lldb-version")),
177 llvm_version: matches.opt_str("llvm-version"),
178 android_cross_path: opt_path(matches, "android-cross-path"),
179 adb_path: opt_str2(matches.opt_str("adb-path")),
180 adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
182 opt_str2(matches.opt_str("target")).contains("android") &&
183 "(none)" != opt_str2(matches.opt_str("adb-test-dir")) &&
184 !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
185 lldb_python_dir: matches.opt_str("lldb-python-dir"),
186 verbose: matches.opt_present("verbose"),
187 quiet: matches.opt_present("quiet"),
188 remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from),
190 cc: matches.opt_str("cc").unwrap(),
191 cxx: matches.opt_str("cxx").unwrap(),
192 cflags: matches.opt_str("cflags").unwrap(),
193 llvm_components: matches.opt_str("llvm-components").unwrap(),
194 llvm_cxxflags: matches.opt_str("llvm-cxxflags").unwrap(),
195 nodejs: matches.opt_str("nodejs"),
199 pub fn log_config(config: &Config) {
201 logv(c, format!("configuration:"));
202 logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
203 logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
204 logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
205 logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
206 logv(c, format!("src_base: {:?}", config.src_base.display()));
207 logv(c, format!("build_base: {:?}", config.build_base.display()));
208 logv(c, format!("stage_id: {}", config.stage_id));
209 logv(c, format!("mode: {}", config.mode));
210 logv(c, format!("run_ignored: {}", config.run_ignored));
211 logv(c, format!("filter: {}",
212 opt_str(&config.filter
214 .map(|re| re.to_owned()))));
215 logv(c, format!("filter_exact: {}", config.filter_exact));
216 logv(c, format!("runtool: {}", opt_str(&config.runtool)));
217 logv(c, format!("host-rustcflags: {}",
218 opt_str(&config.host_rustcflags)));
219 logv(c, format!("target-rustcflags: {}",
220 opt_str(&config.target_rustcflags)));
221 logv(c, format!("target: {}", config.target));
222 logv(c, format!("host: {}", config.host));
223 logv(c, format!("android-cross-path: {:?}",
224 config.android_cross_path.display()));
225 logv(c, format!("adb_path: {:?}", config.adb_path));
226 logv(c, format!("adb_test_dir: {:?}", config.adb_test_dir));
227 logv(c, format!("adb_device_status: {}",
228 config.adb_device_status));
229 logv(c, format!("verbose: {}", config.verbose));
230 logv(c, format!("quiet: {}", config.quiet));
231 logv(c, format!("\n"));
234 pub fn opt_str<'a>(maybestr: &'a Option<String>) -> &'a str {
241 pub fn opt_str2(maybestr: Option<String>) -> String {
243 None => "(none)".to_owned(),
248 pub fn run_tests(config: &Config) {
249 if config.target.contains("android") {
250 if let DebugInfoGdb = config.mode {
251 println!("{} debug-info test uses tcp 5039 port.\
252 please reserve it", config.target);
254 // android debug-info test uses remote debugger so, we test 1 thread
255 // at once as they're all sharing the same TCP port to communicate
258 // we should figure out how to lift this restriction! (run them all
259 // on different ports allocated dynamically).
260 env::set_var("RUST_TEST_THREADS", "1");
266 if let Some(lldb_version) = config.lldb_version.as_ref() {
267 if is_blacklisted_lldb_version(&lldb_version[..]) {
268 println!("WARNING: The used version of LLDB ({}) has a \
269 known issue that breaks debuginfo tests. See \
270 issue #32520 for more information. Skipping all \
277 // Some older versions of LLDB seem to have problems with multiple
278 // instances running in parallel, so only run one test thread at a
280 env::set_var("RUST_TEST_THREADS", "1");
284 if config.remote_test_client.is_some() &&
285 !config.target.contains("android"){
286 println!("WARNING: debuginfo tests are not available when \
287 testing with remote");
291 _ => { /* proceed */ }
294 // FIXME(#33435) Avoid spurious failures in codegen-units/partitioning tests.
295 if let Mode::CodegenUnits = config.mode {
296 let _ = fs::remove_dir_all("tmp/partitioning-tests");
299 let opts = test_opts(config);
300 let tests = make_tests(config);
301 // sadly osx needs some file descriptor limits raised for running tests in
302 // parallel (especially when we have lots and lots of child processes).
303 // For context, see #8904
304 unsafe { raise_fd_limit::raise_fd_limit(); }
305 // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
306 // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
307 env::set_var("__COMPAT_LAYER", "RunAsInvoker");
309 // Let tests know which target they're running as
310 env::set_var("TARGET", &config.target);
312 let res = test::run_tests_console(&opts, tests.into_iter().collect());
315 Ok(false) => panic!("Some tests failed"),
317 println!("I/O failure during tests: {:?}", e);
322 pub fn test_opts(config: &Config) -> test::TestOpts {
324 filter: config.filter.clone(),
325 filter_exact: config.filter_exact,
326 run_ignored: config.run_ignored,
328 logfile: config.logfile.clone(),
330 bench_benchmarks: true,
331 nocapture: match env::var("RUST_TEST_NOCAPTURE") {
332 Ok(val) => &val != "0",
335 color: test::AutoColor,
339 options: test::Options::new(),
343 pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
344 debug!("making tests from {:?}",
345 config.src_base.display());
346 let mut tests = Vec::new();
347 collect_tests_from_dir(config,
356 fn collect_tests_from_dir(config: &Config,
359 relative_dir_path: &Path,
360 tests: &mut Vec<test::TestDescAndFn>)
362 // Ignore directories that contain a file
363 // `compiletest-ignore-dir`.
364 for file in fs::read_dir(dir)? {
366 let name = file.file_name();
367 if name == *"compiletest-ignore-dir" {
370 if name == *"Makefile" && config.mode == Mode::RunMake {
371 let paths = TestPaths {
372 file: dir.to_path_buf(),
373 base: base.to_path_buf(),
374 relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
376 tests.push(make_test(config, &paths));
381 // If we find a test foo/bar.rs, we have to build the
382 // output directory `$build/foo` so we can write
383 // `$build/foo/bar` into it. We do this *now* in this
384 // sequential loop because otherwise, if we do it in the
385 // tests themselves, they race for the privilege of
386 // creating the directories and sometimes fail randomly.
387 let build_dir = config.build_base.join(&relative_dir_path);
388 fs::create_dir_all(&build_dir).unwrap();
390 // Add each `.rs` file as a test, and recurse further on any
391 // subdirectories we find, except for `aux` directories.
392 let dirs = fs::read_dir(dir)?;
395 let file_path = file.path();
396 let file_name = file.file_name();
397 if is_test(&file_name) {
398 debug!("found test file: {:?}", file_path.display());
399 let paths = TestPaths {
401 base: base.to_path_buf(),
402 relative_dir: relative_dir_path.to_path_buf(),
404 tests.push(make_test(config, &paths))
405 } else if file_path.is_dir() {
406 let relative_file_path = relative_dir_path.join(file.file_name());
407 if &file_name == "auxiliary" {
408 // `aux` directories contain other crates used for
409 // cross-crate tests. Don't search them for tests, but
410 // do create a directory in the build dir for them,
411 // since we will dump intermediate output in there
413 let build_dir = config.build_base.join(&relative_file_path);
414 fs::create_dir_all(&build_dir).unwrap();
416 debug!("found directory: {:?}", file_path.display());
417 collect_tests_from_dir(config,
424 debug!("found other file/directory: {:?}", file_path.display());
430 pub fn is_test(file_name: &OsString) -> bool {
431 let file_name = file_name.to_str().unwrap();
433 if !file_name.ends_with(".rs") {
437 // `.`, `#`, and `~` are common temp-file prefixes.
438 let invalid_prefixes = &[".", "#", "~"];
439 !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
442 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
443 let early_props = EarlyProps::from_file(config, &testpaths.file);
445 // The `should-fail` annotation doesn't apply to pretty tests,
446 // since we run the pretty printer across all tests by default.
447 // If desired, we could add a `should-fail-pretty` annotation.
448 let should_panic = match config.mode {
449 Pretty => test::ShouldPanic::No,
450 _ => if early_props.should_fail {
451 test::ShouldPanic::Yes
453 test::ShouldPanic::No
457 // Debugging emscripten code doesn't make sense today
458 let mut ignore = early_props.ignore || !up_to_date(config, testpaths, &early_props);
459 if (config.mode == DebugInfoGdb || config.mode == DebugInfoLldb) &&
460 config.target.contains("emscripten") {
464 test::TestDescAndFn {
465 desc: test::TestDesc {
466 name: make_test_name(config, testpaths),
468 should_panic: should_panic,
470 testfn: make_test_closure(config, testpaths),
474 fn stamp(config: &Config, testpaths: &TestPaths) -> PathBuf {
475 let stamp_name = format!("{}-{}.stamp",
476 testpaths.file.file_name().unwrap()
479 config.build_base.canonicalize()
480 .unwrap_or(config.build_base.clone())
484 fn up_to_date(config: &Config, testpaths: &TestPaths, props: &EarlyProps) -> bool {
485 let stamp = mtime(&stamp(config, testpaths));
486 let mut inputs = vec![
487 mtime(&testpaths.file),
488 mtime(&config.rustc_path),
490 for aux in props.aux.iter() {
491 inputs.push(mtime(&testpaths.file.parent().unwrap()
495 for lib in config.run_lib_path.read_dir().unwrap() {
496 let lib = lib.unwrap();
497 inputs.push(mtime(&lib.path()));
499 inputs.iter().any(|input| *input > stamp)
502 fn mtime(path: &Path) -> FileTime {
503 fs::metadata(path).map(|f| {
504 FileTime::from_last_modification_time(&f)
505 }).unwrap_or(FileTime::zero())
508 pub fn make_test_name(config: &Config, testpaths: &TestPaths) -> test::TestName {
509 // Convert a complete path to something like
511 // run-pass/foo/bar/baz.rs
513 PathBuf::from(config.mode.to_string())
514 .join(&testpaths.relative_dir)
515 .join(&testpaths.file.file_name().unwrap());
516 test::DynTestName(format!("[{}] {}", config.mode, path.display()))
519 pub fn make_test_closure(config: &Config, testpaths: &TestPaths) -> test::TestFn {
520 let config = config.clone();
521 let testpaths = testpaths.clone();
522 test::DynTestFn(Box::new(move |()| {
523 runtest::run(config, &testpaths)
527 /// Returns (Path to GDB, GDB Version, GDB has Rust Support)
528 fn analyze_gdb(gdb: Option<String>) -> (Option<String>, Option<u32>, bool) {
530 const GDB_FALLBACK: &str = "gdb";
532 const GDB_FALLBACK: &str = "gdb.exe";
534 const MIN_GDB_WITH_RUST: u32 = 7011010;
536 let gdb = match gdb {
537 None => GDB_FALLBACK,
538 Some(ref s) if s.is_empty() => GDB_FALLBACK, // may be empty if configure found no gdb
542 let version_line = Command::new(gdb).arg("--version").output().map(|output| {
543 String::from_utf8_lossy(&output.stdout).lines().next().unwrap().to_string()
546 let version = match version_line {
547 Some(line) => extract_gdb_version(&line),
548 None => return (None, None, false),
551 let gdb_native_rust = version.map_or(false, |v| v >= MIN_GDB_WITH_RUST);
553 return (Some(gdb.to_owned()), version, gdb_native_rust);
556 fn extract_gdb_version(full_version_line: &str) -> Option<u32> {
557 let full_version_line = full_version_line.trim();
559 // GDB versions look like this: "major.minor.patch?.yyyymmdd?", with both
560 // of the ? sections being optional
562 // We will parse up to 3 digits for minor and patch, ignoring the date
563 // We limit major to 1 digit, otherwise, on openSUSE, we parse the openSUSE version
565 // don't start parsing in the middle of a number
566 let mut prev_was_digit = false;
567 for (pos, c) in full_version_line.char_indices() {
568 if prev_was_digit || !c.is_digit(10) {
569 prev_was_digit = c.is_digit(10);
573 prev_was_digit = true;
575 let line = &full_version_line[pos..];
577 let next_split = match line.find(|c: char| !c.is_digit(10)) {
579 None => continue, // no minor version
582 if line.as_bytes()[next_split] != b'.' {
583 continue; // no minor version
586 let major = &line[..next_split];
587 let line = &line[next_split + 1..];
589 let (minor, patch) = match line.find(|c: char| !c.is_digit(10)) {
590 Some(idx) => if line.as_bytes()[idx] == b'.' {
591 let patch = &line[idx + 1..];
593 let patch_len = patch.find(|c: char| !c.is_digit(10)).unwrap_or(patch.len());
594 let patch = &patch[..patch_len];
595 let patch = if patch_len > 3 || patch_len == 0 { None } else { Some(patch) };
597 (&line[..idx], patch)
601 None => (line, None),
604 if major.len() != 1 || minor.is_empty() {
608 let major: u32 = major.parse().unwrap();
609 let minor: u32 = minor.parse().unwrap();
610 let patch: u32 = patch.unwrap_or("0").parse().unwrap();
612 return Some(((major * 1000) + minor) * 1000 + patch);
618 fn extract_lldb_version(full_version_line: Option<String>) -> Option<String> {
619 // Extract the major LLDB version from the given version string.
620 // LLDB version strings are different for Apple and non-Apple platforms.
621 // At the moment, this function only supports the Apple variant, which looks
624 // LLDB-179.5 (older versions)
625 // lldb-300.2.51 (new versions)
627 // We are only interested in the major version number, so this function
628 // will return `Some("179")` and `Some("300")` respectively.
630 if let Some(ref full_version_line) = full_version_line {
631 if !full_version_line.trim().is_empty() {
632 let full_version_line = full_version_line.trim();
634 for (pos, l) in full_version_line.char_indices() {
635 if l != 'l' && l != 'L' { continue }
636 if pos + 5 >= full_version_line.len() { continue }
637 let l = full_version_line[pos + 1..].chars().next().unwrap();
638 if l != 'l' && l != 'L' { continue }
639 let d = full_version_line[pos + 2..].chars().next().unwrap();
640 if d != 'd' && d != 'D' { continue }
641 let b = full_version_line[pos + 3..].chars().next().unwrap();
642 if b != 'b' && b != 'B' { continue }
643 let dash = full_version_line[pos + 4..].chars().next().unwrap();
644 if dash != '-' { continue }
646 let vers = full_version_line[pos + 5..].chars().take_while(|c| {
648 }).collect::<String>();
649 if !vers.is_empty() { return Some(vers) }
656 fn is_blacklisted_lldb_version(version: &str) -> bool {
661 fn test_extract_gdb_version() {
662 macro_rules! test { ($($expectation:tt: $input:tt,)*) => {{$(
663 assert_eq!(extract_gdb_version($input), Some($expectation));
667 7000001: "GNU gdb (GDB) CentOS (7.0.1-45.el5.centos)",
669 7002000: "GNU gdb (GDB) Red Hat Enterprise Linux (7.2-90.el6)",
671 7004000: "GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04",
672 7004001: "GNU gdb (GDB) 7.4.1-debian",
674 7006001: "GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7",
676 7007001: "GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1",
677 7007001: "GNU gdb (Debian 7.7.1+dfsg-5) 7.7.1",
678 7007001: "GNU gdb (GDB) Fedora 7.7.1-21.fc20",
680 7008000: "GNU gdb (GDB; openSUSE 13.2) 7.8",
681 7009001: "GNU gdb (GDB) Fedora 7.9.1-20.fc22",
682 7010001: "GNU gdb (GDB) Fedora 7.10.1-31.fc23",
684 7011000: "GNU gdb (Ubuntu 7.11-0ubuntu1) 7.11",
685 7011001: "GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.04) 7.11.1",
686 7011001: "GNU gdb (Debian 7.11.1-2) 7.11.1",
687 7011001: "GNU gdb (GDB) Fedora 7.11.1-86.fc24",
688 7011001: "GNU gdb (GDB; openSUSE Leap 42.1) 7.11.1",
689 7011001: "GNU gdb (GDB; openSUSE Tumbleweed) 7.11.1",
692 7011090: "GNU gdb (Ubuntu 7.11.90.20161005-0ubuntu1) 7.11.90.20161005-git",
695 7012000: "GNU gdb (GDB) 7.12",
696 7012000: "GNU gdb (GDB) 7.12.20161027-git",
697 7012050: "GNU gdb (GDB) 7.12.50.20161027-git",