1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![crate_name = "compiletest"]
13 #![feature(box_syntax)]
14 #![feature(rustc_private)]
15 #![feature(static_in_const)]
26 extern crate rustc_serialize;
27 #[cfg(not(cargobuild))]
28 extern crate serialize as rustc_serialize;
34 extern crate env_logger;
37 use std::ffi::OsString;
40 use std::path::{Path, PathBuf};
41 use std::process::Command;
42 use getopts::{optopt, optflag, reqopt};
44 use common::{Pretty, DebugInfoGdb, DebugInfoLldb, Mode};
48 use self::header::EarlyProps;
62 fn log_init() { env_logger::init().unwrap(); }
63 #[cfg(not(cargobuild))]
67 let config = parse_config(env::args().collect());
69 if config.valgrind_path.is_none() && config.force_valgrind {
70 panic!("Can't find Valgrind to run Valgrind tests");
77 pub fn parse_config(args: Vec<String> ) -> Config {
79 let groups : Vec<getopts::OptGroup> =
80 vec![reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
81 reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
82 reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
83 reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
84 reqopt("", "lldb-python", "path to python to use for doc tests", "PATH"),
85 reqopt("", "docck-python", "path to python to use for doc tests", "PATH"),
86 optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
87 optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
88 optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR"),
89 reqopt("", "src-base", "directory to scan for test files", "PATH"),
90 reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
91 reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
92 reqopt("", "mode", "which sort of compile tests to run",
93 "(compile-fail|parse-fail|run-fail|run-pass|\
94 run-pass-valgrind|pretty|debug-info|incremental|mir-opt)"),
95 optflag("", "ignored", "run tests marked as ignored"),
96 optflag("", "exact", "filters match exactly"),
97 optopt("", "runtool", "supervisor program to run tests under \
98 (eg. emulator, valgrind)", "PROGRAM"),
99 optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
100 optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
101 optflag("", "verbose", "run tests verbosely, showing all output"),
102 optflag("", "quiet", "print one character per test instead of one line"),
103 optopt("", "logfile", "file to log test execution to", "FILE"),
104 optopt("", "target", "the target to build for", "TARGET"),
105 optopt("", "host", "the host to build for", "HOST"),
106 optopt("", "gdb", "path to GDB to use for GDB debuginfo tests", "PATH"),
107 optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
108 optopt("", "llvm-version", "the version of LLVM used", "VERSION STRING"),
109 optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
110 optopt("", "adb-path", "path to the android debugger", "PATH"),
111 optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
112 optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
113 reqopt("", "cc", "path to a C compiler", "PATH"),
114 reqopt("", "cxx", "path to a C++ compiler", "PATH"),
115 reqopt("", "cflags", "flags for the C compiler", "FLAGS"),
116 reqopt("", "llvm-components", "list of LLVM components built in", "LIST"),
117 reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"),
118 optopt("", "nodejs", "the name of nodejs", "PATH"),
119 optflag("h", "help", "show this message")];
121 let (argv0, args_) = args.split_first().unwrap();
122 if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
123 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
124 println!("{}", getopts::usage(&message, &groups));
130 &match getopts::getopts(args_, &groups) {
132 Err(f) => panic!("{:?}", f)
135 if matches.opt_present("h") || matches.opt_present("help") {
136 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
137 println!("{}", getopts::usage(&message, &groups));
142 fn opt_path(m: &getopts::Matches, nm: &str) -> PathBuf {
143 match m.opt_str(nm) {
144 Some(s) => PathBuf::from(&s),
145 None => panic!("no option (=path) found for {}", nm),
149 fn make_absolute(path: PathBuf) -> PathBuf {
150 if path.is_relative() {
151 env::current_dir().unwrap().join(path)
157 let (gdb, gdb_version, gdb_native_rust) = analyze_gdb(matches.opt_str("gdb"));
160 compile_lib_path: make_absolute(opt_path(matches, "compile-lib-path")),
161 run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
162 rustc_path: opt_path(matches, "rustc-path"),
163 rustdoc_path: opt_path(matches, "rustdoc-path"),
164 lldb_python: matches.opt_str("lldb-python").unwrap(),
165 docck_python: matches.opt_str("docck-python").unwrap(),
166 valgrind_path: matches.opt_str("valgrind-path"),
167 force_valgrind: matches.opt_present("force-valgrind"),
168 llvm_filecheck: matches.opt_str("llvm-filecheck").map(|s| PathBuf::from(&s)),
169 src_base: opt_path(matches, "src-base"),
170 build_base: opt_path(matches, "build-base"),
171 stage_id: matches.opt_str("stage-id").unwrap(),
172 mode: matches.opt_str("mode").unwrap().parse().ok().expect("invalid mode"),
173 run_ignored: matches.opt_present("ignored"),
174 filter: matches.free.first().cloned(),
175 filter_exact: matches.opt_present("exact"),
176 logfile: matches.opt_str("logfile").map(|s| PathBuf::from(&s)),
177 runtool: matches.opt_str("runtool"),
178 host_rustcflags: matches.opt_str("host-rustcflags"),
179 target_rustcflags: matches.opt_str("target-rustcflags"),
180 target: opt_str2(matches.opt_str("target")),
181 host: opt_str2(matches.opt_str("host")),
183 gdb_version: gdb_version,
184 gdb_native_rust: gdb_native_rust,
185 lldb_version: extract_lldb_version(matches.opt_str("lldb-version")),
186 llvm_version: matches.opt_str("llvm-version"),
187 android_cross_path: opt_path(matches, "android-cross-path"),
188 adb_path: opt_str2(matches.opt_str("adb-path")),
189 adb_test_dir: format!("{}/{}",
190 opt_str2(matches.opt_str("adb-test-dir")),
191 opt_str2(matches.opt_str("target"))),
193 opt_str2(matches.opt_str("target")).contains("android") &&
194 "(none)" != opt_str2(matches.opt_str("adb-test-dir")) &&
195 !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
196 lldb_python_dir: matches.opt_str("lldb-python-dir"),
197 verbose: matches.opt_present("verbose"),
198 quiet: matches.opt_present("quiet"),
200 cc: matches.opt_str("cc").unwrap(),
201 cxx: matches.opt_str("cxx").unwrap(),
202 cflags: matches.opt_str("cflags").unwrap(),
203 llvm_components: matches.opt_str("llvm-components").unwrap(),
204 llvm_cxxflags: matches.opt_str("llvm-cxxflags").unwrap(),
205 nodejs: matches.opt_str("nodejs"),
209 pub fn log_config(config: &Config) {
211 logv(c, format!("configuration:"));
212 logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
213 logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
214 logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
215 logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
216 logv(c, format!("src_base: {:?}", config.src_base.display()));
217 logv(c, format!("build_base: {:?}", config.build_base.display()));
218 logv(c, format!("stage_id: {}", config.stage_id));
219 logv(c, format!("mode: {}", config.mode));
220 logv(c, format!("run_ignored: {}", config.run_ignored));
221 logv(c, format!("filter: {}",
222 opt_str(&config.filter
224 .map(|re| re.to_owned()))));
225 logv(c, format!("filter_exact: {}", config.filter_exact));
226 logv(c, format!("runtool: {}", opt_str(&config.runtool)));
227 logv(c, format!("host-rustcflags: {}",
228 opt_str(&config.host_rustcflags)));
229 logv(c, format!("target-rustcflags: {}",
230 opt_str(&config.target_rustcflags)));
231 logv(c, format!("target: {}", config.target));
232 logv(c, format!("host: {}", config.host));
233 logv(c, format!("android-cross-path: {:?}",
234 config.android_cross_path.display()));
235 logv(c, format!("adb_path: {:?}", config.adb_path));
236 logv(c, format!("adb_test_dir: {:?}", config.adb_test_dir));
237 logv(c, format!("adb_device_status: {}",
238 config.adb_device_status));
239 logv(c, format!("verbose: {}", config.verbose));
240 logv(c, format!("quiet: {}", config.quiet));
241 logv(c, format!("\n"));
244 pub fn opt_str<'a>(maybestr: &'a Option<String>) -> &'a str {
251 pub fn opt_str2(maybestr: Option<String>) -> String {
253 None => "(none)".to_owned(),
258 pub fn run_tests(config: &Config) {
259 if config.target.contains("android") {
260 if let DebugInfoGdb = config.mode {
261 println!("{} debug-info test uses tcp 5039 port.\
262 please reserve it", config.target);
265 // android debug-info test uses remote debugger
266 // so, we test 1 thread at once.
267 // also trying to isolate problems with adb_run_wrapper.sh ilooping
269 // These tests don't actually run code or don't run for android, so
270 // we don't need to limit ourselves there
281 env::set_var("RUST_TEST_THREADS", "1");
289 if let Some(lldb_version) = config.lldb_version.as_ref() {
290 if is_blacklisted_lldb_version(&lldb_version[..]) {
291 println!("WARNING: The used version of LLDB ({}) has a \
292 known issue that breaks debuginfo tests. See \
293 issue #32520 for more information. Skipping all \
300 // Some older versions of LLDB seem to have problems with multiple
301 // instances running in parallel, so only run one test thread at a
303 env::set_var("RUST_TEST_THREADS", "1");
305 _ => { /* proceed */ }
308 // FIXME(#33435) Avoid spurious failures in codegen-units/partitioning tests.
309 if let Mode::CodegenUnits = config.mode {
310 let _ = fs::remove_dir_all("tmp/partitioning-tests");
313 let opts = test_opts(config);
314 let tests = make_tests(config);
315 // sadly osx needs some file descriptor limits raised for running tests in
316 // parallel (especially when we have lots and lots of child processes).
317 // For context, see #8904
318 unsafe { raise_fd_limit::raise_fd_limit(); }
319 // Prevent issue #21352 UAC blocking .exe containing 'patch' etc. on Windows
320 // If #11207 is resolved (adding manifest to .exe) this becomes unnecessary
321 env::set_var("__COMPAT_LAYER", "RunAsInvoker");
323 // Let tests know which target they're running as
324 env::set_var("TARGET", &config.target);
326 let res = test::run_tests_console(&opts, tests.into_iter().collect());
329 Ok(false) => panic!("Some tests failed"),
331 println!("I/O failure during tests: {:?}", e);
336 pub fn test_opts(config: &Config) -> test::TestOpts {
338 filter: config.filter.clone(),
339 filter_exact: config.filter_exact,
340 run_ignored: config.run_ignored,
342 logfile: config.logfile.clone(),
344 bench_benchmarks: true,
345 nocapture: match env::var("RUST_TEST_NOCAPTURE") {
346 Ok(val) => &val != "0",
349 color: test::AutoColor,
356 pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
357 debug!("making tests from {:?}",
358 config.src_base.display());
359 let mut tests = Vec::new();
360 collect_tests_from_dir(config,
369 fn collect_tests_from_dir(config: &Config,
372 relative_dir_path: &Path,
373 tests: &mut Vec<test::TestDescAndFn>)
375 // Ignore directories that contain a file
376 // `compiletest-ignore-dir`.
377 for file in fs::read_dir(dir)? {
379 let name = file.file_name();
380 if name == *"compiletest-ignore-dir" {
383 if name == *"Makefile" && config.mode == Mode::RunMake {
384 let paths = TestPaths {
385 file: dir.to_path_buf(),
386 base: base.to_path_buf(),
387 relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
389 tests.push(make_test(config, &paths));
394 // If we find a test foo/bar.rs, we have to build the
395 // output directory `$build/foo` so we can write
396 // `$build/foo/bar` into it. We do this *now* in this
397 // sequential loop because otherwise, if we do it in the
398 // tests themselves, they race for the privilege of
399 // creating the directories and sometimes fail randomly.
400 let build_dir = config.build_base.join(&relative_dir_path);
401 fs::create_dir_all(&build_dir).unwrap();
403 // Add each `.rs` file as a test, and recurse further on any
404 // subdirectories we find, except for `aux` directories.
405 let dirs = fs::read_dir(dir)?;
408 let file_path = file.path();
409 let file_name = file.file_name();
410 if is_test(&file_name) {
411 debug!("found test file: {:?}", file_path.display());
412 let paths = TestPaths {
414 base: base.to_path_buf(),
415 relative_dir: relative_dir_path.to_path_buf(),
417 tests.push(make_test(config, &paths))
418 } else if file_path.is_dir() {
419 let relative_file_path = relative_dir_path.join(file.file_name());
420 if &file_name == "auxiliary" {
421 // `aux` directories contain other crates used for
422 // cross-crate tests. Don't search them for tests, but
423 // do create a directory in the build dir for them,
424 // since we will dump intermediate output in there
426 let build_dir = config.build_base.join(&relative_file_path);
427 fs::create_dir_all(&build_dir).unwrap();
429 debug!("found directory: {:?}", file_path.display());
430 collect_tests_from_dir(config,
437 debug!("found other file/directory: {:?}", file_path.display());
443 pub fn is_test(file_name: &OsString) -> bool {
444 let file_name = file_name.to_str().unwrap();
446 if !file_name.ends_with(".rs") {
450 // `.`, `#`, and `~` are common temp-file prefixes.
451 let invalid_prefixes = &[".", "#", "~"];
452 !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
455 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
456 let early_props = EarlyProps::from_file(config, &testpaths.file);
458 // The `should-fail` annotation doesn't apply to pretty tests,
459 // since we run the pretty printer across all tests by default.
460 // If desired, we could add a `should-fail-pretty` annotation.
461 let should_panic = match config.mode {
462 Pretty => test::ShouldPanic::No,
463 _ => if early_props.should_fail {
464 test::ShouldPanic::Yes
466 test::ShouldPanic::No
470 // Debugging emscripten code doesn't make sense today
471 let mut ignore = early_props.ignore;
472 if (config.mode == DebugInfoGdb || config.mode == DebugInfoLldb) &&
473 config.target.contains("emscripten") {
477 test::TestDescAndFn {
478 desc: test::TestDesc {
479 name: make_test_name(config, testpaths),
481 should_panic: should_panic,
483 testfn: make_test_closure(config, testpaths),
487 pub fn make_test_name(config: &Config, testpaths: &TestPaths) -> test::TestName {
488 // Convert a complete path to something like
490 // run-pass/foo/bar/baz.rs
492 PathBuf::from(config.mode.to_string())
493 .join(&testpaths.relative_dir)
494 .join(&testpaths.file.file_name().unwrap());
495 test::DynTestName(format!("[{}] {}", config.mode, path.display()))
498 pub fn make_test_closure(config: &Config, testpaths: &TestPaths) -> test::TestFn {
499 let config = config.clone();
500 let testpaths = testpaths.clone();
501 test::DynTestFn(Box::new(move |()| {
502 runtest::run(config, &testpaths)
506 /// Returns (Path to GDB, GDB Version, GDB has Rust Support)
507 fn analyze_gdb(gdb: Option<String>) -> (Option<String>, Option<u32>, bool) {
509 const GDB_FALLBACK: &str = "gdb";
511 const GDB_FALLBACK: &str = "gdb.exe";
513 const MIN_GDB_WITH_RUST: u32 = 7011010;
515 let gdb = match gdb {
516 None => GDB_FALLBACK,
517 Some(ref s) if s.is_empty() => GDB_FALLBACK, // may be empty if configure found no gdb
521 let version_line = Command::new(gdb).arg("--version").output().map(|output| {
522 String::from_utf8_lossy(&output.stdout).lines().next().unwrap().to_string()
525 let version = match version_line {
526 Some(line) => extract_gdb_version(&line),
527 None => return (None, None, false),
530 let gdb_native_rust = version.map_or(false, |v| v >= MIN_GDB_WITH_RUST);
532 return (Some(gdb.to_owned()), version, gdb_native_rust);
535 fn extract_gdb_version(full_version_line: &str) -> Option<u32> {
536 let full_version_line = full_version_line.trim();
538 // GDB versions look like this: "major.minor.patch?.yyyymmdd?", with both
539 // of the ? sections being optional
541 // We will parse up to 3 digits for minor and patch, ignoring the date
542 // We limit major to 1 digit, otherwise, on openSUSE, we parse the openSUSE version
544 // don't start parsing in the middle of a number
545 let mut prev_was_digit = false;
546 for (pos, c) in full_version_line.char_indices() {
547 if prev_was_digit || !c.is_digit(10) {
548 prev_was_digit = c.is_digit(10);
552 prev_was_digit = true;
554 let line = &full_version_line[pos..];
556 let next_split = match line.find(|c: char| !c.is_digit(10)) {
558 None => continue, // no minor version
561 if line.as_bytes()[next_split] != b'.' {
562 continue; // no minor version
565 let major = &line[..next_split];
566 let line = &line[next_split + 1..];
568 let (minor, patch) = match line.find(|c: char| !c.is_digit(10)) {
569 Some(idx) => if line.as_bytes()[idx] == b'.' {
570 let patch = &line[idx + 1..];
572 let patch_len = patch.find(|c: char| !c.is_digit(10)).unwrap_or(patch.len());
573 let patch = &patch[..patch_len];
574 let patch = if patch_len > 3 || patch_len == 0 { None } else { Some(patch) };
576 (&line[..idx], patch)
580 None => (line, None),
583 if major.len() != 1 || minor.is_empty() {
587 let major: u32 = major.parse().unwrap();
588 let minor: u32 = minor.parse().unwrap();
589 let patch: u32 = patch.unwrap_or("0").parse().unwrap();
591 return Some(((major * 1000) + minor) * 1000 + patch);
597 fn extract_lldb_version(full_version_line: Option<String>) -> Option<String> {
598 // Extract the major LLDB version from the given version string.
599 // LLDB version strings are different for Apple and non-Apple platforms.
600 // At the moment, this function only supports the Apple variant, which looks
603 // LLDB-179.5 (older versions)
604 // lldb-300.2.51 (new versions)
606 // We are only interested in the major version number, so this function
607 // will return `Some("179")` and `Some("300")` respectively.
609 if let Some(ref full_version_line) = full_version_line {
610 if !full_version_line.trim().is_empty() {
611 let full_version_line = full_version_line.trim();
613 for (pos, l) in full_version_line.char_indices() {
614 if l != 'l' && l != 'L' { continue }
615 if pos + 5 >= full_version_line.len() { continue }
616 let l = full_version_line[pos + 1..].chars().next().unwrap();
617 if l != 'l' && l != 'L' { continue }
618 let d = full_version_line[pos + 2..].chars().next().unwrap();
619 if d != 'd' && d != 'D' { continue }
620 let b = full_version_line[pos + 3..].chars().next().unwrap();
621 if b != 'b' && b != 'B' { continue }
622 let dash = full_version_line[pos + 4..].chars().next().unwrap();
623 if dash != '-' { continue }
625 let vers = full_version_line[pos + 5..].chars().take_while(|c| {
627 }).collect::<String>();
628 if !vers.is_empty() { return Some(vers) }
635 fn is_blacklisted_lldb_version(version: &str) -> bool {
640 fn test_extract_gdb_version() {
641 macro_rules! test { ($($expectation:tt: $input:tt,)*) => {{$(
642 assert_eq!(extract_gdb_version($input), Some($expectation));
646 7000001: "GNU gdb (GDB) CentOS (7.0.1-45.el5.centos)",
648 7002000: "GNU gdb (GDB) Red Hat Enterprise Linux (7.2-90.el6)",
650 7004000: "GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04",
651 7004001: "GNU gdb (GDB) 7.4.1-debian",
653 7006001: "GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7",
655 7007001: "GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1",
656 7007001: "GNU gdb (Debian 7.7.1+dfsg-5) 7.7.1",
657 7007001: "GNU gdb (GDB) Fedora 7.7.1-21.fc20",
659 7008000: "GNU gdb (GDB; openSUSE 13.2) 7.8",
660 7009001: "GNU gdb (GDB) Fedora 7.9.1-20.fc22",
661 7010001: "GNU gdb (GDB) Fedora 7.10.1-31.fc23",
663 7011000: "GNU gdb (Ubuntu 7.11-0ubuntu1) 7.11",
664 7011001: "GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.04) 7.11.1",
665 7011001: "GNU gdb (Debian 7.11.1-2) 7.11.1",
666 7011001: "GNU gdb (GDB) Fedora 7.11.1-86.fc24",
667 7011001: "GNU gdb (GDB; openSUSE Leap 42.1) 7.11.1",
668 7011001: "GNU gdb (GDB; openSUSE Tumbleweed) 7.11.1",
671 7011090: "GNU gdb (Ubuntu 7.11.90.20161005-0ubuntu1) 7.11.90.20161005-git",
674 7012000: "GNU gdb (GDB) 7.12",
675 7012000: "GNU gdb (GDB) 7.12.20161027-git",
676 7012050: "GNU gdb (GDB) 7.12.50.20161027-git",