1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![crate_type = "bin"]
12 #![feature(phase, slicing_syntax, globs, unboxed_closures)]
18 #[phase(plugin, link)] extern crate log;
25 use std::str::FromStr;
26 use std::thunk::Thunk;
27 use getopts::{optopt, optflag, reqopt};
29 use common::{Pretty, DebugInfoGdb, DebugInfoLldb, Codegen};
41 let args = os::args();
42 let config = parse_config(args);
44 if config.valgrind_path.is_none() && config.force_valgrind {
45 panic!("Can't find Valgrind to run Valgrind tests");
52 pub fn parse_config(args: Vec<String> ) -> Config {
54 let groups : Vec<getopts::OptGroup> =
55 vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
56 reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
57 reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
58 optopt("", "clang-path", "path to executable for codegen tests", "PATH"),
59 optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
60 optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
61 optopt("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR"),
62 reqopt("", "src-base", "directory to scan for test files", "PATH"),
63 reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
64 reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
65 reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
66 reqopt("", "mode", "which sort of compile tests to run",
67 "(compile-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)"),
68 optflag("", "ignored", "run tests marked as ignored"),
69 optopt("", "runtool", "supervisor program to run tests under \
70 (eg. emulator, valgrind)", "PROGRAM"),
71 optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
72 optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
73 optflag("", "verbose", "run tests verbosely, showing all output"),
74 optopt("", "logfile", "file to log test execution to", "FILE"),
75 optopt("", "save-metrics", "file to save metrics to", "FILE"),
76 optopt("", "ratchet-metrics", "file to ratchet metrics against", "FILE"),
77 optopt("", "ratchet-noise-percent",
78 "percent change in metrics to consider noise", "N"),
79 optflag("", "jit", "run tests under the JIT"),
80 optopt("", "target", "the target to build for", "TARGET"),
81 optopt("", "host", "the host to build for", "HOST"),
82 optopt("", "gdb-version", "the version of GDB used", "VERSION STRING"),
83 optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
84 optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
85 optopt("", "adb-path", "path to the android debugger", "PATH"),
86 optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
87 optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
88 optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite", "A.B"),
89 optflag("h", "help", "show this message"));
91 assert!(!args.is_empty());
92 let argv0 = args[0].clone();
93 let args_ = args.tail();
94 if args[1].as_slice() == "-h" || args[1].as_slice() == "--help" {
95 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
96 println!("{}", getopts::usage(message.as_slice(), groups.as_slice()));
102 &match getopts::getopts(args_.as_slice(), groups.as_slice()) {
104 Err(f) => panic!("{}", f)
107 if matches.opt_present("h") || matches.opt_present("help") {
108 let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
109 println!("{}", getopts::usage(message.as_slice(), groups.as_slice()));
114 fn opt_path(m: &getopts::Matches, nm: &str) -> Path {
115 Path::new(m.opt_str(nm).unwrap())
118 let filter = if !matches.free.is_empty() {
119 let s = matches.free[0].as_slice();
120 match regex::Regex::new(s) {
123 println!("failed to parse filter /{}/: {}", s, e);
132 compile_lib_path: matches.opt_str("compile-lib-path").unwrap(),
133 run_lib_path: matches.opt_str("run-lib-path").unwrap(),
134 rustc_path: opt_path(matches, "rustc-path"),
135 clang_path: matches.opt_str("clang-path").map(|s| Path::new(s)),
136 valgrind_path: matches.opt_str("valgrind-path"),
137 force_valgrind: matches.opt_present("force-valgrind"),
138 llvm_bin_path: matches.opt_str("llvm-bin-path").map(|s| Path::new(s)),
139 src_base: opt_path(matches, "src-base"),
140 build_base: opt_path(matches, "build-base"),
141 aux_base: opt_path(matches, "aux-base"),
142 stage_id: matches.opt_str("stage-id").unwrap(),
143 mode: FromStr::from_str(matches.opt_str("mode")
145 .as_slice()).expect("invalid mode"),
146 run_ignored: matches.opt_present("ignored"),
148 cfail_regex: Regex::new(errors::EXPECTED_PATTERN).unwrap(),
149 logfile: matches.opt_str("logfile").map(|s| Path::new(s)),
150 save_metrics: matches.opt_str("save-metrics").map(|s| Path::new(s)),
152 matches.opt_str("ratchet-metrics").map(|s| Path::new(s)),
153 ratchet_noise_percent:
154 matches.opt_str("ratchet-noise-percent")
155 .and_then(|s| s.as_slice().parse::<f64>()),
156 runtool: matches.opt_str("runtool"),
157 host_rustcflags: matches.opt_str("host-rustcflags"),
158 target_rustcflags: matches.opt_str("target-rustcflags"),
159 jit: matches.opt_present("jit"),
160 target: opt_str2(matches.opt_str("target")),
161 host: opt_str2(matches.opt_str("host")),
162 gdb_version: extract_gdb_version(matches.opt_str("gdb-version")),
163 lldb_version: extract_lldb_version(matches.opt_str("lldb-version")),
164 android_cross_path: opt_path(matches, "android-cross-path"),
165 adb_path: opt_str2(matches.opt_str("adb-path")),
166 adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")),
168 "arm-linux-androideabi" ==
169 opt_str2(matches.opt_str("target")).as_slice() &&
171 opt_str2(matches.opt_str("adb-test-dir")).as_slice() &&
172 !opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
173 lldb_python_dir: matches.opt_str("lldb-python-dir"),
174 test_shard: test::opt_shard(matches.opt_str("test-shard")),
175 verbose: matches.opt_present("verbose"),
179 pub fn log_config(config: &Config) {
181 logv(c, format!("configuration:"));
182 logv(c, format!("compile_lib_path: {}", config.compile_lib_path));
183 logv(c, format!("run_lib_path: {}", config.run_lib_path));
184 logv(c, format!("rustc_path: {}", config.rustc_path.display()));
185 logv(c, format!("src_base: {}", config.src_base.display()));
186 logv(c, format!("build_base: {}", config.build_base.display()));
187 logv(c, format!("stage_id: {}", config.stage_id));
188 logv(c, format!("mode: {}", config.mode));
189 logv(c, format!("run_ignored: {}", config.run_ignored));
190 logv(c, format!("filter: {}",
191 opt_str(&config.filter
193 .map(|re| re.to_string()))));
194 logv(c, format!("runtool: {}", opt_str(&config.runtool)));
195 logv(c, format!("host-rustcflags: {}",
196 opt_str(&config.host_rustcflags)));
197 logv(c, format!("target-rustcflags: {}",
198 opt_str(&config.target_rustcflags)));
199 logv(c, format!("jit: {}", config.jit));
200 logv(c, format!("target: {}", config.target));
201 logv(c, format!("host: {}", config.host));
202 logv(c, format!("android-cross-path: {}",
203 config.android_cross_path.display()));
204 logv(c, format!("adb_path: {}", config.adb_path));
205 logv(c, format!("adb_test_dir: {}", config.adb_test_dir));
206 logv(c, format!("adb_device_status: {}",
207 config.adb_device_status));
208 match config.test_shard {
209 None => logv(c, "test_shard: (all)".to_string()),
210 Some((a,b)) => logv(c, format!("test_shard: {}.{}", a, b))
212 logv(c, format!("verbose: {}", config.verbose));
213 logv(c, format!("\n"));
216 pub fn opt_str<'a>(maybestr: &'a Option<String>) -> &'a str {
219 Some(ref s) => s.as_slice(),
223 pub fn opt_str2(maybestr: Option<String>) -> String {
225 None => "(none)".to_string(),
230 pub fn run_tests(config: &Config) {
231 if config.target.as_slice() == "arm-linux-androideabi" {
234 println!("arm-linux-androideabi debug-info \
235 test uses tcp 5039 port. please reserve it");
240 //arm-linux-androideabi debug-info test uses remote debugger
241 //so, we test 1 task at once.
242 // also trying to isolate problems with adb_run_wrapper.sh ilooping
243 os::setenv("RUST_TEST_TASKS","1");
248 // Some older versions of LLDB seem to have problems with multiple
249 // instances running in parallel, so only run one test task at a
251 os::setenv("RUST_TEST_TASKS", "1");
253 _ => { /* proceed */ }
256 let opts = test_opts(config);
257 let tests = make_tests(config);
258 // sadly osx needs some file descriptor limits raised for running tests in
259 // parallel (especially when we have lots and lots of child processes).
260 // For context, see #8904
261 io::test::raise_fd_limit();
262 let res = test::run_tests_console(&opts, tests.into_iter().collect());
265 Ok(false) => panic!("Some tests failed"),
267 println!("I/O failure during tests: {}", e);
272 pub fn test_opts(config: &Config) -> test::TestOpts {
274 filter: match config.filter {
276 Some(ref filter) => Some(filter.clone()),
278 run_ignored: config.run_ignored,
279 logfile: config.logfile.clone(),
281 run_benchmarks: true,
282 ratchet_metrics: config.ratchet_metrics.clone(),
283 ratchet_noise_percent: config.ratchet_noise_percent.clone(),
284 save_metrics: config.save_metrics.clone(),
285 test_shard: config.test_shard.clone(),
287 color: test::AutoColor,
290 show_all_stats: false,
294 pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
295 debug!("making tests from {}",
296 config.src_base.display());
297 let mut tests = Vec::new();
298 let dirs = fs::readdir(&config.src_base).unwrap();
299 for file in dirs.iter() {
300 let file = file.clone();
301 debug!("inspecting file {}", file.display());
302 if is_test(config, &file) {
303 let t = make_test(config, &file, || {
305 Codegen => make_metrics_test_closure(config, &file),
306 _ => make_test_closure(config, &file)
315 pub fn is_test(config: &Config, testfile: &Path) -> bool {
316 // Pretty-printer does not work with .rc files yet
317 let valid_extensions =
319 Pretty => vec!(".rs".to_string()),
320 _ => vec!(".rc".to_string(), ".rs".to_string())
322 let invalid_prefixes = vec!(".".to_string(), "#".to_string(), "~".to_string());
323 let name = testfile.filename_str().unwrap();
325 let mut valid = false;
327 for ext in valid_extensions.iter() {
328 if name.ends_with(ext.as_slice()) {
333 for pre in invalid_prefixes.iter() {
334 if name.starts_with(pre.as_slice()) {
342 pub fn make_test<F>(config: &Config, testfile: &Path, f: F) -> test::TestDescAndFn where
343 F: FnOnce() -> test::TestFn,
345 test::TestDescAndFn {
346 desc: test::TestDesc {
347 name: make_test_name(config, testfile),
348 ignore: header::is_test_ignored(config, testfile),
349 should_fail: test::ShouldFail::No,
355 pub fn make_test_name(config: &Config, testfile: &Path) -> test::TestName {
357 // Try to elide redundant long paths
358 fn shorten(path: &Path) -> String {
359 let filename = path.filename_str();
360 let p = path.dir_path();
361 let dir = p.filename_str();
362 format!("{}/{}", dir.unwrap_or(""), filename.unwrap_or(""))
365 test::DynTestName(format!("[{}] {}", config.mode, shorten(testfile)))
368 pub fn make_test_closure(config: &Config, testfile: &Path) -> test::TestFn {
369 let config = (*config).clone();
370 // FIXME (#9639): This needs to handle non-utf8 paths
371 let testfile = testfile.as_str().unwrap().to_string();
372 test::DynTestFn(Thunk::new(move || {
373 runtest::run(config, testfile)
377 pub fn make_metrics_test_closure(config: &Config, testfile: &Path) -> test::TestFn {
378 let config = (*config).clone();
379 // FIXME (#9639): This needs to handle non-utf8 paths
380 let testfile = testfile.as_str().unwrap().to_string();
381 test::DynMetricFn(box move |: mm: &mut test::MetricMap| {
382 runtest::run_metrics(config, testfile, mm)
386 fn extract_gdb_version(full_version_line: Option<String>) -> Option<String> {
387 match full_version_line {
388 Some(ref full_version_line)
389 if full_version_line.as_slice().trim().len() > 0 => {
390 let full_version_line = full_version_line.as_slice().trim();
392 let re = Regex::new(r"(^|[^0-9])([0-9]\.[0-9])([^0-9]|$)").unwrap();
394 match re.captures(full_version_line) {
396 Some(captures.at(2).unwrap_or("").to_string())
399 println!("Could not extract GDB version from line '{}'",
409 fn extract_lldb_version(full_version_line: Option<String>) -> Option<String> {
410 // Extract the major LLDB version from the given version string.
411 // LLDB version strings are different for Apple and non-Apple platforms.
412 // At the moment, this function only supports the Apple variant, which looks
415 // LLDB-179.5 (older versions)
416 // lldb-300.2.51 (new versions)
418 // We are only interested in the major version number, so this function
419 // will return `Some("179")` and `Some("300")` respectively.
421 match full_version_line {
422 Some(ref full_version_line)
423 if full_version_line.as_slice().trim().len() > 0 => {
424 let full_version_line = full_version_line.as_slice().trim();
426 let re = Regex::new(r"[Ll][Ll][Dd][Bb]-([0-9]+)").unwrap();
428 match re.captures(full_version_line) {
430 Some(captures.at(1).unwrap_or("").to_string())
433 println!("Could not extract LLDB version from line '{}'",