extern mod extra;
use std::os;
+use std::f64;
use extra::getopts;
use extra::getopts::groups::{optopt, optflag, reqopt};
optopt("", "rustcflags", "flags to pass to rustc", "FLAGS"),
optflag("", "verbose", "run tests verbosely, showing all output"),
optopt("", "logfile", "file to log test execution to", "FILE"),
+ optopt("", "save-metrics", "file to save metrics to", "FILE"),
+ optopt("", "ratchet-metrics", "file to ratchet metrics against", "FILE"),
+ optopt("", "ratchet-noise-percent",
+ "percent change in metrics to consider noise", "N"),
optflag("", "jit", "run tests under the JIT"),
optflag("", "newrt", "run tests on the new runtime / scheduler"),
optopt("", "target", "the target to build for", "TARGET"),
Some(copy matches.free[0])
} else { None },
logfile: getopts::opt_maybe_str(matches, "logfile").map(|s| Path(*s)),
+ save_metrics: getopts::opt_maybe_str(matches, "save-metrics").map(|s| Path(*s)),
+ ratchet_metrics:
+ getopts::opt_maybe_str(matches, "ratchet-metrics").map(|s| Path(*s)),
+ ratchet_noise_percent:
+ getopts::opt_maybe_str(matches,
+ "ratchet-noise-percent").map(|s|
+ f64::from_str(*s).get()),
runtool: getopts::opt_maybe_str(matches, "runtool"),
rustcflags: getopts::opt_maybe_str(matches, "rustcflags"),
jit: getopts::opt_present(matches, "jit"),
run_ignored: config.run_ignored,
logfile: copy config.logfile,
run_tests: true,
- run_benchmarks: false,
- ratchet_metrics: None,
- ratchet_noise_percent: None,
- save_metrics: None,
+ run_benchmarks: true,
+ ratchet_metrics: copy config.ratchet_metrics,
+ ratchet_noise_percent: copy config.ratchet_noise_percent,
+ save_metrics: copy config.save_metrics,
}
}
let file = copy *file;
debug!("inspecting file %s", file.to_str());
if is_test(config, file) {
- tests.push(make_test(config, file))
+ let t = do make_test(config, file) {
+ match config.mode {
+ mode_codegen => make_metrics_test_closure(config, file),
+ _ => make_test_closure(config, file)
+ }
+ };
+ tests.push(t)
}
}
tests
return valid;
}
-pub fn make_test(config: &config, testfile: &Path) -> test::TestDescAndFn {
+pub fn make_test(config: &config, testfile: &Path,
+ f: &fn()->test::TestFn) -> test::TestDescAndFn {
test::TestDescAndFn {
desc: test::TestDesc {
name: make_test_name(config, testfile),
ignore: header::is_test_ignored(config, testfile),
should_fail: false
},
- testfn: make_test_closure(config, testfile),
+ testfn: f(),
}
}
let testfile = Cell::new(testfile.to_str());
test::DynTestFn(|| { runtest::run(config.take(), testfile.take()) })
}
+
+pub fn make_metrics_test_closure(config: &config, testfile: &Path) -> test::TestFn {
+ use std::cell::Cell;
+ let config = Cell::new(copy *config);
+ let testfile = Cell::new(testfile.to_str());
+ test::DynMetricFn(|mm| { runtest::run_metrics(config.take(), testfile.take(), mm) })
+}
use std::uint;
use std::vec;
+use extra::test::MetricMap;
+
pub fn run(config: config, testfile: ~str) {
+ let mut _mm = MetricMap::new();
+ run_metrics(config, testfile, &mut _mm);
+}
+
+pub fn run_metrics(config: config, testfile: ~str, mm: &mut MetricMap) {
if config.verbose {
// We're going to be dumping a lot of info. Start on a new line.
io::stdout().write_str("\n\n");
mode_run_pass => run_rpass_test(&config, &props, &testfile),
mode_pretty => run_pretty_test(&config, &props, &testfile),
mode_debug_info => run_debuginfo_test(&config, &props, &testfile),
- mode_codegen => run_codegen_test(&config, &props, &testfile)
+ mode_codegen => run_codegen_test(&config, &props, &testfile, mm)
}
}
}
-fn run_codegen_test(config: &config, props: &TestProps, testfile: &Path) {
+fn count_extracted_lines(p: &Path) -> uint {
+ let x = io::read_whole_file_str(&p.with_filetype("ll")).get();
+ x.line_iter().len_()
+}
+
+
+fn run_codegen_test(config: &config, props: &TestProps,
+ testfile: &Path, mm: &mut MetricMap) {
if config.llvm_bin_path.is_none() {
fatal(~"missing --llvm-bin-path");
fatal_ProcRes(~"disassembling extract failed", &ProcRes);
}
+ let base = output_base_name(config, testfile);
+ let base_extract = append_suffix_to_stem(&base, "extract");
+
+ let base_clang = append_suffix_to_stem(&base, "clang");
+ let base_clang_extract = append_suffix_to_stem(&base_clang, "extract");
+ let base_lines = count_extracted_lines(&base_extract);
+ let clang_lines = count_extracted_lines(&base_clang_extract);
+ mm.insert_metric("clang-codegen-ratio",
+ (base_lines as f64) / (clang_lines as f64),
+ 0.001);
}