2 use std::path::{Path, PathBuf};
3 use std::process::{Command, ExitStatus};
4 use std::sync::atomic::{AtomicUsize, Ordering};
8 use comments::ErrorMatch;
9 use crossbeam::queue::SegQueue;
12 use crate::comments::Comments;
20 /// Arguments passed to the binary that is executed.
21 pub args: Vec<String>,
22 /// `None` to run on the host, otherwise a target triple
23 pub target: Option<String>,
24 /// Filters applied to stderr output before processing it
25 pub stderr_filters: Filter,
26 /// Filters applied to stdout output before processing it
27 pub stdout_filters: Filter,
28 /// The folder in which to start searching for .rs files
29 pub root_dir: PathBuf,
32 pub output_conflict_handling: OutputConflictHandling,
33 /// Only run tests with one of these strings in their path/name
34 pub path_filter: Vec<String>,
38 pub enum OutputConflictHandling {
39 /// The default: emit a diff of the expected/actual output.
41 /// Ignore mismatches in the stderr/stdout files.
43 /// Instead of erroring if the stderr/stdout differs from the expected
44 /// automatically replace it with the found output (after applying filters).
48 pub type Filter = Vec<(Regex, &'static str)>;
50 pub fn run_tests(config: Config) {
51 eprintln!(" Compiler flags: {:?}", config.args);
53 // Get the triple with which to run the tests
54 let target = config.target.clone().unwrap_or_else(|| config.get_host());
56 // A queue for files or folders to process
57 let todo = SegQueue::new();
58 todo.push(config.root_dir.clone());
60 // Some statistics and failure reports.
61 let failures = Mutex::new(vec![]);
62 let succeeded = AtomicUsize::default();
63 let ignored = AtomicUsize::default();
64 let filtered = AtomicUsize::default();
66 crossbeam::scope(|s| {
67 for _ in 0..std::thread::available_parallelism().unwrap().get() {
69 while let Some(path) = todo.pop() {
70 // Collect everything inside directories
72 for entry in std::fs::read_dir(path).unwrap() {
73 todo.push(entry.unwrap().path());
77 // Only look at .rs files
78 if !path.extension().map(|ext| ext == "rs").unwrap_or(false) {
81 if !config.path_filter.is_empty() {
82 let path_display = path.display().to_string();
83 if !config.path_filter.iter().any(|filter| path_display.contains(filter)) {
84 filtered.fetch_add(1, Ordering::Relaxed);
88 let comments = Comments::parse_file(&path);
89 // Ignore file if only/ignore rules do (not) apply
90 if ignore_file(&comments, &target) {
91 ignored.fetch_add(1, Ordering::Relaxed);
92 eprintln!("{} ... {}", path.display(), "ignored (in-test comment)".yellow());
95 // Run the test for all revisions
97 comments.revisions.clone().unwrap_or_else(|| vec![String::new()])
99 let (m, errors) = run_test(&path, &config, &target, &revision, &comments);
101 // Using a single `eprintln!` to prevent messages from threads from getting intermingled.
102 let mut msg = format!("{} ", path.display());
103 if !revision.is_empty() {
104 write!(msg, "(revision `{revision}`) ").unwrap();
106 write!(msg, "... ").unwrap();
107 if errors.is_empty() {
108 eprintln!("{msg}{}", "ok".green());
109 succeeded.fetch_add(1, Ordering::Relaxed);
111 eprintln!("{msg}{}", "FAILED".red().bold());
112 failures.lock().unwrap().push((path.clone(), m, revision, errors));
121 // Print all errors in a single thread to show reliable output
122 let failures = failures.into_inner().unwrap();
123 let succeeded = succeeded.load(Ordering::Relaxed);
124 let ignored = ignored.load(Ordering::Relaxed);
125 let filtered = filtered.load(Ordering::Relaxed);
126 if !failures.is_empty() {
127 for (path, miri, revision, errors) in &failures {
129 eprint!("{}", path.display().to_string().underline());
130 if !revision.is_empty() {
131 eprint!(" (revision `{}`)", revision);
133 eprint!(" {}", "FAILED".red());
135 eprintln!("command: {:?}", miri);
137 let mut dump_stderr = None;
138 for error in errors {
140 Error::ExitStatus(mode, exit_status) => eprintln!("{mode:?} got {exit_status}"),
141 Error::PatternNotFound { stderr, pattern, definition_line } => {
142 eprintln!("`{pattern}` {} in stderr output", "not found".red());
144 "expected because of pattern here: {}:{definition_line}",
145 path.display().to_string().bold()
147 dump_stderr = Some(stderr.clone())
149 Error::NoPatternsFound =>
150 eprintln!("{}", "no error patterns found in failure test".red()),
151 Error::PatternFoundInPassTest =>
152 eprintln!("{}", "error pattern found in success test".red()),
153 Error::OutputDiffers { path, actual, expected } => {
155 eprintln!("actual output differed from expected {}", path.display());
156 eprintln!("{}", pretty_assertions::StrComparison::new(expected, actual));
162 if let Some(stderr) = dump_stderr {
163 eprintln!("actual stderr:");
164 eprintln!("{}", stderr);
169 "test result: {}. {} tests failed, {} tests passed, {} ignored, {} filtered out",
171 failures.len().to_string().red().bold(),
172 succeeded.to_string().green(),
173 ignored.to_string().yellow(),
174 filtered.to_string().yellow(),
176 std::process::exit(1);
180 "test result: {}. {} tests passed, {} ignored, {} filtered out",
182 succeeded.to_string().green(),
183 ignored.to_string().yellow(),
184 filtered.to_string().yellow(),
191 /// Got an invalid exit status for the given mode.
192 ExitStatus(Mode, ExitStatus),
196 definition_line: usize,
198 /// A ui test checking for failure does not have any failure patterns
200 /// A ui test checking for success has failure patterns
201 PatternFoundInPassTest,
202 /// Stderr/Stdout differed from the `.stderr`/`.stdout` file present.
210 type Errors = Vec<Error>;
218 ) -> (Command, Errors) {
220 let mut miri = Command::new(&config.program);
221 miri.args(config.args.iter());
223 if !revision.is_empty() {
224 miri.arg(format!("--cfg={revision}"));
226 for arg in &comments.compile_flags {
229 for (k, v) in &comments.env_vars {
232 let output = miri.output().expect("could not execute miri");
233 let mut errors = config.mode.ok(output.status);
234 // Always remove annotation comments from stderr.
235 let annotations = Regex::new(r"\s*//~.*").unwrap();
236 let stderr = std::str::from_utf8(&output.stderr).unwrap();
237 let stderr = annotations.replace_all(stderr, "");
238 let stdout = std::str::from_utf8(&output.stdout).unwrap();
239 // Check output files (if any)
240 let revised = |extension: &str| {
241 if revision.is_empty() {
242 extension.to_string()
244 format!("{}.{}", revision, extension)
247 // Check output files against actual output
254 &config.stderr_filters,
264 &config.stdout_filters,
268 // Check error annotations in the source against output
269 check_annotations(&stderr, &mut errors, config, revision, comments);
273 fn check_annotations(
274 unnormalized_stderr: &str,
280 let mut found_annotation = false;
281 if let Some((ref error_pattern, definition_line)) = comments.error_pattern {
282 if !unnormalized_stderr.contains(error_pattern) {
283 errors.push(Error::PatternNotFound {
284 stderr: unnormalized_stderr.to_string(),
285 pattern: error_pattern.to_string(),
289 found_annotation = true;
291 for &ErrorMatch { ref matched, revision: ref rev, definition_line } in &comments.error_matches {
292 // FIXME: check that the error happens on the marked line
294 if let Some(rev) = rev {
300 if !unnormalized_stderr.contains(matched) {
301 errors.push(Error::PatternNotFound {
302 stderr: unnormalized_stderr.to_string(),
303 pattern: matched.to_string(),
307 found_annotation = true;
309 match (config.mode, found_annotation) {
310 (Mode::Pass, true) | (Mode::Panic, true) => errors.push(Error::PatternFoundInPassTest),
311 (Mode::Fail, false) => errors.push(Error::NoPatternsFound),
326 let output = normalize(path, output, filters, comments);
327 let path = output_path(path, comments, kind, target);
328 match config.output_conflict_handling {
329 OutputConflictHandling::Bless =>
330 if output.is_empty() {
331 let _ = std::fs::remove_file(path);
333 std::fs::write(path, &output).unwrap();
335 OutputConflictHandling::Error => {
336 let expected_output = std::fs::read_to_string(&path).unwrap_or_default();
337 if output != expected_output {
338 errors.push(Error::OutputDiffers {
341 expected: expected_output,
345 OutputConflictHandling::Ignore => {}
349 fn output_path(path: &Path, comments: &Comments, kind: String, target: &str) -> PathBuf {
350 if comments.stderr_per_bitwidth {
351 return path.with_extension(format!("{}.{kind}", get_pointer_width(target)));
353 path.with_extension(kind)
356 fn ignore_file(comments: &Comments, target: &str) -> bool {
357 for s in &comments.ignore {
358 if target.contains(s) {
361 if get_pointer_width(target) == s {
365 for s in &comments.only {
366 if !target.contains(s) {
369 if get_pointer_width(target) != s {
376 // Taken 1:1 from compiletest-rs
377 fn get_pointer_width(triple: &str) -> &'static str {
378 if (triple.contains("64") && !triple.ends_with("gnux32") && !triple.ends_with("gnu_ilp32"))
379 || triple.starts_with("s390x")
382 } else if triple.starts_with("avr") {
389 fn normalize(path: &Path, text: &str, filters: &Filter, comments: &Comments) -> String {
391 let mut text = text.replace(&path.parent().unwrap().display().to_string(), "$DIR");
392 if let Some(lib_path) = option_env!("RUSTC_LIB_PATH") {
393 text = text.replace(lib_path, "RUSTLIB");
396 for (regex, replacement) in filters.iter() {
397 text = regex.replace_all(&text, *replacement).to_string();
400 for (from, to) in &comments.normalize_stderr {
401 text = from.replace_all(&text, to).to_string();
407 fn get_host(&self) -> String {
408 rustc_version::VersionMeta::for_command(std::process::Command::new(&self.program))
409 .expect("failed to parse rustc version info")
414 #[derive(Copy, Clone, Debug)]
416 // The test passes a full execution of the rustc driver
418 // The rustc driver panicked
420 // The rustc driver emitted an error
425 fn ok(self, status: ExitStatus) -> Errors {
426 match (status.code().unwrap(), self) {
427 (1, Mode::Fail) | (101, Mode::Panic) | (0, Mode::Pass) => vec![],
428 _ => vec![Error::ExitStatus(self, status)],