2 use std::path::{Path, PathBuf};
3 use std::process::{Command, ExitStatus};
4 use std::sync::atomic::{AtomicUsize, Ordering};
8 use comments::ErrorMatch;
9 use crossbeam::queue::SegQueue;
11 use rustc_stderr::{Level, Message};
13 use crate::comments::Comments;
22 /// Arguments passed to the binary that is executed.
23 pub args: Vec<String>,
24 /// `None` to run on the host, otherwise a target triple
25 pub target: Option<String>,
26 /// Filters applied to stderr output before processing it
27 pub stderr_filters: Filter,
28 /// Filters applied to stdout output before processing it
29 pub stdout_filters: Filter,
30 /// The folder in which to start searching for .rs files
31 pub root_dir: PathBuf,
34 pub output_conflict_handling: OutputConflictHandling,
35 /// Only run tests with one of these strings in their path/name
36 pub path_filter: Vec<String>,
40 pub enum OutputConflictHandling {
41 /// The default: emit a diff of the expected/actual output.
43 /// Ignore mismatches in the stderr/stdout files.
45 /// Instead of erroring if the stderr/stdout differs from the expected
46 /// automatically replace it with the found output (after applying filters).
50 pub type Filter = Vec<(Regex, &'static str)>;
52 pub fn run_tests(config: Config) {
53 eprintln!(" Compiler flags: {:?}", config.args);
55 // Get the triple with which to run the tests
56 let target = config.target.clone().unwrap_or_else(|| config.get_host());
58 // A queue for files or folders to process
59 let todo = SegQueue::new();
60 todo.push(config.root_dir.clone());
62 // Some statistics and failure reports.
63 let failures = Mutex::new(vec![]);
64 let succeeded = AtomicUsize::default();
65 let ignored = AtomicUsize::default();
66 let filtered = AtomicUsize::default();
68 crossbeam::scope(|s| {
69 for _ in 0..std::thread::available_parallelism().unwrap().get() {
71 while let Some(path) = todo.pop() {
72 // Collect everything inside directories
74 for entry in std::fs::read_dir(path).unwrap() {
75 todo.push(entry.unwrap().path());
79 // Only look at .rs files
80 if !path.extension().map(|ext| ext == "rs").unwrap_or(false) {
83 if !config.path_filter.is_empty() {
84 let path_display = path.display().to_string();
85 if !config.path_filter.iter().any(|filter| path_display.contains(filter)) {
86 filtered.fetch_add(1, Ordering::Relaxed);
90 let comments = Comments::parse_file(&path);
91 // Ignore file if only/ignore rules do (not) apply
92 if ignore_file(&comments, &target) {
93 ignored.fetch_add(1, Ordering::Relaxed);
97 "ignored (in-test comment)".yellow()
101 // Run the test for all revisions
103 comments.revisions.clone().unwrap_or_else(|| vec![String::new()])
105 let (m, errors, stderr) =
106 run_test(&path, &config, &target, &revision, &comments);
108 // Using a single `eprintln!` to prevent messages from threads from getting intermingled.
109 let mut msg = format!("{} ", path.display());
110 if !revision.is_empty() {
111 write!(msg, "(revision `{revision}`) ").unwrap();
113 write!(msg, "... ").unwrap();
114 if errors.is_empty() {
115 eprintln!("{msg}{}", "ok".green());
116 succeeded.fetch_add(1, Ordering::Relaxed);
118 eprintln!("{msg}{}", "FAILED".red().bold());
119 failures.lock().unwrap().push((
134 // Print all errors in a single thread to show reliable output
135 let failures = failures.into_inner().unwrap();
136 let succeeded = succeeded.load(Ordering::Relaxed);
137 let ignored = ignored.load(Ordering::Relaxed);
138 let filtered = filtered.load(Ordering::Relaxed);
139 if !failures.is_empty() {
140 for (path, miri, revision, errors, stderr) in &failures {
142 eprint!("{}", path.display().to_string().underline());
143 if !revision.is_empty() {
144 eprint!(" (revision `{}`)", revision);
146 eprint!(" {}", "FAILED".red());
148 eprintln!("command: {:?}", miri);
150 // `None` means never dump, as we already dumped it for an `OutputDiffers`
151 // `Some(false)` means there's no reason to dump, as all errors are independent of the stderr
152 // `Some(true)` means that there was a pattern in the .rs file that was not found in the output.
153 let mut dump_stderr = Some(false);
154 for error in errors {
156 Error::ExitStatus(mode, exit_status) => eprintln!("{mode:?} got {exit_status}"),
157 Error::PatternNotFound { pattern, definition_line } => {
158 eprintln!("`{pattern}` {} in stderr output", "not found".red());
160 "expected because of pattern here: {}:{definition_line}",
161 path.display().to_string().bold()
163 dump_stderr = dump_stderr.map(|_| true);
165 Error::NoPatternsFound => {
166 eprintln!("{}", "no error patterns found in failure test".red());
168 Error::PatternFoundInPassTest =>
169 eprintln!("{}", "error pattern found in success test".red()),
170 Error::OutputDiffers { path, actual, expected } => {
171 if path.extension().unwrap() == "stderr" {
174 eprintln!("actual output differed from expected {}", path.display());
175 eprintln!("{}", pretty_assertions::StrComparison::new(expected, actual));
178 Error::ErrorsWithoutPattern { path: None, msgs } => {
180 "There were {} unmatched diagnostics that occurred outside the testfile and had not pattern",
183 for Message { level, message } in msgs {
184 eprintln!(" {level:?}: {message}")
187 Error::ErrorsWithoutPattern { path: Some((path, line)), msgs } => {
189 "There were {} unmatched diagnostics at {}:{line}",
193 for Message { level, message } in msgs {
194 eprintln!(" {level:?}: {message}")
197 Error::ErrorPatternWithoutErrorAnnotation(path, line) => {
199 "Annotation at {}:{line} matched an error diagnostic but did not have `ERROR` before its message",
206 if let Some(true) = dump_stderr {
207 eprintln!("actual stderr:");
208 eprintln!("{}", stderr);
213 "test result: {}. {} tests failed, {} tests passed, {} ignored, {} filtered out",
215 failures.len().to_string().red().bold(),
216 succeeded.to_string().green(),
217 ignored.to_string().yellow(),
218 filtered.to_string().yellow(),
220 std::process::exit(1);
224 "test result: {}. {} tests passed, {} ignored, {} filtered out",
226 succeeded.to_string().green(),
227 ignored.to_string().yellow(),
228 filtered.to_string().yellow(),
235 /// Got an invalid exit status for the given mode.
236 ExitStatus(Mode, ExitStatus),
239 definition_line: usize,
241 /// A ui test checking for failure does not have any failure patterns
243 /// A ui test checking for success has failure patterns
244 PatternFoundInPassTest,
245 /// Stderr/Stdout differed from the `.stderr`/`.stdout` file present.
251 ErrorsWithoutPattern {
253 path: Option<(PathBuf, usize)>,
255 ErrorPatternWithoutErrorAnnotation(PathBuf, usize),
258 type Errors = Vec<Error>;
266 ) -> (Command, Errors, String) {
268 let mut miri = Command::new(&config.program);
269 miri.args(config.args.iter());
271 if !revision.is_empty() {
272 miri.arg(format!("--cfg={revision}"));
274 miri.arg("--error-format=json");
275 for arg in &comments.compile_flags {
278 for (k, v) in &comments.env_vars {
281 let output = miri.output().expect("could not execute miri");
282 let mut errors = config.mode.ok(output.status);
283 let stderr = check_test_result(
293 (miri, errors, stderr)
296 fn check_test_result(
306 // Always remove annotation comments from stderr.
307 let diagnostics = rustc_stderr::process(path, stderr);
308 let stdout = std::str::from_utf8(stdout).unwrap();
309 // Check output files (if any)
310 let revised = |extension: &str| {
311 if revision.is_empty() {
312 extension.to_string()
314 format!("{}.{}", revision, extension)
317 // Check output files against actual output
319 &diagnostics.rendered,
324 &config.stderr_filters,
334 &config.stdout_filters,
338 // Check error annotations in the source against output
340 diagnostics.messages,
341 diagnostics.messages_from_unknown_file_or_line,
351 fn check_annotations(
352 mut messages: Vec<Vec<Message>>,
353 mut messages_from_unknown_file_or_line: Vec<Message>,
360 if let Some((ref error_pattern, definition_line)) = comments.error_pattern {
361 let mut found = false;
363 // first check the diagnostics messages outside of our file. We check this first, so that
364 // you can mix in-file annotations with // error-pattern annotations, even if there is overlap
366 if let Some(i) = messages_from_unknown_file_or_line
368 .position(|msg| msg.message.contains(error_pattern))
370 messages_from_unknown_file_or_line.remove(i);
374 // if nothing was found, check the ones inside our file. We permit this because some tests may have
375 // flaky line numbers for their messages.
377 for line in &mut messages {
378 if let Some(i) = line.iter().position(|msg| msg.message.contains(error_pattern)) {
387 errors.push(Error::PatternNotFound {
388 pattern: error_pattern.to_string(),
394 // The order on `Level` is such that `Error` is the highest level.
395 // We will ensure that *all* diagnostics of level at least `lowest_annotation_level`
397 let mut lowest_annotation_level = Level::Error;
398 for &ErrorMatch { ref matched, revision: ref rev, definition_line, line, level } in
399 &comments.error_matches
401 if let Some(rev) = rev {
406 if let Some(level) = level {
407 // If we found a diagnostic with a level annotation, make sure that all
408 // diagnostics of that level have annotations, even if we don't end up finding a matching diagnostic
410 lowest_annotation_level = std::cmp::min(lowest_annotation_level, level);
413 if let Some(msgs) = messages.get_mut(line) {
414 let found = msgs.iter().position(|msg| {
415 msg.message.contains(matched)
416 // in case there is no level on the annotation, match any level.
417 && level.map_or(true, |level| {
421 if let Some(found) = found {
422 let msg = msgs.remove(found);
423 if msg.level == Level::Error && level.is_none() {
425 .push(Error::ErrorPatternWithoutErrorAnnotation(path.to_path_buf(), line));
431 errors.push(Error::PatternNotFound { pattern: matched.to_string(), definition_line });
434 let filter = |msgs: Vec<Message>| -> Vec<_> {
435 msgs.into_iter().filter(|msg| msg.level >= lowest_annotation_level).collect()
438 let messages_from_unknown_file_or_line = filter(messages_from_unknown_file_or_line);
439 if !messages_from_unknown_file_or_line.is_empty() {
440 errors.push(Error::ErrorsWithoutPattern {
442 msgs: messages_from_unknown_file_or_line,
446 for (line, msgs) in messages.into_iter().enumerate() {
447 let msgs = filter(msgs);
448 if !msgs.is_empty() {
450 .push(Error::ErrorsWithoutPattern { path: Some((path.to_path_buf(), line)), msgs });
454 match (config.mode, comments.error_pattern.is_some() || !comments.error_matches.is_empty()) {
455 (Mode::Pass, true) | (Mode::Panic, true) => errors.push(Error::PatternFoundInPassTest),
456 (Mode::Fail, false) => errors.push(Error::NoPatternsFound),
471 let output = normalize(path, output, filters, comments);
472 let path = output_path(path, comments, kind, target);
473 match config.output_conflict_handling {
474 OutputConflictHandling::Bless =>
475 if output.is_empty() {
476 let _ = std::fs::remove_file(path);
478 std::fs::write(path, &output).unwrap();
480 OutputConflictHandling::Error => {
481 let expected_output = std::fs::read_to_string(&path).unwrap_or_default();
482 if output != expected_output {
483 errors.push(Error::OutputDiffers {
486 expected: expected_output,
490 OutputConflictHandling::Ignore => {}
494 fn output_path(path: &Path, comments: &Comments, kind: String, target: &str) -> PathBuf {
495 if comments.stderr_per_bitwidth {
496 return path.with_extension(format!("{}.{kind}", get_pointer_width(target)));
498 path.with_extension(kind)
501 fn ignore_file(comments: &Comments, target: &str) -> bool {
502 for s in &comments.ignore {
503 if target.contains(s) {
506 if get_pointer_width(target) == s {
510 for s in &comments.only {
511 if !target.contains(s) {
514 if get_pointer_width(target) != s {
521 // Taken 1:1 from compiletest-rs
522 fn get_pointer_width(triple: &str) -> &'static str {
523 if (triple.contains("64") && !triple.ends_with("gnux32") && !triple.ends_with("gnu_ilp32"))
524 || triple.starts_with("s390x")
527 } else if triple.starts_with("avr") {
534 fn normalize(path: &Path, text: &str, filters: &Filter, comments: &Comments) -> String {
536 let mut text = text.replace(&path.parent().unwrap().display().to_string(), "$DIR");
537 if let Some(lib_path) = option_env!("RUSTC_LIB_PATH") {
538 text = text.replace(lib_path, "RUSTLIB");
541 for (regex, replacement) in filters.iter() {
542 text = regex.replace_all(&text, *replacement).to_string();
545 for (from, to) in &comments.normalize_stderr {
546 text = from.replace_all(&text, to).to_string();
552 fn get_host(&self) -> String {
553 rustc_version::VersionMeta::for_command(std::process::Command::new(&self.program))
554 .expect("failed to parse rustc version info")
559 #[derive(Copy, Clone, Debug)]
561 // The test passes a full execution of the rustc driver
563 // The rustc driver panicked
565 // The rustc driver emitted an error
570 fn ok(self, status: ExitStatus) -> Errors {
571 match (status.code().unwrap(), self) {
572 (1, Mode::Fail) | (101, Mode::Panic) | (0, Mode::Pass) => vec![],
573 _ => vec![Error::ExitStatus(self, status)],