1 #![allow(clippy::enum_variant_names, clippy::useless_format, clippy::too_many_arguments)]
3 use std::collections::VecDeque;
5 use std::path::{Path, PathBuf};
6 use std::process::{Command, ExitStatus};
7 use std::sync::atomic::{AtomicUsize, Ordering};
11 use color_eyre::eyre::Result;
13 use parser::{ErrorMatch, Pattern};
15 use rustc_stderr::{Level, Message};
17 use crate::parser::{Comments, Condition};
26 /// Arguments passed to the binary that is executed.
27 pub args: Vec<String>,
28 /// `None` to run on the host, otherwise a target triple
29 pub target: Option<String>,
30 /// Filters applied to stderr output before processing it
31 pub stderr_filters: Filter,
32 /// Filters applied to stdout output before processing it
33 pub stdout_filters: Filter,
34 /// The folder in which to start searching for .rs files
35 pub root_dir: PathBuf,
38 pub output_conflict_handling: OutputConflictHandling,
39 /// Only run tests with one of these strings in their path/name
40 pub path_filter: Vec<String>,
44 pub enum OutputConflictHandling {
45 /// The default: emit a diff of the expected/actual output.
47 /// Ignore mismatches in the stderr/stdout files.
49 /// Instead of erroring if the stderr/stdout differs from the expected
50 /// automatically replace it with the found output (after applying filters).
54 pub type Filter = Vec<(Regex, &'static str)>;
56 pub fn run_tests(config: Config) -> Result<()> {
57 eprintln!(" Compiler flags: {:?}", config.args);
59 // Get the triple with which to run the tests
60 let target = config.target.clone().unwrap_or_else(|| config.get_host());
62 // A channel for files to process
63 let (submit, receive) = crossbeam::channel::unbounded();
65 // Some statistics and failure reports.
66 let failures = Mutex::new(vec![]);
67 let succeeded = AtomicUsize::default();
68 let ignored = AtomicUsize::default();
69 let filtered = AtomicUsize::default();
71 crossbeam::scope(|s| -> Result<()> {
72 // Create a thread that is in charge of walking the directory and submitting jobs.
73 // It closes the channel when it is done.
75 let mut todo = VecDeque::new();
76 todo.push_back(config.root_dir.clone());
77 while let Some(path) = todo.pop_front() {
79 // Enqueue everything inside this directory.
80 // We want it sorted, to have some control over scheduling of slow tests.
82 std::fs::read_dir(path).unwrap().collect::<Result<Vec<_>, _>>().unwrap();
83 entries.sort_by_key(|e| e.file_name());
84 for entry in entries {
85 todo.push_back(entry.path());
87 } else if path.extension().map(|ext| ext == "rs").unwrap_or(false) {
88 // Forward .rs files to the test workers.
89 submit.send(path).unwrap();
92 // There will be no more jobs. This signals the workers to quit.
93 // (This also ensures `submit` is moved into this closure.)
97 let mut threads = vec![];
99 // Create N worker threads that receive files to test.
100 for _ in 0..std::thread::available_parallelism().unwrap().get() {
101 threads.push(s.spawn(|_| -> Result<()> {
102 for path in &receive {
103 if !config.path_filter.is_empty() {
104 let path_display = path.display().to_string();
105 if !config.path_filter.iter().any(|filter| path_display.contains(filter)) {
106 filtered.fetch_add(1, Ordering::Relaxed);
110 let comments = Comments::parse_file(&path)?;
111 // Ignore file if only/ignore rules do (not) apply
112 if !test_file_conditions(&comments, &target, &config) {
113 ignored.fetch_add(1, Ordering::Relaxed);
117 "ignored (in-test comment)".yellow()
121 // Run the test for all revisions
123 comments.revisions.clone().unwrap_or_else(|| vec![String::new()])
125 let (m, errors, stderr) =
126 run_test(&path, &config, &target, &revision, &comments);
128 // Using a single `eprintln!` to prevent messages from threads from getting intermingled.
129 let mut msg = format!("{} ", path.display());
130 if !revision.is_empty() {
131 write!(msg, "(revision `{revision}`) ").unwrap();
133 write!(msg, "... ").unwrap();
134 if errors.is_empty() {
135 eprintln!("{msg}{}", "ok".green());
136 succeeded.fetch_add(1, Ordering::Relaxed);
138 eprintln!("{msg}{}", "FAILED".red().bold());
139 failures.lock().unwrap().push((
152 for thread in threads {
153 thread.join().unwrap()?;
159 // Print all errors in a single thread to show reliable output
160 let failures = failures.into_inner().unwrap();
161 let succeeded = succeeded.load(Ordering::Relaxed);
162 let ignored = ignored.load(Ordering::Relaxed);
163 let filtered = filtered.load(Ordering::Relaxed);
164 if !failures.is_empty() {
165 for (path, miri, revision, errors, stderr) in &failures {
167 eprint!("{}", path.display().to_string().underline());
168 if !revision.is_empty() {
169 eprint!(" (revision `{}`)", revision);
171 eprint!(" {}", "FAILED".red());
173 eprintln!("command: {:?}", miri);
175 let mut dump_stderr = true;
176 for error in errors {
178 Error::ExitStatus(mode, exit_status) => eprintln!("{mode:?} got {exit_status}"),
179 Error::PatternNotFound { pattern, definition_line } => {
181 Pattern::SubString(s) =>
182 eprintln!("substring `{s}` {} in stderr output", "not found".red()),
184 eprintln!("`/{r}/` does {} stderr output", "not match".red()),
187 "expected because of pattern here: {}:{definition_line}",
188 path.display().to_string().bold()
191 Error::NoPatternsFound => {
192 eprintln!("{}", "no error patterns found in failure test".red());
194 Error::PatternFoundInPassTest =>
195 eprintln!("{}", "error pattern found in success test".red()),
196 Error::OutputDiffers { path, actual, expected } => {
197 if path.extension().unwrap() == "stderr" {
200 eprintln!("actual output differed from expected {}", path.display());
201 eprintln!("{}", pretty_assertions::StrComparison::new(expected, actual));
204 Error::ErrorsWithoutPattern { path: None, msgs } => {
206 "There were {} unmatched diagnostics that occurred outside the testfile and had not pattern",
209 for Message { level, message } in msgs {
210 eprintln!(" {level:?}: {message}")
213 Error::ErrorsWithoutPattern { path: Some((path, line)), msgs } => {
215 "There were {} unmatched diagnostics at {}:{line}",
219 for Message { level, message } in msgs {
220 eprintln!(" {level:?}: {message}")
226 // Unless we already dumped the stderr via an OutputDiffers diff, let's dump it here.
228 eprintln!("actual stderr:");
229 eprintln!("{}", stderr);
233 eprintln!("{}", "failures:".red().underline());
234 for (path, _miri, _revision, _errors, _stderr) in &failures {
235 eprintln!(" {}", path.display());
239 "test result: {}. {} tests failed, {} tests passed, {} ignored, {} filtered out",
241 failures.len().to_string().red().bold(),
242 succeeded.to_string().green(),
243 ignored.to_string().yellow(),
244 filtered.to_string().yellow(),
246 std::process::exit(1);
250 "test result: {}. {} tests passed, {} ignored, {} filtered out",
252 succeeded.to_string().green(),
253 ignored.to_string().yellow(),
254 filtered.to_string().yellow(),
262 /// Got an invalid exit status for the given mode.
263 ExitStatus(Mode, ExitStatus),
266 definition_line: usize,
268 /// A ui test checking for failure does not have any failure patterns
270 /// A ui test checking for success has failure patterns
271 PatternFoundInPassTest,
272 /// Stderr/Stdout differed from the `.stderr`/`.stdout` file present.
278 ErrorsWithoutPattern {
280 path: Option<(PathBuf, usize)>,
284 type Errors = Vec<Error>;
292 ) -> (Command, Errors, String) {
294 let mut miri = Command::new(&config.program);
295 miri.args(config.args.iter());
297 if !revision.is_empty() {
298 miri.arg(format!("--cfg={revision}"));
300 miri.arg("--error-format=json");
301 for arg in &comments.compile_flags {
304 for (k, v) in &comments.env_vars {
307 let output = miri.output().expect("could not execute miri");
308 let mut errors = config.mode.ok(output.status);
309 let stderr = check_test_result(
319 (miri, errors, stderr)
322 fn check_test_result(
332 // Always remove annotation comments from stderr.
333 let diagnostics = rustc_stderr::process(path, stderr);
334 let stdout = std::str::from_utf8(stdout).unwrap();
335 // Check output files (if any)
336 let revised = |extension: &str| {
337 if revision.is_empty() {
338 extension.to_string()
340 format!("{}.{}", revision, extension)
343 // Check output files against actual output
345 &diagnostics.rendered,
350 &config.stderr_filters,
360 &config.stdout_filters,
364 // Check error annotations in the source against output
366 diagnostics.messages,
367 diagnostics.messages_from_unknown_file_or_line,
377 fn check_annotations(
378 mut messages: Vec<Vec<Message>>,
379 mut messages_from_unknown_file_or_line: Vec<Message>,
386 if let Some((ref error_pattern, definition_line)) = comments.error_pattern {
387 // first check the diagnostics messages outside of our file. We check this first, so that
388 // you can mix in-file annotations with //@error-pattern annotations, even if there is overlap
390 if let Some(i) = messages_from_unknown_file_or_line
392 .position(|msg| error_pattern.matches(&msg.message))
394 messages_from_unknown_file_or_line.remove(i);
396 errors.push(Error::PatternNotFound { pattern: error_pattern.clone(), definition_line });
400 // The order on `Level` is such that `Error` is the highest level.
401 // We will ensure that *all* diagnostics of level at least `lowest_annotation_level`
403 let mut lowest_annotation_level = Level::Error;
404 for &ErrorMatch { ref pattern, revision: ref rev, definition_line, line, level } in
405 &comments.error_matches
407 if let Some(rev) = rev {
413 // If we found a diagnostic with a level annotation, make sure that all
414 // diagnostics of that level have annotations, even if we don't end up finding a matching diagnostic
416 lowest_annotation_level = std::cmp::min(lowest_annotation_level, level);
418 if let Some(msgs) = messages.get_mut(line) {
420 msgs.iter().position(|msg| pattern.matches(&msg.message) && msg.level == level);
421 if let Some(found) = found {
427 errors.push(Error::PatternNotFound { pattern: pattern.clone(), definition_line });
430 let filter = |msgs: Vec<Message>| -> Vec<_> {
434 >= comments.require_annotations_for_level.unwrap_or(lowest_annotation_level)
439 let messages_from_unknown_file_or_line = filter(messages_from_unknown_file_or_line);
440 if !messages_from_unknown_file_or_line.is_empty() {
441 errors.push(Error::ErrorsWithoutPattern {
443 msgs: messages_from_unknown_file_or_line,
447 for (line, msgs) in messages.into_iter().enumerate() {
448 let msgs = filter(msgs);
449 if !msgs.is_empty() {
451 .push(Error::ErrorsWithoutPattern { path: Some((path.to_path_buf(), line)), msgs });
455 match (config.mode, comments.error_pattern.is_some() || !comments.error_matches.is_empty()) {
456 (Mode::Pass, true) | (Mode::Panic, true) => errors.push(Error::PatternFoundInPassTest),
457 (Mode::Fail, false) => errors.push(Error::NoPatternsFound),
472 let output = normalize(path, output, filters, comments);
473 let path = output_path(path, comments, kind, target);
474 match config.output_conflict_handling {
475 OutputConflictHandling::Bless =>
476 if output.is_empty() {
477 let _ = std::fs::remove_file(path);
479 std::fs::write(path, &output).unwrap();
481 OutputConflictHandling::Error => {
482 let expected_output = std::fs::read_to_string(&path).unwrap_or_default();
483 if output != expected_output {
484 errors.push(Error::OutputDiffers {
487 expected: expected_output,
491 OutputConflictHandling::Ignore => {}
495 fn output_path(path: &Path, comments: &Comments, kind: String, target: &str) -> PathBuf {
496 if comments.stderr_per_bitwidth {
497 return path.with_extension(format!("{}bit.{kind}", get_pointer_width(target)));
499 path.with_extension(kind)
502 fn test_condition(condition: &Condition, target: &str, config: &Config) -> bool {
504 Condition::Bitwidth(bits) => get_pointer_width(target) == *bits,
505 Condition::Target(t) => target.contains(t),
506 Condition::OnHost => config.target.is_none(),
510 /// Returns whether according to the in-file conditions, this file should be run.
511 fn test_file_conditions(comments: &Comments, target: &str, config: &Config) -> bool {
512 if comments.ignore.iter().any(|c| test_condition(c, target, config)) {
515 comments.only.iter().all(|c| test_condition(c, target, config))
518 // Taken 1:1 from compiletest-rs
519 fn get_pointer_width(triple: &str) -> u8 {
520 if (triple.contains("64") && !triple.ends_with("gnux32") && !triple.ends_with("gnu_ilp32"))
521 || triple.starts_with("s390x")
524 } else if triple.starts_with("avr") {
531 fn normalize(path: &Path, text: &str, filters: &Filter, comments: &Comments) -> String {
533 let mut text = text.replace(&path.parent().unwrap().display().to_string(), "$DIR");
534 if let Some(lib_path) = option_env!("RUSTC_LIB_PATH") {
535 text = text.replace(lib_path, "RUSTLIB");
538 for (regex, replacement) in filters.iter() {
539 text = regex.replace_all(&text, *replacement).to_string();
542 for (from, to) in &comments.normalize_stderr {
543 text = from.replace_all(&text, to).to_string();
549 fn get_host(&self) -> String {
550 rustc_version::VersionMeta::for_command(std::process::Command::new(&self.program))
551 .expect("failed to parse rustc version info")
556 #[derive(Copy, Clone, Debug)]
558 // The test passes a full execution of the rustc driver
560 // The rustc driver panicked
562 // The rustc driver emitted an error
567 fn ok(self, status: ExitStatus) -> Errors {
568 match (status.code().unwrap(), self) {
569 (1, Mode::Fail) | (101, Mode::Panic) | (0, Mode::Pass) => vec![],
570 _ => vec![Error::ExitStatus(self, status)],