1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
12 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
13 use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
15 use header::TestProps;
22 use std::collections::HashSet;
24 use std::fs::{self, File};
25 use std::io::BufReader;
26 use std::io::prelude::*;
27 use std::net::TcpStream;
28 use std::path::{Path, PathBuf, Component};
29 use std::process::{Command, Output, ExitStatus};
31 pub fn run(config: Config, testpaths: &TestPaths) {
32 match &*config.target {
34 "arm-linux-androideabi" | "aarch64-linux-android" => {
35 if !config.adb_device_status {
36 panic!("android device not available");
44 // We're going to be dumping a lot of info. Start on a new line.
47 debug!("running {:?}", testpaths.file.display());
48 let props = header::load_props(&testpaths.file);
49 debug!("loaded props");
51 CompileFail => run_cfail_test(&config, &props, &testpaths),
52 ParseFail => run_cfail_test(&config, &props, &testpaths),
53 RunFail => run_rfail_test(&config, &props, &testpaths),
54 RunPass => run_rpass_test(&config, &props, &testpaths),
55 RunPassValgrind => run_valgrind_test(&config, &props, &testpaths),
56 Pretty => run_pretty_test(&config, &props, &testpaths),
57 DebugInfoGdb => run_debuginfo_gdb_test(&config, &props, &testpaths),
58 DebugInfoLldb => run_debuginfo_lldb_test(&config, &props, &testpaths),
59 Codegen => run_codegen_test(&config, &props, &testpaths),
60 Rustdoc => run_rustdoc_test(&config, &props, &testpaths),
61 CodegenUnits => run_codegen_units_test(&config, &props, &testpaths),
65 fn get_output(props: &TestProps, proc_res: &ProcRes) -> String {
66 if props.check_stdout {
67 format!("{}{}", proc_res.stdout, proc_res.stderr)
69 proc_res.stderr.clone()
73 fn run_cfail_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
74 let proc_res = compile_test(config, props, testpaths);
76 if proc_res.status.success() {
77 fatal_proc_rec(&format!("{} test compiled successfully!", config.mode)[..],
81 check_correct_failure_status(&proc_res);
83 if proc_res.status.success() {
84 fatal("process did not return an error status");
87 let output_to_check = get_output(props, &proc_res);
88 let expected_errors = errors::load_errors(&testpaths.file);
89 if !expected_errors.is_empty() {
90 if !props.error_patterns.is_empty() {
91 fatal("both error pattern and expected errors specified");
93 check_expected_errors(expected_errors, testpaths, &proc_res);
95 check_error_patterns(props, testpaths, &output_to_check, &proc_res);
97 check_no_compiler_crash(&proc_res);
98 check_forbid_output(props, &output_to_check, &proc_res);
101 fn run_rfail_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
102 let proc_res = compile_test(config, props, testpaths);
104 if !proc_res.status.success() {
105 fatal_proc_rec("compilation failed!", &proc_res);
108 let proc_res = exec_compiled_test(config, props, testpaths);
110 // The value our Makefile configures valgrind to return on failure
111 const VALGRIND_ERR: i32 = 100;
112 if proc_res.status.code() == Some(VALGRIND_ERR) {
113 fatal_proc_rec("run-fail test isn't valgrind-clean!", &proc_res);
116 let output_to_check = get_output(props, &proc_res);
117 check_correct_failure_status(&proc_res);
118 check_error_patterns(props, testpaths, &output_to_check, &proc_res);
121 fn check_correct_failure_status(proc_res: &ProcRes) {
122 // The value the rust runtime returns on failure
123 const RUST_ERR: i32 = 101;
124 if proc_res.status.code() != Some(RUST_ERR) {
126 &format!("failure produced the wrong error: {}",
132 fn run_rpass_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
133 let proc_res = compile_test(config, props, testpaths);
135 if !proc_res.status.success() {
136 fatal_proc_rec("compilation failed!", &proc_res);
139 let proc_res = exec_compiled_test(config, props, testpaths);
141 if !proc_res.status.success() {
142 fatal_proc_rec("test run failed!", &proc_res);
146 fn run_valgrind_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
147 if config.valgrind_path.is_none() {
148 assert!(!config.force_valgrind);
149 return run_rpass_test(config, props, testpaths);
152 let mut proc_res = compile_test(config, props, testpaths);
154 if !proc_res.status.success() {
155 fatal_proc_rec("compilation failed!", &proc_res);
158 let mut new_config = config.clone();
159 new_config.runtool = new_config.valgrind_path.clone();
160 proc_res = exec_compiled_test(&new_config, props, testpaths);
162 if !proc_res.status.success() {
163 fatal_proc_rec("test run failed!", &proc_res);
167 fn run_pretty_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
168 if props.pp_exact.is_some() {
169 logv(config, "testing for exact pretty-printing".to_owned());
171 logv(config, "testing for converging pretty-printing".to_owned());
175 match props.pp_exact { Some(_) => 1, None => 2 };
177 let mut src = String::new();
178 File::open(&testpaths.file).unwrap().read_to_string(&mut src).unwrap();
179 let mut srcs = vec!(src);
182 while round < rounds {
183 logv(config, format!("pretty-printing round {}", round));
184 let proc_res = print_source(config,
187 srcs[round].to_owned(),
190 if !proc_res.status.success() {
191 fatal_proc_rec(&format!("pretty-printing failed in round {}", round),
195 let ProcRes{ stdout, .. } = proc_res;
200 let mut expected = match props.pp_exact {
202 let filepath = testpaths.file.parent().unwrap().join(file);
203 let mut s = String::new();
204 File::open(&filepath).unwrap().read_to_string(&mut s).unwrap();
207 None => { srcs[srcs.len() - 2].clone() }
209 let mut actual = srcs[srcs.len() - 1].clone();
211 if props.pp_exact.is_some() {
212 // Now we have to care about line endings
213 let cr = "\r".to_owned();
214 actual = actual.replace(&cr, "").to_owned();
215 expected = expected.replace(&cr, "").to_owned();
218 compare_source(&expected, &actual);
220 // If we're only making sure that the output matches then just stop here
221 if props.pretty_compare_only { return; }
223 // Finally, let's make sure it actually appears to remain valid code
224 let proc_res = typecheck_source(config, props, testpaths, actual);
226 if !proc_res.status.success() {
227 fatal_proc_rec("pretty-printed source does not typecheck", &proc_res);
229 if !props.pretty_expanded { return }
231 // additionally, run `--pretty expanded` and try to build it.
232 let proc_res = print_source(config, props, testpaths, srcs[round].clone(), "expanded");
233 if !proc_res.status.success() {
234 fatal_proc_rec("pretty-printing (expanded) failed", &proc_res);
237 let ProcRes{ stdout: expanded_src, .. } = proc_res;
238 let proc_res = typecheck_source(config, props, testpaths, expanded_src);
239 if !proc_res.status.success() {
240 fatal_proc_rec("pretty-printed source (expanded) does not typecheck",
246 fn print_source(config: &Config,
248 testpaths: &TestPaths,
250 pretty_type: &str) -> ProcRes {
251 let aux_dir = aux_output_dir_name(config, testpaths);
252 compose_and_run(config,
257 pretty_type.to_owned()),
258 props.exec_env.clone(),
259 &config.compile_lib_path,
260 Some(aux_dir.to_str().unwrap()),
264 fn make_pp_args(config: &Config,
266 testpaths: &TestPaths,
267 pretty_type: String) -> ProcArgs {
268 let aux_dir = aux_output_dir_name(config, testpaths);
269 // FIXME (#9639): This needs to handle non-utf8 paths
270 let mut args = vec!("-".to_owned(),
271 "-Zunstable-options".to_owned(),
272 "--unpretty".to_owned(),
274 format!("--target={}", config.target),
276 aux_dir.to_str().unwrap().to_owned());
277 args.extend(split_maybe_args(&config.target_rustcflags));
278 args.extend(props.compile_flags.iter().cloned());
280 prog: config.rustc_path.to_str().unwrap().to_owned(),
285 fn compare_source(expected: &str, actual: &str) {
286 if expected != actual {
287 error("pretty-printed source does not match expected source");
290 ------------------------------------------\n\
292 ------------------------------------------\n\
294 ------------------------------------------\n\
296 ------------------------------------------\n\
303 fn typecheck_source(config: &Config, props: &TestProps,
304 testpaths: &TestPaths, src: String) -> ProcRes {
305 let args = make_typecheck_args(config, props, testpaths);
306 compose_and_run_compiler(config, props, testpaths, args, Some(src))
309 fn make_typecheck_args(config: &Config, props: &TestProps, testpaths: &TestPaths) -> ProcArgs {
310 let aux_dir = aux_output_dir_name(config, testpaths);
311 let target = if props.force_host {
316 // FIXME (#9639): This needs to handle non-utf8 paths
317 let mut args = vec!("-".to_owned(),
318 "-Zno-trans".to_owned(),
319 format!("--target={}", target),
321 config.build_base.to_str().unwrap().to_owned(),
323 aux_dir.to_str().unwrap().to_owned());
324 args.extend(split_maybe_args(&config.target_rustcflags));
325 args.extend(props.compile_flags.iter().cloned());
326 // FIXME (#9639): This needs to handle non-utf8 paths
328 prog: config.rustc_path.to_str().unwrap().to_owned(),
334 fn run_debuginfo_gdb_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
335 let mut config = Config {
336 target_rustcflags: cleanup_debug_info_options(&config.target_rustcflags),
337 host_rustcflags: cleanup_debug_info_options(&config.host_rustcflags),
341 let config = &mut config;
342 let DebuggerCommands {
346 } = parse_debugger_commands(testpaths, "gdb");
347 let mut cmds = commands.join("\n");
349 // compile test file (it should have 'compile-flags:-g' in the header)
350 let compiler_run_result = compile_test(config, props, testpaths);
351 if !compiler_run_result.status.success() {
352 fatal_proc_rec("compilation failed!", &compiler_run_result);
355 let exe_file = make_exe_name(config, testpaths);
357 let debugger_run_result;
358 match &*config.target {
359 "arm-linux-androideabi" | "aarch64-linux-android" => {
361 cmds = cmds.replace("run", "continue");
363 // write debugger script
364 let mut script_str = String::with_capacity(2048);
365 script_str.push_str(&format!("set charset {}\n", charset()));
366 script_str.push_str(&format!("file {}\n", exe_file.to_str().unwrap()));
367 script_str.push_str("target remote :5039\n");
368 script_str.push_str(&format!("set solib-search-path \
369 ./{}/stage2/lib/rustlib/{}/lib/\n",
370 config.host, config.target));
371 for line in &breakpoint_lines {
372 script_str.push_str(&format!("break {:?}:{}\n",
379 script_str.push_str(&cmds);
380 script_str.push_str("\nquit\n");
382 debug!("script_str = {}", script_str);
383 dump_output_file(config,
394 exe_file.to_str().unwrap().to_owned(),
395 config.adb_test_dir.clone()
397 vec!(("".to_owned(), "".to_owned())),
399 .expect(&format!("failed to exec `{:?}`", config.adb_path));
405 "forward".to_owned(),
406 "tcp:5039".to_owned(),
407 "tcp:5039".to_owned()
409 vec!(("".to_owned(), "".to_owned())),
411 .expect(&format!("failed to exec `{:?}`", config.adb_path));
413 let adb_arg = format!("export LD_LIBRARY_PATH={}; \
414 gdbserver{} :5039 {}/{}",
415 config.adb_test_dir.clone(),
416 if config.target.contains("aarch64")
418 config.adb_test_dir.clone(),
419 exe_file.file_name().unwrap().to_str()
422 let mut process = procsrv::run_background("",
433 .expect(&format!("failed to exec `{:?}`", config.adb_path));
435 //waiting 1 second for gdbserver start
436 ::std::thread::sleep(::std::time::Duration::new(1,0));
437 if TcpStream::connect("127.0.0.1:5039").is_ok() {
442 let tool_path = match config.android_cross_path.to_str() {
443 Some(x) => x.to_owned(),
444 None => fatal("cannot find android cross path")
447 let debugger_script = make_out_name(config, testpaths, "debugger.script");
448 // FIXME (#9639): This needs to handle non-utf8 paths
450 vec!("-quiet".to_owned(),
453 format!("-command={}", debugger_script.to_str().unwrap()));
455 let mut gdb_path = tool_path;
456 gdb_path.push_str(&format!("/bin/{}-gdb", config.target));
457 let procsrv::Result {
465 vec!(("".to_owned(), "".to_owned())),
467 .expect(&format!("failed to exec `{:?}`", gdb_path));
469 let cmdline = make_cmdline("",
470 &format!("{}-gdb", config.target),
472 logv(config, format!("executing {}", cmdline));
476 debugger_run_result = ProcRes {
477 status: Status::Normal(status),
482 if process.kill().is_err() {
483 println!("Adb process is already finished.");
488 let rust_src_root = find_rust_src_root(config)
489 .expect("Could not find Rust source root");
490 let rust_pp_module_rel_path = Path::new("./src/etc");
491 let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
495 // write debugger script
496 let mut script_str = String::with_capacity(2048);
497 script_str.push_str(&format!("set charset {}\n", charset()));
498 script_str.push_str("show version\n");
500 match config.gdb_version {
501 Some(ref version) => {
502 println!("NOTE: compiletest thinks it is using GDB version {}",
505 if header::gdb_version_to_int(version) >
506 header::gdb_version_to_int("7.4") {
507 // Add the directory containing the pretty printers to
508 // GDB's script auto loading safe path
510 &format!("add-auto-load-safe-path {}\n",
511 rust_pp_module_abs_path.replace(r"\", r"\\"))
516 println!("NOTE: compiletest does not know which version of \
521 // The following line actually doesn't have to do anything with
522 // pretty printing, it just tells GDB to print values on one line:
523 script_str.push_str("set print pretty off\n");
525 // Add the pretty printer directory to GDB's source-file search path
526 script_str.push_str(&format!("directory {}\n",
527 rust_pp_module_abs_path));
529 // Load the target executable
530 script_str.push_str(&format!("file {}\n",
531 exe_file.to_str().unwrap()
532 .replace(r"\", r"\\")));
534 // Add line breakpoints
535 for line in &breakpoint_lines {
536 script_str.push_str(&format!("break '{}':{}\n",
537 testpaths.file.file_name().unwrap()
542 script_str.push_str(&cmds);
543 script_str.push_str("\nquit\n");
545 debug!("script_str = {}", script_str);
546 dump_output_file(config,
551 // run debugger script with gdb
552 fn debugger() -> &'static str {
553 if cfg!(windows) {"gdb.exe"} else {"gdb"}
556 let debugger_script = make_out_name(config, testpaths, "debugger.script");
558 // FIXME (#9639): This needs to handle non-utf8 paths
560 vec!("-quiet".to_owned(),
563 format!("-command={}", debugger_script.to_str().unwrap()));
565 let proc_args = ProcArgs {
566 prog: debugger().to_owned(),
570 let environment = vec![("PYTHONPATH".to_owned(), rust_pp_module_abs_path)];
572 debugger_run_result = compose_and_run(config,
576 &config.run_lib_path,
582 if !debugger_run_result.status.success() {
583 fatal("gdb failed to execute");
586 check_debugger_output(&debugger_run_result, &check_lines);
589 fn find_rust_src_root(config: &Config) -> Option<PathBuf> {
590 let mut path = config.src_base.clone();
591 let path_postfix = Path::new("src/etc/lldb_batchmode.py");
594 if path.join(&path_postfix).is_file() {
602 fn run_debuginfo_lldb_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
603 if config.lldb_python_dir.is_none() {
604 fatal("Can't run LLDB test because LLDB's python path is not set.");
607 let mut config = Config {
608 target_rustcflags: cleanup_debug_info_options(&config.target_rustcflags),
609 host_rustcflags: cleanup_debug_info_options(&config.host_rustcflags),
613 let config = &mut config;
615 // compile test file (it should have 'compile-flags:-g' in the header)
616 let compile_result = compile_test(config, props, testpaths);
617 if !compile_result.status.success() {
618 fatal_proc_rec("compilation failed!", &compile_result);
621 let exe_file = make_exe_name(config, testpaths);
623 match config.lldb_version {
624 Some(ref version) => {
625 println!("NOTE: compiletest thinks it is using LLDB version {}",
629 println!("NOTE: compiletest does not know which version of \
634 // Parse debugger commands etc from test files
635 let DebuggerCommands {
640 } = parse_debugger_commands(testpaths, "lldb");
642 // Write debugger script:
643 // We don't want to hang when calling `quit` while the process is still running
644 let mut script_str = String::from("settings set auto-confirm true\n");
646 // Make LLDB emit its version, so we have it documented in the test output
647 script_str.push_str("version\n");
649 // Switch LLDB into "Rust mode"
650 let rust_src_root = find_rust_src_root(config)
651 .expect("Could not find Rust source root");
652 let rust_pp_module_rel_path = Path::new("./src/etc/lldb_rust_formatters.py");
653 let rust_pp_module_abs_path = rust_src_root.join(rust_pp_module_rel_path)
658 script_str.push_str(&format!("command script import {}\n",
659 &rust_pp_module_abs_path[..])[..]);
660 script_str.push_str("type summary add --no-value ");
661 script_str.push_str("--python-function lldb_rust_formatters.print_val ");
662 script_str.push_str("-x \".*\" --category Rust\n");
663 script_str.push_str("type category enable Rust\n");
665 // Set breakpoints on every line that contains the string "#break"
666 for line in &breakpoint_lines {
667 script_str.push_str(&format!("breakpoint set --line {}\n", line));
670 // Append the other commands
671 for line in &commands {
672 script_str.push_str(line);
673 script_str.push_str("\n");
676 // Finally, quit the debugger
677 script_str.push_str("\nquit\n");
679 // Write the script into a file
680 debug!("script_str = {}", script_str);
681 dump_output_file(config,
685 let debugger_script = make_out_name(config, testpaths, "debugger.script");
687 // Let LLDB execute the script via lldb_batchmode.py
688 let debugger_run_result = run_lldb(config,
694 if !debugger_run_result.status.success() {
695 fatal_proc_rec("Error while running LLDB", &debugger_run_result);
698 check_debugger_output(&debugger_run_result, &check_lines);
700 fn run_lldb(config: &Config,
701 testpaths: &TestPaths,
702 test_executable: &Path,
703 debugger_script: &Path,
704 rust_src_root: &Path)
706 // Prepare the lldb_batchmode which executes the debugger script
707 let lldb_script_path = rust_src_root.join("src/etc/lldb_batchmode.py");
710 Command::new(&config.python)
711 .arg(&lldb_script_path)
712 .arg(test_executable)
713 .arg(debugger_script)
715 config.lldb_python_dir.as_ref().unwrap()))
719 fn cmd2procres(config: &Config, testpaths: &TestPaths, cmd: &mut Command)
721 let (status, out, err) = match cmd.output() {
722 Ok(Output { status, stdout, stderr }) => {
724 String::from_utf8(stdout).unwrap(),
725 String::from_utf8(stderr).unwrap())
728 fatal(&format!("Failed to setup Python process for \
729 LLDB script: {}", e))
733 dump_output(config, testpaths, &out, &err);
735 status: Status::Normal(status),
738 cmdline: format!("{:?}", cmd)
742 struct DebuggerCommands {
743 commands: Vec<String>,
744 check_lines: Vec<String>,
745 breakpoint_lines: Vec<usize>,
748 fn parse_debugger_commands(testpaths: &TestPaths, debugger_prefix: &str)
749 -> DebuggerCommands {
750 let command_directive = format!("{}-command", debugger_prefix);
751 let check_directive = format!("{}-check", debugger_prefix);
753 let mut breakpoint_lines = vec!();
754 let mut commands = vec!();
755 let mut check_lines = vec!();
757 let reader = BufReader::new(File::open(&testpaths.file).unwrap());
758 for line in reader.lines() {
761 if line.contains("#break") {
762 breakpoint_lines.push(counter);
765 header::parse_name_value_directive(
767 &command_directive).map(|cmd| {
771 header::parse_name_value_directive(
773 &check_directive).map(|cmd| {
774 check_lines.push(cmd)
778 fatal(&format!("Error while parsing debugger commands: {}", e))
786 check_lines: check_lines,
787 breakpoint_lines: breakpoint_lines,
791 fn cleanup_debug_info_options(options: &Option<String>) -> Option<String> {
792 if options.is_none() {
796 // Remove options that are either unwanted (-O) or may lead to duplicates due to RUSTFLAGS.
797 let options_to_remove = [
800 "--debuginfo".to_owned()
803 split_maybe_args(options).into_iter()
804 .filter(|x| !options_to_remove.contains(x))
805 .collect::<Vec<String>>()
810 fn check_debugger_output(debugger_run_result: &ProcRes, check_lines: &[String]) {
811 let num_check_lines = check_lines.len();
812 if num_check_lines > 0 {
813 // Allow check lines to leave parts unspecified (e.g., uninitialized
814 // bits in the wrong case of an enum) with the notation "[...]".
815 let check_fragments: Vec<Vec<String>> =
816 check_lines.iter().map(|s| {
823 // check if each line in props.check_lines appears in the
826 for line in debugger_run_result.stdout.lines() {
827 let mut rest = line.trim();
828 let mut first = true;
829 let mut failed = false;
830 for frag in &check_fragments[i] {
831 let found = if first {
832 if rest.starts_with(frag) {
846 rest = &rest[(i + frag.len())..];
851 if !failed && rest.is_empty() {
854 if i == num_check_lines {
859 if i != num_check_lines {
860 fatal_proc_rec(&format!("line not found in debugger output: {}",
861 check_lines.get(i).unwrap()),
862 debugger_run_result);
867 fn check_error_patterns(props: &TestProps,
868 testpaths: &TestPaths,
869 output_to_check: &str,
870 proc_res: &ProcRes) {
871 if props.error_patterns.is_empty() {
872 fatal(&format!("no error pattern specified in {:?}",
873 testpaths.file.display()));
875 let mut next_err_idx = 0;
876 let mut next_err_pat = &props.error_patterns[next_err_idx];
877 let mut done = false;
878 for line in output_to_check.lines() {
879 if line.contains(next_err_pat) {
880 debug!("found error pattern {}", next_err_pat);
882 if next_err_idx == props.error_patterns.len() {
883 debug!("found all error patterns");
887 next_err_pat = &props.error_patterns[next_err_idx];
892 let missing_patterns = &props.error_patterns[next_err_idx..];
893 if missing_patterns.len() == 1 {
894 fatal_proc_rec(&format!("error pattern '{}' not found!", missing_patterns[0]),
897 for pattern in missing_patterns {
898 error(&format!("error pattern '{}' not found!", *pattern));
900 fatal_proc_rec("multiple error patterns not found", proc_res);
904 fn check_no_compiler_crash(proc_res: &ProcRes) {
905 for line in proc_res.stderr.lines() {
906 if line.starts_with("error: internal compiler error:") {
907 fatal_proc_rec("compiler encountered internal error",
913 fn check_forbid_output(props: &TestProps,
914 output_to_check: &str,
915 proc_res: &ProcRes) {
916 for pat in &props.forbid_output {
917 if output_to_check.contains(pat) {
918 fatal_proc_rec("forbidden pattern found in compiler output", proc_res);
923 fn check_expected_errors(expected_errors: Vec<errors::ExpectedError>,
924 testpaths: &TestPaths,
925 proc_res: &ProcRes) {
927 // true if we found the error in question
928 let mut found_flags = vec![false; expected_errors.len()];
930 if proc_res.status.success() {
931 fatal("process did not return an error status");
934 let prefixes = expected_errors.iter().map(|ee| {
935 let expected = format!("{}:{}:", testpaths.file.display(), ee.line);
936 // On windows just translate all '\' path separators to '/'
937 expected.replace(r"\", "/")
938 }).collect::<Vec<String>>();
940 let (expect_help, expect_note) =
941 expected_errors.iter()
942 .fold((false, false),
943 |(acc_help, acc_note), ee|
944 (acc_help || ee.kind == "help:", acc_note ||
945 ee.kind == "note:"));
947 fn prefix_matches(line: &str, prefix: &str) -> bool {
948 use std::ascii::AsciiExt;
949 // On windows just translate all '\' path separators to '/'
950 let line = line.replace(r"\", "/");
952 line.to_ascii_lowercase().starts_with(&prefix.to_ascii_lowercase())
954 line.starts_with(prefix)
958 // A multi-line error will have followup lines which start with a space
960 fn continuation( line: &str) -> bool {
961 line.starts_with(" ") || line.starts_with("(")
964 // Scan and extract our error/warning messages,
966 // filename:line1:col1: line2:col2: *error:* msg
967 // filename:line1:col1: line2:col2: *warning:* msg
968 // where line1:col1: is the starting point, line2:col2:
969 // is the ending point, and * represents ANSI color codes.
971 // This pattern is ambiguous on windows, because filename may contain
972 // a colon, so any path prefix must be detected and removed first.
973 for line in proc_res.stderr.lines() {
974 let mut was_expected = false;
976 for (i, ee) in expected_errors.iter().enumerate() {
978 debug!("prefix={} ee.kind={} ee.msg={} line={}",
983 // Suggestions have no line number in their output, so take on the line number of
984 // the previous expected error
985 if ee.kind == "suggestion" {
986 assert!(expected_errors[prev].kind == "help",
987 "SUGGESTIONs must be preceded by a HELP");
988 if line.contains(&ee.msg) {
989 found_flags[i] = true;
994 if (prefix_matches(line, &prefixes[i]) || continuation(line)) &&
995 line.contains(&ee.kind) &&
996 line.contains(&ee.msg) {
997 found_flags[i] = true;
1005 // ignore this msg which gets printed at the end
1006 if line.contains("aborting due to") {
1007 was_expected = true;
1010 if !was_expected && is_unexpected_compiler_message(line, expect_help, expect_note) {
1011 fatal_proc_rec(&format!("unexpected compiler message: '{}'",
1017 for (i, &flag) in found_flags.iter().enumerate() {
1019 let ee = &expected_errors[i];
1020 fatal_proc_rec(&format!("expected {} on line {} not found: {}",
1021 ee.kind, ee.line, ee.msg),
1027 fn is_unexpected_compiler_message(line: &str, expect_help: bool, expect_note: bool) -> bool {
1028 let mut c = Path::new(line).components();
1029 let line = match c.next() {
1030 Some(Component::Prefix(_)) => c.as_path().to_str().unwrap(),
1035 return scan_until_char(line, ':', &mut i) &&
1036 scan_char(line, ':', &mut i) &&
1037 scan_integer(line, &mut i) &&
1038 scan_char(line, ':', &mut i) &&
1039 scan_integer(line, &mut i) &&
1040 scan_char(line, ':', &mut i) &&
1041 scan_char(line, ' ', &mut i) &&
1042 scan_integer(line, &mut i) &&
1043 scan_char(line, ':', &mut i) &&
1044 scan_integer(line, &mut i) &&
1045 scan_char(line, ' ', &mut i) &&
1046 (scan_string(line, "error", &mut i) ||
1047 scan_string(line, "warning", &mut i) ||
1048 (expect_help && scan_string(line, "help", &mut i)) ||
1049 (expect_note && scan_string(line, "note", &mut i))
1053 fn scan_until_char(haystack: &str, needle: char, idx: &mut usize) -> bool {
1054 if *idx >= haystack.len() {
1057 let opt = haystack[(*idx)..].find(needle);
1061 *idx = opt.unwrap();
1065 fn scan_char(haystack: &str, needle: char, idx: &mut usize) -> bool {
1066 if *idx >= haystack.len() {
1069 let ch = haystack.char_at(*idx);
1073 *idx += ch.len_utf8();
1077 fn scan_integer(haystack: &str, idx: &mut usize) -> bool {
1079 while i < haystack.len() {
1080 let ch = haystack.char_at(i);
1081 if ch < '0' || '9' < ch {
1093 fn scan_string(haystack: &str, needle: &str, idx: &mut usize) -> bool {
1094 let mut haystack_i = *idx;
1095 let mut needle_i = 0;
1096 while needle_i < needle.len() {
1097 if haystack_i >= haystack.len() {
1100 let ch = haystack.char_at(haystack_i);
1101 haystack_i += ch.len_utf8();
1102 if !scan_char(needle, ch, &mut needle_i) {
1128 fn code(&self) -> Option<i32> {
1130 Status::Parsed(i) => Some(i),
1131 Status::Normal(ref e) => e.code(),
1135 fn success(&self) -> bool {
1137 Status::Parsed(i) => i == 0,
1138 Status::Normal(ref e) => e.success(),
1143 impl fmt::Display for Status {
1144 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1146 Status::Parsed(i) => write!(f, "exit code: {}", i),
1147 Status::Normal(ref e) => e.fmt(f),
1152 fn compile_test(config: &Config, props: &TestProps,
1153 testpaths: &TestPaths) -> ProcRes {
1154 let aux_dir = aux_output_dir_name(config, testpaths);
1155 // FIXME (#9639): This needs to handle non-utf8 paths
1156 let link_args = vec!("-L".to_owned(),
1157 aux_dir.to_str().unwrap().to_owned());
1158 let args = make_compile_args(config,
1161 |a, b| TargetLocation::ThisFile(make_exe_name(a, b)), testpaths);
1162 compose_and_run_compiler(config, props, testpaths, args, None)
1165 fn document(config: &Config,
1167 testpaths: &TestPaths,
1170 if props.build_aux_docs {
1171 for rel_ab in &props.aux_builds {
1172 let aux_testpaths = compute_aux_test_paths(config, testpaths, rel_ab);
1173 let aux_props = header::load_props(&aux_testpaths.file);
1174 let auxres = document(config, &aux_props, &aux_testpaths, out_dir);
1175 if !auxres.status.success() {
1181 let aux_dir = aux_output_dir_name(config, testpaths);
1182 let mut args = vec!["-L".to_owned(),
1183 aux_dir.to_str().unwrap().to_owned(),
1185 out_dir.to_str().unwrap().to_owned(),
1186 testpaths.file.to_str().unwrap().to_owned()];
1187 args.extend(props.compile_flags.iter().cloned());
1188 let args = ProcArgs {
1189 prog: config.rustdoc_path.to_str().unwrap().to_owned(),
1192 compose_and_run_compiler(config, props, testpaths, args, None)
1195 fn exec_compiled_test(config: &Config, props: &TestProps,
1196 testpaths: &TestPaths) -> ProcRes {
1198 let env = props.exec_env.clone();
1200 match &*config.target {
1202 "arm-linux-androideabi" | "aarch64-linux-android" => {
1203 _arm_exec_compiled_test(config, props, testpaths, env)
1207 let aux_dir = aux_output_dir_name(config, testpaths);
1208 compose_and_run(config,
1210 make_run_args(config, props, testpaths),
1212 &config.run_lib_path,
1213 Some(aux_dir.to_str().unwrap()),
1219 fn compute_aux_test_paths(config: &Config,
1220 testpaths: &TestPaths,
1224 let abs_ab = config.aux_base.join(rel_ab);
1227 base: testpaths.base.clone(),
1228 relative_dir: Path::new(rel_ab).parent()
1229 .map(|p| p.to_path_buf())
1230 .unwrap_or_else(|| PathBuf::new())
1234 fn compose_and_run_compiler(config: &Config, props: &TestProps,
1235 testpaths: &TestPaths, args: ProcArgs,
1236 input: Option<String>) -> ProcRes {
1237 if !props.aux_builds.is_empty() {
1238 ensure_dir(&aux_output_dir_name(config, testpaths));
1241 let aux_dir = aux_output_dir_name(config, testpaths);
1242 // FIXME (#9639): This needs to handle non-utf8 paths
1243 let extra_link_args = vec!["-L".to_owned(),
1244 aux_dir.to_str().unwrap().to_owned()];
1246 for rel_ab in &props.aux_builds {
1247 let aux_testpaths = compute_aux_test_paths(config, testpaths, rel_ab);
1248 let aux_props = header::load_props(&aux_testpaths.file);
1249 let mut crate_type = if aux_props.no_prefer_dynamic {
1252 // We primarily compile all auxiliary libraries as dynamic libraries
1253 // to avoid code size bloat and large binaries as much as possible
1254 // for the test suite (otherwise including libstd statically in all
1255 // executables takes up quite a bit of space).
1257 // For targets like MUSL or Emscripten, however, there is no support for
1258 // dynamic libraries so we just go back to building a normal library. Note,
1259 // however, that for MUSL if the library is built with `force_host` then
1260 // it's ok to be a dylib as the host should always support dylibs.
1261 if (config.target.contains("musl") && !aux_props.force_host) ||
1262 config.target.contains("emscripten")
1264 vec!("--crate-type=lib".to_owned())
1266 vec!("--crate-type=dylib".to_owned())
1269 crate_type.extend(extra_link_args.clone());
1271 make_compile_args(config,
1275 let f = make_lib_name(a, &b.file, testpaths);
1276 let parent = f.parent().unwrap();
1277 TargetLocation::ThisDirectory(parent.to_path_buf())
1280 let auxres = compose_and_run(config,
1284 &config.compile_lib_path,
1285 Some(aux_dir.to_str().unwrap()),
1287 if !auxres.status.success() {
1289 &format!("auxiliary build of {:?} failed to compile: ",
1290 aux_testpaths.file.display()),
1294 match &*config.target {
1295 "arm-linux-androideabi" | "aarch64-linux-android" => {
1296 _arm_push_aux_shared_library(config, testpaths);
1302 compose_and_run(config,
1306 &config.compile_lib_path,
1307 Some(aux_dir.to_str().unwrap()),
1311 fn ensure_dir(path: &Path) {
1312 if path.is_dir() { return; }
1313 fs::create_dir_all(path).unwrap();
1316 fn compose_and_run(config: &Config,
1317 testpaths: &TestPaths,
1318 ProcArgs{ args, prog }: ProcArgs,
1319 procenv: Vec<(String, String)> ,
1321 aux_path: Option<&str>,
1322 input: Option<String>) -> ProcRes {
1323 return program_output(config, testpaths, lib_path,
1324 prog, aux_path, args, procenv, input);
1327 enum TargetLocation {
1329 ThisDirectory(PathBuf),
1332 fn make_compile_args<F>(config: &Config,
1334 extras: Vec<String> ,
1336 testpaths: &TestPaths)
1338 F: FnOnce(&Config, &TestPaths) -> TargetLocation,
1340 let xform_file = xform(config, testpaths);
1341 let target = if props.force_host {
1346 // FIXME (#9639): This needs to handle non-utf8 paths
1347 let mut args = vec!(testpaths.file.to_str().unwrap().to_owned(),
1349 config.build_base.to_str().unwrap().to_owned(),
1350 format!("--target={}", target));
1351 args.extend_from_slice(&extras);
1352 if !props.no_prefer_dynamic {
1353 args.push("-C".to_owned());
1354 args.push("prefer-dynamic".to_owned());
1356 let path = match xform_file {
1357 TargetLocation::ThisFile(path) => {
1358 args.push("-o".to_owned());
1361 TargetLocation::ThisDirectory(path) => {
1362 args.push("--out-dir".to_owned());
1366 args.push(path.to_str().unwrap().to_owned());
1367 if props.force_host {
1368 args.extend(split_maybe_args(&config.host_rustcflags));
1370 args.extend(split_maybe_args(&config.target_rustcflags));
1372 args.extend(props.compile_flags.iter().cloned());
1374 prog: config.rustc_path.to_str().unwrap().to_owned(),
1379 fn make_lib_name(config: &Config, auxfile: &Path, testpaths: &TestPaths) -> PathBuf {
1380 // what we return here is not particularly important, as it
1381 // happens; rustc ignores everything except for the directory.
1382 let auxname = output_testname(auxfile);
1383 aux_output_dir_name(config, testpaths).join(&auxname)
1386 fn make_exe_name(config: &Config, testpaths: &TestPaths) -> PathBuf {
1387 let mut f = output_base_name(config, testpaths);
1388 // FIXME: This is using the host architecture exe suffix, not target!
1389 if config.target == "asmjs-unknown-emscripten" {
1390 let mut fname = f.file_name().unwrap().to_os_string();
1392 f.set_file_name(&fname);
1393 } else if !env::consts::EXE_SUFFIX.is_empty() {
1394 let mut fname = f.file_name().unwrap().to_os_string();
1395 fname.push(env::consts::EXE_SUFFIX);
1396 f.set_file_name(&fname);
1401 fn make_run_args(config: &Config, props: &TestProps, testpaths: &TestPaths)
1403 // If we've got another tool to run under (valgrind),
1404 // then split apart its command
1405 let mut args = split_maybe_args(&config.runtool);
1407 // If this is emscripten, then run tests under nodejs
1408 if config.target == "asmjs-unknown-emscripten" {
1409 args.push("nodejs".to_owned());
1412 let exe_file = make_exe_name(config, testpaths);
1414 // FIXME (#9639): This needs to handle non-utf8 paths
1415 args.push(exe_file.to_str().unwrap().to_owned());
1417 // Add the arguments in the run_flags directive
1418 args.extend(split_maybe_args(&props.run_flags));
1420 let prog = args.remove(0);
1427 fn split_maybe_args(argstr: &Option<String>) -> Vec<String> {
1433 if s.chars().all(|c| c.is_whitespace()) {
1444 fn program_output(config: &Config, testpaths: &TestPaths, lib_path: &str, prog: String,
1445 aux_path: Option<&str>, args: Vec<String>,
1446 env: Vec<(String, String)>,
1447 input: Option<String>) -> ProcRes {
1450 let cmdline = make_cmdline(lib_path,
1453 logv(config, format!("executing {}", cmdline));
1456 let procsrv::Result {
1460 } = procsrv::run(lib_path,
1465 input).expect(&format!("failed to exec `{}`", prog));
1466 dump_output(config, testpaths, &out, &err);
1468 status: Status::Normal(status),
1475 fn make_cmdline(libpath: &str, prog: &str, args: &[String]) -> String {
1478 // Linux and mac don't require adjusting the library search path
1480 format!("{} {}", prog, args.join(" "))
1482 // Build the LD_LIBRARY_PATH variable as it would be seen on the command line
1483 // for diagnostic purposes
1484 fn lib_path_cmd_prefix(path: &str) -> String {
1485 format!("{}=\"{}\"", util::lib_path_env_var(), util::make_new_path(path))
1488 format!("{} {} {}", lib_path_cmd_prefix(libpath), prog, args.join(" "))
1492 fn dump_output(config: &Config, testpaths: &TestPaths, out: &str, err: &str) {
1493 dump_output_file(config, testpaths, out, "out");
1494 dump_output_file(config, testpaths, err, "err");
1495 maybe_dump_to_stdout(config, out, err);
1498 fn dump_output_file(config: &Config,
1499 testpaths: &TestPaths,
1502 let outfile = make_out_name(config, testpaths, extension);
1503 File::create(&outfile).unwrap().write_all(out.as_bytes()).unwrap();
1506 fn make_out_name(config: &Config, testpaths: &TestPaths, extension: &str) -> PathBuf {
1507 output_base_name(config, testpaths).with_extension(extension)
1510 fn aux_output_dir_name(config: &Config, testpaths: &TestPaths) -> PathBuf {
1511 let f = output_base_name(config, testpaths);
1512 let mut fname = f.file_name().unwrap().to_os_string();
1513 fname.push(&format!(".{}.libaux", config.mode));
1514 f.with_file_name(&fname)
1517 fn output_testname(filepath: &Path) -> PathBuf {
1518 PathBuf::from(filepath.file_stem().unwrap())
1521 fn output_base_name(config: &Config, testpaths: &TestPaths) -> PathBuf {
1522 let dir = config.build_base.join(&testpaths.relative_dir);
1524 // Note: The directory `dir` is created during `collect_tests_from_dir`
1526 .join(&output_testname(&testpaths.file))
1527 .with_extension(&config.stage_id)
1530 fn maybe_dump_to_stdout(config: &Config, out: &str, err: &str) {
1532 println!("------{}------------------------------", "stdout");
1533 println!("{}", out);
1534 println!("------{}------------------------------", "stderr");
1535 println!("{}", err);
1536 println!("------------------------------------------");
1540 fn error(err: &str) { println!("\nerror: {}", err); }
1542 fn fatal(err: &str) -> ! { error(err); panic!(); }
1544 fn fatal_proc_rec(err: &str, proc_res: &ProcRes) -> ! {
1550 ------------------------------------------\n\
1552 ------------------------------------------\n\
1554 ------------------------------------------\n\
1556 ------------------------------------------\n\
1558 err, proc_res.status, proc_res.cmdline, proc_res.stdout,
1563 fn _arm_exec_compiled_test(config: &Config,
1565 testpaths: &TestPaths,
1566 env: Vec<(String, String)>)
1568 let args = make_run_args(config, props, testpaths);
1569 let cmdline = make_cmdline("",
1573 // get bare program string
1574 let mut tvec: Vec<String> = args.prog
1578 let prog_short = tvec.pop().unwrap();
1581 let copy_result = procsrv::run("",
1587 config.adb_test_dir.clone()
1589 vec!(("".to_owned(), "".to_owned())),
1590 Some("".to_owned()))
1591 .expect(&format!("failed to exec `{}`", config.adb_path));
1594 println!("push ({}) {} {} {}",
1601 logv(config, format!("executing ({}) {}", config.target, cmdline));
1603 let mut runargs = Vec::new();
1605 // run test via adb_run_wrapper
1606 runargs.push("shell".to_owned());
1607 for (key, val) in env {
1608 runargs.push(format!("{}={}", key, val));
1610 runargs.push(format!("{}/../adb_run_wrapper.sh", config.adb_test_dir));
1611 runargs.push(format!("{}", config.adb_test_dir));
1612 runargs.push(format!("{}", prog_short));
1614 for tv in &args.args {
1615 runargs.push(tv.to_owned());
1621 vec!(("".to_owned(), "".to_owned())), Some("".to_owned()))
1622 .expect(&format!("failed to exec `{}`", config.adb_path));
1624 // get exitcode of result
1625 runargs = Vec::new();
1626 runargs.push("shell".to_owned());
1627 runargs.push("cat".to_owned());
1628 runargs.push(format!("{}/{}.exitcode", config.adb_test_dir, prog_short));
1630 let procsrv::Result{ out: exitcode_out, err: _, status: _ } =
1635 vec!(("".to_owned(), "".to_owned())),
1636 Some("".to_owned()))
1637 .expect(&format!("failed to exec `{}`", config.adb_path));
1639 let mut exitcode: i32 = 0;
1640 for c in exitcode_out.chars() {
1641 if !c.is_numeric() { break; }
1642 exitcode = exitcode * 10 + match c {
1643 '0' ... '9' => c as i32 - ('0' as i32),
1648 // get stdout of result
1649 runargs = Vec::new();
1650 runargs.push("shell".to_owned());
1651 runargs.push("cat".to_owned());
1652 runargs.push(format!("{}/{}.stdout", config.adb_test_dir, prog_short));
1654 let procsrv::Result{ out: stdout_out, err: _, status: _ } =
1659 vec!(("".to_owned(), "".to_owned())),
1660 Some("".to_owned()))
1661 .expect(&format!("failed to exec `{}`", config.adb_path));
1663 // get stderr of result
1664 runargs = Vec::new();
1665 runargs.push("shell".to_owned());
1666 runargs.push("cat".to_owned());
1667 runargs.push(format!("{}/{}.stderr", config.adb_test_dir, prog_short));
1669 let procsrv::Result{ out: stderr_out, err: _, status: _ } =
1674 vec!(("".to_owned(), "".to_owned())),
1675 Some("".to_owned()))
1676 .expect(&format!("failed to exec `{}`", config.adb_path));
1684 status: Status::Parsed(exitcode),
1691 fn _arm_push_aux_shared_library(config: &Config, testpaths: &TestPaths) {
1692 let tdir = aux_output_dir_name(config, testpaths);
1694 let dirs = fs::read_dir(&tdir).unwrap();
1696 let file = file.unwrap().path();
1697 if file.extension().and_then(|s| s.to_str()) == Some("so") {
1698 // FIXME (#9639): This needs to handle non-utf8 paths
1699 let copy_result = procsrv::run("",
1707 config.adb_test_dir.to_owned(),
1709 vec!(("".to_owned(),
1711 Some("".to_owned()))
1712 .expect(&format!("failed to exec `{}`", config.adb_path));
1715 println!("push ({}) {:?} {} {}",
1716 config.target, file.display(),
1717 copy_result.out, copy_result.err);
1723 // codegen tests (using FileCheck)
1725 fn compile_test_and_save_ir(config: &Config, props: &TestProps,
1726 testpaths: &TestPaths) -> ProcRes {
1727 let aux_dir = aux_output_dir_name(config, testpaths);
1728 // FIXME (#9639): This needs to handle non-utf8 paths
1729 let mut link_args = vec!("-L".to_owned(),
1730 aux_dir.to_str().unwrap().to_owned());
1731 let llvm_args = vec!("--emit=llvm-ir".to_owned(),);
1732 link_args.extend(llvm_args);
1733 let args = make_compile_args(config,
1736 |a, b| TargetLocation::ThisDirectory(
1737 output_base_name(a, b).parent()
1738 .unwrap().to_path_buf()),
1740 compose_and_run_compiler(config, props, testpaths, args, None)
1743 fn check_ir_with_filecheck(config: &Config, testpaths: &TestPaths) -> ProcRes {
1744 let irfile = output_base_name(config, testpaths).with_extension("ll");
1745 let prog = config.llvm_bin_path.as_ref().unwrap().join("FileCheck");
1746 let proc_args = ProcArgs {
1747 // FIXME (#9639): This needs to handle non-utf8 paths
1748 prog: prog.to_str().unwrap().to_owned(),
1749 args: vec!(format!("-input-file={}", irfile.to_str().unwrap()),
1750 testpaths.file.to_str().unwrap().to_owned())
1752 compose_and_run(config, testpaths, proc_args, Vec::new(), "", None, None)
1755 fn run_codegen_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
1757 if config.llvm_bin_path.is_none() {
1758 fatal("missing --llvm-bin-path");
1761 let mut proc_res = compile_test_and_save_ir(config, props, testpaths);
1762 if !proc_res.status.success() {
1763 fatal_proc_rec("compilation failed!", &proc_res);
1766 proc_res = check_ir_with_filecheck(config, testpaths);
1767 if !proc_res.status.success() {
1768 fatal_proc_rec("verification with 'FileCheck' failed",
1773 fn charset() -> &'static str {
1774 // FreeBSD 10.1 defaults to GDB 6.1.1 which doesn't support "auto" charset
1775 if cfg!(target_os = "bitrig") {
1777 } else if cfg!(target_os = "freebsd") {
1784 fn run_rustdoc_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
1785 let out_dir = output_base_name(config, testpaths);
1786 let _ = fs::remove_dir_all(&out_dir);
1787 ensure_dir(&out_dir);
1789 let proc_res = document(config, props, testpaths, &out_dir);
1790 if !proc_res.status.success() {
1791 fatal_proc_rec("rustdoc failed!", &proc_res);
1793 let root = find_rust_src_root(config).unwrap();
1795 let res = cmd2procres(config,
1797 Command::new(&config.python)
1798 .arg(root.join("src/etc/htmldocck.py"))
1800 .arg(&testpaths.file));
1801 if !res.status.success() {
1802 fatal_proc_rec("htmldocck failed!", &res);
1806 fn run_codegen_units_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
1807 let proc_res = compile_test(config, props, testpaths);
1809 if !proc_res.status.success() {
1810 fatal_proc_rec("compilation failed!", &proc_res);
1813 check_no_compiler_crash(&proc_res);
1815 let prefix = "TRANS_ITEM ";
1817 let actual: HashSet<String> = proc_res
1820 .filter(|line| line.starts_with(prefix))
1821 .map(|s| (&s[prefix.len()..]).to_string())
1824 let expected: HashSet<String> = errors::load_errors(&testpaths.file)
1826 .map(|e| e.msg.trim().to_string())
1829 if actual != expected {
1830 let mut missing: Vec<_> = expected.difference(&actual).collect();
1833 let mut too_much: Vec<_> = actual.difference(&expected).collect();
1836 println!("Expected and actual sets of codegen-items differ.\n\
1837 These items should have been contained but were not:\n\n\
1839 These items were contained but should not have been:\n\n\
1841 missing.iter().fold("".to_string(), |s1, s2| s1 + "\n" + s2),
1842 too_much.iter().fold("".to_string(), |s1, s2| s1 + "\n" + s2));