4 The Rust toolchain test runner for Fuchsia.
6 For instructions on running the compiler test suite, see
7 https://doc.rust-lang.org/stable/rustc/platform-support/fuchsia.html#aarch64-unknown-fuchsia-and-x86_64-unknown-fuchsia
11 from dataclasses import dataclass
22 from typing import ClassVar, List
26 class TestEnvironment:
30 package_server_pid: int = None
32 libstd_name: str = None
33 libtest_name: str = None
38 tmp_dir = os.environ.get("TEST_TOOLCHAIN_TMP_DIR")
39 if tmp_dir is not None:
40 return os.path.abspath(tmp_dir)
41 return os.path.join(os.path.dirname(__file__), "tmp~")
44 def env_file_path(cls):
45 return os.path.join(cls.tmp_dir(), "test_env.json")
48 def from_args(cls, args):
50 os.path.abspath(args.rust),
51 os.path.abspath(args.sdk),
57 def read_from_file(cls):
58 with open(cls.env_file_path(), encoding="utf-8") as f:
59 test_env = json.loads(f.read())
63 test_env["target_arch"],
64 libstd_name=test_env["libstd_name"],
65 libtest_name=test_env["libtest_name"],
66 emu_addr=test_env["emu_addr"],
67 package_server_pid=test_env["package_server_pid"],
68 verbose=test_env["verbose"],
72 if self.target_arch == "x64":
74 if self.target_arch == "arm64":
76 raise Exception(f"Unrecognized target architecture {self.target_arch}")
78 def write_to_file(self):
79 with open(self.env_file_path(), "w", encoding="utf-8") as f:
80 f.write(json.dumps(self.__dict__))
83 return os.path.join(self.tmp_dir(), "ssh")
85 def ssh_keyfile_path(self):
86 return os.path.join(self.ssh_dir(), "fuchsia_ed25519")
88 def ssh_authfile_path(self):
89 return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys")
91 def vdl_output_path(self):
92 return os.path.join(self.tmp_dir(), "vdl_output")
94 def package_server_log_path(self):
95 return os.path.join(self.tmp_dir(), "package_server_log")
97 def emulator_log_path(self):
98 return os.path.join(self.tmp_dir(), "emulator_log")
100 def packages_dir(self):
101 return os.path.join(self.tmp_dir(), "packages")
103 def output_dir(self):
104 return os.path.join(self.tmp_dir(), "output")
106 TEST_REPO_NAME: ClassVar[str] = "rust-testing"
109 return os.path.join(self.tmp_dir(), self.TEST_REPO_NAME)
111 def rustlib_dir(self):
112 if self.target_arch == "x64":
113 return "x86_64-unknown-fuchsia"
114 if self.target_arch == "arm64":
115 return "aarch64-unknown-fuchsia"
116 raise Exception(f"Unrecognized target architecture {self.target_arch}")
124 def rustlibs_dir(self):
133 machine = platform.machine()
134 if machine == "x86_64":
138 raise Exception(f"Unrecognized host architecture {machine}")
140 def tool_path(self, tool):
141 return os.path.join(self.sdk_dir, "tools", self.sdk_arch(), tool)
143 def host_arch_triple(self):
144 machine = platform.machine()
145 if machine == "x86_64":
146 return "x86_64-unknown-linux-gnu"
148 return "aarch64-unknown-linux-gnu"
149 raise Exception(f"Unrecognized host architecture {machine}")
151 def zxdb_script_path(self):
152 return os.path.join(self.tmp_dir(), "zxdb_script")
154 def log_info(self, msg):
157 def log_debug(self, msg):
161 def subprocess_output(self):
164 return subprocess.DEVNULL
166 def ffx_daemon_log_path(self):
167 return os.path.join(self.tmp_dir(), "ffx_daemon_log")
169 def ffx_isolate_dir(self):
170 return os.path.join(self.tmp_dir(), "ffx_isolate")
172 def ffx_home_dir(self):
173 return os.path.join(self.ffx_isolate_dir(), "user-home")
175 def ffx_tmp_dir(self):
176 return os.path.join(self.ffx_isolate_dir(), "tmp")
178 def ffx_log_dir(self):
179 return os.path.join(self.ffx_isolate_dir(), "log")
181 def ffx_user_config_dir(self):
182 return os.path.join(self.ffx_xdg_config_home(), "Fuchsia", "ffx", "config")
184 def ffx_user_config_path(self):
185 return os.path.join(self.ffx_user_config_dir(), "config.json")
187 def ffx_xdg_config_home(self):
188 if platform.system() == "Darwin":
189 return os.path.join(self.ffx_home_dir(), "Library", "Preferences")
190 return os.path.join(self.ffx_home_dir(), ".local", "share")
192 def ffx_ascendd_path(self):
193 return os.path.join(self.ffx_tmp_dir(), "ascendd")
195 def start_ffx_isolation(self):
196 # Most of this is translated directly from ffx's isolate library
197 os.mkdir(self.ffx_isolate_dir())
198 os.mkdir(self.ffx_home_dir())
199 os.mkdir(self.ffx_tmp_dir())
200 os.mkdir(self.ffx_log_dir())
202 fuchsia_dir = os.path.join(self.ffx_home_dir(), ".fuchsia")
203 os.mkdir(fuchsia_dir)
205 fuchsia_debug_dir = os.path.join(fuchsia_dir, "debug")
206 os.mkdir(fuchsia_debug_dir)
208 metrics_dir = os.path.join(fuchsia_dir, "metrics")
209 os.mkdir(metrics_dir)
211 analytics_path = os.path.join(metrics_dir, "analytics-status")
212 with open(analytics_path, "w", encoding="utf-8") as analytics_file:
213 print("0", file=analytics_file)
215 ffx_path = os.path.join(metrics_dir, "ffx")
216 with open(ffx_path, "w", encoding="utf-8") as ffx_file:
217 print("1", file=ffx_file)
219 os.makedirs(self.ffx_user_config_dir())
222 self.ffx_user_config_path(), "w", encoding="utf-8"
223 ) as config_json_file:
224 user_config_for_test = {
227 "dir": self.ffx_log_dir(),
230 "socket": self.ffx_ascendd_path(),
233 "pub": self.ssh_authfile_path(),
234 "priv": self.ssh_keyfile_path(),
238 "experimental_structured_output": True,
241 print(json.dumps(user_config_for_test), file=config_json_file)
243 ffx_env_path = os.path.join(self.ffx_user_config_dir(), ".ffx_env")
244 with open(ffx_env_path, "w", encoding="utf-8") as ffx_env_file:
245 ffx_env_config_for_test = {
246 "user": self.ffx_user_config_path(),
250 print(json.dumps(ffx_env_config_for_test), file=ffx_env_file)
253 # We want this to be a long-running process that persists after the script finishes
254 # pylint: disable=consider-using-with
256 self.ffx_daemon_log_path(), "w", encoding="utf-8"
257 ) as ffx_daemon_log_file:
260 self.tool_path("ffx"),
262 self.ffx_user_config_path(),
266 env=self.ffx_cmd_env(),
267 stdout=ffx_daemon_log_file,
268 stderr=ffx_daemon_log_file,
271 def ffx_cmd_env(self):
273 "HOME": self.ffx_home_dir(),
274 "XDG_CONFIG_HOME": self.ffx_xdg_config_home(),
275 "ASCENDD": self.ffx_ascendd_path(),
276 "FUCHSIA_SSH_KEY": self.ssh_keyfile_path(),
277 # We want to use our own specified temp directory
278 "TMP": self.tmp_dir(),
279 "TEMP": self.tmp_dir(),
280 "TMPDIR": self.tmp_dir(),
281 "TEMPDIR": self.tmp_dir(),
286 def stop_ffx_isolation(self):
287 subprocess.check_call(
289 self.tool_path("ffx"),
291 self.ffx_user_config_path(),
295 env=self.ffx_cmd_env(),
296 stdout=self.subprocess_output(),
297 stderr=self.subprocess_output(),
301 """Sets up the testing environment and prepares to run tests.
304 args: The command-line arguments to this command.
306 During setup, this function will:
307 - Locate necessary shared libraries
308 - Create a new temp directory (this is where all temporary files are stored)
310 - Start an update server
311 - Create a new package repo and register it with the emulator
312 - Write test environment settings to a temporary file
315 # Initialize temp directory
316 if not os.path.exists(self.tmp_dir()):
317 os.mkdir(self.tmp_dir())
318 elif len(os.listdir(self.tmp_dir())) != 0:
319 raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})")
321 os.mkdir(self.ssh_dir())
322 os.mkdir(self.output_dir())
324 # Find libstd and libtest
325 libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
326 libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
329 raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
331 if not libtest_paths:
332 raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
334 self.libstd_name = os.path.basename(libstd_paths[0])
335 self.libtest_name = os.path.basename(libtest_paths[0])
337 # Generate SSH keys for the emulator to use
338 self.log_info("Generating SSH keys...")
339 subprocess.check_call(
347 self.ssh_keyfile_path(),
349 "Generated by fuchsia-test-runner.py",
351 stdout=self.subprocess_output(),
352 stderr=self.subprocess_output(),
354 authfile_contents = subprocess.check_output(
359 self.ssh_keyfile_path(),
361 stderr=self.subprocess_output(),
363 with open(self.ssh_authfile_path(), "wb") as authfile:
364 authfile.write(authfile_contents)
366 # Start ffx isolation
367 self.log_info("Starting ffx isolation...")
368 self.start_ffx_isolation()
370 # Start emulator (this will generate the vdl output)
371 self.log_info("Starting emulator...")
372 subprocess.check_call(
374 self.tool_path("fvdl"),
383 self.vdl_output_path(),
385 self.emulator_log_path(),
389 stdout=self.subprocess_output(),
390 stderr=self.subprocess_output(),
393 # Parse vdl output for relevant information
394 with open(self.vdl_output_path(), encoding="utf-8") as f:
395 vdl_content = f.read()
397 r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"',
400 self.emu_addr = matches.group(1)
402 # Create new package repo
403 self.log_info("Creating package repo...")
404 subprocess.check_call(
406 self.tool_path("pm"),
411 stdout=self.subprocess_output(),
412 stderr=self.subprocess_output(),
415 # Start package server
416 self.log_info("Starting package server...")
418 self.package_server_log_path(), "w", encoding="utf-8"
419 ) as package_server_log:
420 # We want this to be a long-running process that persists after the script finishes
421 # pylint: disable=consider-using-with
422 self.package_server_pid = subprocess.Popen(
424 self.tool_path("pm"),
432 stdout=package_server_log,
433 stderr=package_server_log,
436 # Register package server with emulator
437 self.log_info("Registering package server...")
438 ssh_client = subprocess.check_output(
442 self.ssh_keyfile_path(),
444 "StrictHostKeyChecking=accept-new",
451 repo_addr = ssh_client.split()[0].replace("%", "%25")
452 repo_url = f"http://[{repo_addr}]:8084/config.json"
453 subprocess.check_call(
457 self.ssh_keyfile_path(),
459 "StrictHostKeyChecking=accept-new",
462 f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}",
464 stdout=self.subprocess_output(),
465 stderr=self.subprocess_output(),
471 self.log_info("Success! Your environment is ready to run tests.")
473 # FIXME: shardify this
474 # `facet` statement required for TCP testing via
475 # protocol `fuchsia.posix.socket.Provider`. See
476 # https://fuchsia.dev/fuchsia-src/development/testing/components/test_runner_framework?hl=en#legacy_non-hermetic_tests
477 CML_TEMPLATE: ClassVar[
482 runner: "elf_test_runner",
483 binary: "bin/{exe_name}",
484 forward_stderr_to: "log",
485 forward_stdout_to: "log",
490 {{ protocol: "fuchsia.test.Suite" }},
494 protocol: "fuchsia.test.Suite",
499 {{ storage: "data", path: "/data" }},
500 {{ protocol: [ "fuchsia.process.Launcher" ] }},
501 {{ protocol: [ "fuchsia.posix.socket.Provider" ] }}
504 "fuchsia.test": {{ type: "system" }},
509 MANIFEST_TEMPLATE = """
510 meta/package={package_dir}/meta/package
511 meta/{package_name}.cm={package_dir}/meta/{package_name}.cm
512 bin/{exe_name}={bin_path}
513 lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name}
514 lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name}
515 lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/lib/libc.so
516 lib/libzircon.so={sdk_dir}/arch/{target_arch}/sysroot/lib/libzircon.so
517 lib/libfdio.so={sdk_dir}/arch/{target_arch}/lib/libfdio.so
520 TEST_ENV_VARS: ClassVar[List[str]] = [
530 """Runs the requested test in the testing environment.
533 args: The command-line arguments to this command.
535 The return code of the test (0 for success, else failure).
537 To run a test, this function will:
538 - Create, compile, archive, and publish a test package
539 - Run the test package on the emulator
540 - Forward the test's stdout and stderr as this script's stdout and stderr
543 bin_path = os.path.abspath(args.bin_path)
545 # Build a unique, deterministic name for the test using the name of the
546 # binary and the last 6 hex digits of the hash of the full path
547 def path_checksum(path):
549 m.update(path.encode("utf-8"))
550 return m.hexdigest()[0:6]
552 base_name = os.path.basename(os.path.dirname(args.bin_path))
553 exe_name = base_name.lower().replace(".", "_")
554 package_name = f"{exe_name}_{path_checksum(bin_path)}"
556 package_dir = os.path.join(self.packages_dir(), package_name)
557 cml_path = os.path.join(package_dir, "meta", f"{package_name}.cml")
558 cm_path = os.path.join(package_dir, "meta", f"{package_name}.cm")
559 manifest_path = os.path.join(package_dir, f"{package_name}.manifest")
560 far_path = os.path.join(package_dir, f"{package_name}-0.far")
562 shared_libs = args.shared_libs[: args.n]
563 arguments = args.shared_libs[args.n :]
565 test_output_dir = os.path.join(self.output_dir(), package_name)
567 # Clean and create temporary output directory
568 if os.path.exists(test_output_dir):
569 shutil.rmtree(test_output_dir)
571 os.mkdir(test_output_dir)
574 log_path = os.path.join(test_output_dir, "log")
575 with open(log_path, "w", encoding="utf-8") as log_file:
578 print(msg, file=log_file)
581 log(f"Bin path: {bin_path}")
583 log("Setting up package...")
586 subprocess.check_call(
588 self.tool_path("pm"),
599 log("Writing CML...")
601 # Write and compile CML
602 with open(cml_path, "w", encoding="utf-8") as cml:
603 # Collect environment variables
605 for var_name in self.TEST_ENV_VARS:
606 var_value = os.getenv(var_name)
607 if var_value is not None:
608 env_vars += f'\n "{var_name}={var_value}",'
610 # Default to no backtrace for test suite
611 if os.getenv("RUST_BACKTRACE") == None:
612 env_vars += f'\n "RUST_BACKTRACE=0",'
615 self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name)
618 log("Compiling CML...")
620 subprocess.check_call(
622 self.tool_path("cmc"),
634 log("Writing manifest...")
636 # Write, build, and archive manifest
637 with open(manifest_path, "w", encoding="utf-8") as manifest:
639 self.MANIFEST_TEMPLATE.format(
642 package_dir=package_dir,
643 package_name=package_name,
644 rust_dir=self.rust_dir,
645 rustlib_dir=self.rustlib_dir(),
646 sdk_dir=self.sdk_dir,
647 libstd_name=self.libstd_name,
648 libtest_name=self.libtest_name,
649 target_arch=self.target_arch,
652 for shared_lib in shared_libs:
653 manifest.write(f"lib/{os.path.basename(shared_lib)}={shared_lib}\n")
655 log("Compiling and archiving manifest...")
657 subprocess.check_call(
659 self.tool_path("pm"),
669 subprocess.check_call(
671 self.tool_path("pm"),
682 log("Publishing package to repo...")
684 # Publish package to repo
685 subprocess.check_call(
687 self.tool_path("pm"),
699 log("Running ffx test...")
701 # Run test on emulator
704 self.tool_path("ffx"),
706 self.ffx_user_config_path(),
709 f"fuchsia-pkg://{self.TEST_REPO_NAME}/{package_name}#meta/{package_name}.cm",
710 "--min-severity-logs",
712 "--output-directory",
717 env=self.ffx_cmd_env(),
723 log("Reporting test suite output...")
725 # Read test suite output
726 run_summary_path = os.path.join(test_output_dir, "run_summary.json")
727 if os.path.exists(run_summary_path):
728 with open(run_summary_path, encoding="utf-8") as f:
729 run_summary = json.loads(f.read())
731 suite = run_summary["data"]["suites"][0]
732 case = suite["cases"][0]
734 return_code = 0 if case["outcome"] == "PASSED" else 1
736 artifacts = case["artifacts"]
737 artifact_dir = case["artifact_dir"]
741 for path, artifact in artifacts.items():
742 artifact_path = os.path.join(test_output_dir, artifact_dir, path)
743 artifact_type = artifact["artifact_type"]
745 if artifact_type == "STDERR":
746 stderr_path = artifact_path
747 elif artifact_type == "STDOUT":
748 stdout_path = artifact_path
750 if stdout_path is not None and os.path.exists(stdout_path):
751 with open(stdout_path, encoding="utf-8") as f:
752 print(f.read(), file=sys.stdout, end="")
754 if stderr_path is not None and os.path.exists(stderr_path):
755 with open(stderr_path, encoding="utf-8") as f:
756 print(f.read(), file=sys.stderr, end="")
758 log("Failed to open test run summary")
766 """Shuts down and cleans up the testing environment.
769 args: The command-line arguments to this command.
771 The return code of the test (0 for success, else failure).
773 During cleanup, this function will stop the emulator, package server, and
774 update server, then delete all temporary files. If an error is encountered
775 while stopping any running processes, the temporary files will not be deleted.
776 Passing --delete-tmp will force the process to delete the files anyway.
779 self.log_debug("Reporting logs...")
781 # Print test log files
782 for test_dir in os.listdir(self.output_dir()):
783 log_path = os.path.join(self.output_dir(), test_dir, "log")
784 self.log_debug(f"\n---- Logs for test '{test_dir}' ----\n")
785 if os.path.exists(log_path):
786 with open(log_path, encoding="utf-8") as log:
787 self.log_debug(log.read())
789 self.log_debug("No logs found")
791 # Print the emulator log
792 self.log_debug("\n---- Emulator logs ----\n")
793 if os.path.exists(self.emulator_log_path()):
794 with open(self.emulator_log_path(), encoding="utf-8") as log:
795 self.log_debug(log.read())
797 self.log_debug("No emulator logs found")
799 # Print the package server log
800 self.log_debug("\n---- Package server log ----\n")
801 if os.path.exists(self.package_server_log_path()):
802 with open(self.package_server_log_path(), encoding="utf-8") as log:
803 self.log_debug(log.read())
805 self.log_debug("No package server log found")
807 # Print the ffx daemon log
808 self.log_debug("\n---- ffx daemon log ----\n")
809 if os.path.exists(self.ffx_daemon_log_path()):
810 with open(self.ffx_daemon_log_path(), encoding="utf-8") as log:
811 self.log_debug(log.read())
813 self.log_debug("No ffx daemon log found")
815 # Stop package server
816 self.log_info("Stopping package server...")
817 os.kill(self.package_server_pid, signal.SIGTERM)
819 # Shut down the emulator
820 self.log_info("Stopping emulator...")
821 subprocess.check_call(
823 self.tool_path("fvdl"),
827 self.vdl_output_path(),
829 stdout=self.subprocess_output(),
830 stderr=self.subprocess_output(),
834 self.log_info("Stopping ffx isolation...")
835 self.stop_ffx_isolation()
837 def delete_tmp(self):
838 # Remove temporary files
839 self.log_info("Deleting temporary files...")
840 shutil.rmtree(self.tmp_dir(), ignore_errors=True)
842 def debug(self, args):
844 self.tool_path("ffx"),
846 self.ffx_user_config_path(),
851 os.path.join(self.sdk_dir, ".build-id"),
853 os.path.join(self.libs_dir(), ".build-id"),
856 # Add rust source if it's available
857 if args.rust_src is not None:
863 # Add fuchsia source if it's available
864 if args.fuchsia_src is not None:
867 os.path.join(args.fuchsia_src, "out", "default"),
870 # Load debug symbols for the test binary and automatically attach
871 if args.test is not None:
872 if args.rust_src is None:
874 "A Rust source path is required with the `test` argument"
877 test_name = os.path.splitext(os.path.basename(args.test))[0]
879 build_dir = os.path.join(
882 self.host_arch_triple(),
884 test_dir = os.path.join(
887 os.path.dirname(args.test),
891 with open(self.zxdb_script_path(), mode="w", encoding="utf-8") as f:
892 print(f"attach {test_name[:31]}", file=f)
898 self.zxdb_script_path(),
901 # Add any other zxdb arguments the user passed
902 if args.zxdb_args is not None:
903 command += args.zxdb_args
905 # Connect to the running emulator with zxdb
906 subprocess.run(command, env=self.ffx_cmd_env(), check=False)
910 test_env = TestEnvironment.from_args(args)
916 test_env = TestEnvironment.read_from_file()
917 return test_env.run(args)
921 test_env = TestEnvironment.read_from_file()
923 if not args.no_delete:
924 test_env.delete_tmp()
928 def delete_tmp(args):
930 test_env = TestEnvironment.read_from_file()
931 test_env.delete_tmp()
936 test_env = TestEnvironment.read_from_file()
942 parser = argparse.ArgumentParser()
944 def print_help(args):
949 parser.set_defaults(func=print_help)
951 subparsers = parser.add_subparsers(help="valid sub-commands")
953 start_parser = subparsers.add_parser(
954 "start", help="initializes the testing environment"
956 start_parser.add_argument(
958 help="the directory of the installed Rust compiler for Fuchsia",
961 start_parser.add_argument(
963 help="the directory of the fuchsia SDK",
966 start_parser.add_argument(
968 help="prints more output from executed processes",
971 start_parser.add_argument(
973 help="the architecture of the image to test",
976 start_parser.set_defaults(func=start)
978 run_parser = subparsers.add_parser(
979 "run", help="run a test in the testing environment"
981 run_parser.add_argument(
982 "n", help="the number of shared libs passed along with the executable", type=int
984 run_parser.add_argument("bin_path", help="path to the binary to run")
985 run_parser.add_argument(
987 help="the shared libs passed along with the binary",
988 nargs=argparse.REMAINDER,
990 run_parser.set_defaults(func=run)
992 stop_parser = subparsers.add_parser(
993 "stop", help="shuts down and cleans up the testing environment"
995 stop_parser.add_argument(
999 help="don't delete temporary files after stopping",
1001 stop_parser.set_defaults(func=stop)
1003 delete_parser = subparsers.add_parser(
1005 help="deletes temporary files after the testing environment has been manually cleaned up",
1007 delete_parser.set_defaults(func=delete_tmp)
1009 debug_parser = subparsers.add_parser(
1011 help="connect to the active testing environment with zxdb",
1013 debug_parser.add_argument(
1016 help="the path to the Rust source being tested",
1018 debug_parser.add_argument(
1021 help="the path to the Fuchsia source",
1023 debug_parser.add_argument(
1026 help="the path to the test to debug (e.g. ui/box/new.rs)",
1028 debug_parser.add_argument(
1031 nargs=argparse.REMAINDER,
1032 help="any additional arguments to pass to zxdb",
1034 debug_parser.set_defaults(func=debug)
1036 args = parser.parse_args()
1037 return args.func(args)
1040 if __name__ == "__main__":