src/ci/docker/scripts/fuchsia-test-runner.py

   1 #!/usr/bin/env python3
   2
   3 """
   4 The Rust toolchain test runner for Fuchsia.
   5
   6 For instructions on running the compiler test suite, see
   7 https://doc.rust-lang.org/stable/rustc/platform-support/fuchsia.html#aarch64-unknown-fuchsia-and-x86_64-unknown-fuchsia
   8 """
   9
  10 import argparse
  11 from dataclasses import dataclass
  12 import glob
  13 import hashlib
  14 import json
  15 import os
  16 import platform
  17 import re
  18 import shutil
  19 import signal
  20 import subprocess
  21 import sys
  22 from typing import ClassVar, List
  23
  24
  25 @dataclass
  26 class TestEnvironment:
  27     rust_dir: str
  28     sdk_dir: str
  29     target_arch: str
  30     package_server_pid: int = None
  31     emu_addr: str = None
  32     libstd_name: str = None
  33     libtest_name: str = None
  34     verbose: bool = False
  35
  36     @staticmethod
  37     def tmp_dir():
  38         tmp_dir = os.environ.get("TEST_TOOLCHAIN_TMP_DIR")
  39         if tmp_dir is not None:
  40             return os.path.abspath(tmp_dir)
  41         return os.path.join(os.path.dirname(__file__), "tmp~")
  42
  43     @classmethod
  44     def env_file_path(cls):
  45         return os.path.join(cls.tmp_dir(), "test_env.json")
  46
  47     @classmethod
  48     def from_args(cls, args):
  49         return cls(
  50             os.path.abspath(args.rust),
  51             os.path.abspath(args.sdk),
  52             args.target_arch,
  53             verbose=args.verbose,
  54         )
  55
  56     @classmethod
  57     def read_from_file(cls):
  58         with open(cls.env_file_path(), encoding="utf-8") as f:
  59             test_env = json.loads(f.read())
  60             return cls(
  61                 test_env["rust_dir"],
  62                 test_env["sdk_dir"],
  63                 test_env["target_arch"],
  64                 libstd_name=test_env["libstd_name"],
  65                 libtest_name=test_env["libtest_name"],
  66                 emu_addr=test_env["emu_addr"],
  67                 package_server_pid=test_env["package_server_pid"],
  68                 verbose=test_env["verbose"],
  69             )
  70
  71     def image_name(self):
  72         if self.target_arch == "x64":
  73             return "qemu-x64"
  74         if self.target_arch == "arm64":
  75             return "qemu-arm64"
  76         raise Exception(f"Unrecognized target architecture {self.target_arch}")
  77
  78     def write_to_file(self):
  79         with open(self.env_file_path(), "w", encoding="utf-8") as f:
  80             f.write(json.dumps(self.__dict__))
  81
  82     def ssh_dir(self):
  83         return os.path.join(self.tmp_dir(), "ssh")
  84
  85     def ssh_keyfile_path(self):
  86         return os.path.join(self.ssh_dir(), "fuchsia_ed25519")
  87
  88     def ssh_authfile_path(self):
  89         return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys")
  90
  91     def vdl_output_path(self):
  92         return os.path.join(self.tmp_dir(), "vdl_output")
  93
  94     def package_server_log_path(self):
  95         return os.path.join(self.tmp_dir(), "package_server_log")
  96
  97     def emulator_log_path(self):
  98         return os.path.join(self.tmp_dir(), "emulator_log")
  99
 100     def packages_dir(self):
 101         return os.path.join(self.tmp_dir(), "packages")
 102
 103     def output_dir(self):
 104         return os.path.join(self.tmp_dir(), "output")
 105
 106     TEST_REPO_NAME: ClassVar[str] = "rust-testing"
 107
 108     def repo_dir(self):
 109         return os.path.join(self.tmp_dir(), self.TEST_REPO_NAME)
 110
 111     def rustlib_dir(self):
 112         if self.target_arch == "x64":
 113             return "x86_64-unknown-fuchsia"
 114         if self.target_arch == "arm64":
 115             return "aarch64-unknown-fuchsia"
 116         raise Exception(f"Unrecognized target architecture {self.target_arch}")
 117
 118     def libs_dir(self):
 119         return os.path.join(
 120             self.rust_dir,
 121             "lib",
 122         )
 123
 124     def rustlibs_dir(self):
 125         return os.path.join(
 126             self.libs_dir(),
 127             "rustlib",
 128             self.rustlib_dir(),
 129             "lib",
 130         )
 131
 132     def sdk_arch(self):
 133         machine = platform.machine()
 134         if machine == "x86_64":
 135             return "x64"
 136         if machine == "arm":
 137             return "a64"
 138         raise Exception(f"Unrecognized host architecture {machine}")
 139
 140     def tool_path(self, tool):
 141         return os.path.join(self.sdk_dir, "tools", self.sdk_arch(), tool)
 142
 143     def host_arch_triple(self):
 144         machine = platform.machine()
 145         if machine == "x86_64":
 146             return "x86_64-unknown-linux-gnu"
 147         if machine == "arm":
 148             return "aarch64-unknown-linux-gnu"
 149         raise Exception(f"Unrecognized host architecture {machine}")
 150
 151     def zxdb_script_path(self):
 152         return os.path.join(self.tmp_dir(), "zxdb_script")
 153
 154     def log_info(self, msg):
 155         print(msg)
 156
 157     def log_debug(self, msg):
 158         if self.verbose:
 159             print(msg)
 160
 161     def subprocess_output(self):
 162         if self.verbose:
 163             return sys.stdout
 164         return subprocess.DEVNULL
 165
 166     def ffx_daemon_log_path(self):
 167         return os.path.join(self.tmp_dir(), "ffx_daemon_log")
 168
 169     def ffx_isolate_dir(self):
 170         return os.path.join(self.tmp_dir(), "ffx_isolate")
 171
 172     def ffx_home_dir(self):
 173         return os.path.join(self.ffx_isolate_dir(), "user-home")
 174
 175     def ffx_tmp_dir(self):
 176         return os.path.join(self.ffx_isolate_dir(), "tmp")
 177
 178     def ffx_log_dir(self):
 179         return os.path.join(self.ffx_isolate_dir(), "log")
 180
 181     def ffx_user_config_dir(self):
 182         return os.path.join(self.ffx_xdg_config_home(), "Fuchsia", "ffx", "config")
 183
 184     def ffx_user_config_path(self):
 185         return os.path.join(self.ffx_user_config_dir(), "config.json")
 186
 187     def ffx_xdg_config_home(self):
 188         if platform.system() == "Darwin":
 189             return os.path.join(self.ffx_home_dir(), "Library", "Preferences")
 190         return os.path.join(self.ffx_home_dir(), ".local", "share")
 191
 192     def ffx_ascendd_path(self):
 193         return os.path.join(self.ffx_tmp_dir(), "ascendd")
 194
 195     def start_ffx_isolation(self):
 196         # Most of this is translated directly from ffx's isolate library
 197         os.mkdir(self.ffx_isolate_dir())
 198         os.mkdir(self.ffx_home_dir())
 199         os.mkdir(self.ffx_tmp_dir())
 200         os.mkdir(self.ffx_log_dir())
 201
 202         fuchsia_dir = os.path.join(self.ffx_home_dir(), ".fuchsia")
 203         os.mkdir(fuchsia_dir)
 204
 205         fuchsia_debug_dir = os.path.join(fuchsia_dir, "debug")
 206         os.mkdir(fuchsia_debug_dir)
 207
 208         metrics_dir = os.path.join(fuchsia_dir, "metrics")
 209         os.mkdir(metrics_dir)
 210
 211         analytics_path = os.path.join(metrics_dir, "analytics-status")
 212         with open(analytics_path, "w", encoding="utf-8") as analytics_file:
 213             print("0", file=analytics_file)
 214
 215         ffx_path = os.path.join(metrics_dir, "ffx")
 216         with open(ffx_path, "w", encoding="utf-8") as ffx_file:
 217             print("1", file=ffx_file)
 218
 219         os.makedirs(self.ffx_user_config_dir())
 220
 221         with open(
 222             self.ffx_user_config_path(), "w", encoding="utf-8"
 223         ) as config_json_file:
 224             user_config_for_test = {
 225                 "log": {
 226                     "enabled": True,
 227                     "dir": self.ffx_log_dir(),
 228                 },
 229                 "overnet": {
 230                     "socket": self.ffx_ascendd_path(),
 231                 },
 232                 "ssh": {
 233                     "pub": self.ssh_authfile_path(),
 234                     "priv": self.ssh_keyfile_path(),
 235                 },
 236                 "test": {
 237                     "is_isolated": True,
 238                     "experimental_structured_output": True,
 239                 },
 240             }
 241             print(json.dumps(user_config_for_test), file=config_json_file)
 242
 243         ffx_env_path = os.path.join(self.ffx_user_config_dir(), ".ffx_env")
 244         with open(ffx_env_path, "w", encoding="utf-8") as ffx_env_file:
 245             ffx_env_config_for_test = {
 246                 "user": self.ffx_user_config_path(),
 247                 "build": None,
 248                 "global": None,
 249             }
 250             print(json.dumps(ffx_env_config_for_test), file=ffx_env_file)
 251
 252         # Start ffx daemon
 253         # We want this to be a long-running process that persists after the script finishes
 254         # pylint: disable=consider-using-with
 255         with open(
 256             self.ffx_daemon_log_path(), "w", encoding="utf-8"
 257         ) as ffx_daemon_log_file:
 258             subprocess.Popen(
 259                 [
 260                     self.tool_path("ffx"),
 261                     "--config",
 262                     self.ffx_user_config_path(),
 263                     "daemon",
 264                     "start",
 265                 ],
 266                 env=self.ffx_cmd_env(),
 267                 stdout=ffx_daemon_log_file,
 268                 stderr=ffx_daemon_log_file,
 269             )
 270
 271     def ffx_cmd_env(self):
 272         result = {
 273             "HOME": self.ffx_home_dir(),
 274             "XDG_CONFIG_HOME": self.ffx_xdg_config_home(),
 275             "ASCENDD": self.ffx_ascendd_path(),
 276             "FUCHSIA_SSH_KEY": self.ssh_keyfile_path(),
 277             # We want to use our own specified temp directory
 278             "TMP": self.tmp_dir(),
 279             "TEMP": self.tmp_dir(),
 280             "TMPDIR": self.tmp_dir(),
 281             "TEMPDIR": self.tmp_dir(),
 282         }
 283
 284         return result
 285
 286     def stop_ffx_isolation(self):
 287         subprocess.check_call(
 288             [
 289                 self.tool_path("ffx"),
 290                 "--config",
 291                 self.ffx_user_config_path(),
 292                 "daemon",
 293                 "stop",
 294             ],
 295             env=self.ffx_cmd_env(),
 296             stdout=self.subprocess_output(),
 297             stderr=self.subprocess_output(),
 298         )
 299
 300     def start(self):
 301         """Sets up the testing environment and prepares to run tests.
 302
 303         Args:
 304             args: The command-line arguments to this command.
 305
 306         During setup, this function will:
 307         - Locate necessary shared libraries
 308         - Create a new temp directory (this is where all temporary files are stored)
 309         - Start an emulator
 310         - Start an update server
 311         - Create a new package repo and register it with the emulator
 312         - Write test environment settings to a temporary file
 313         """
 314
 315         # Initialize temp directory
 316         if not os.path.exists(self.tmp_dir()):
 317             os.mkdir(self.tmp_dir())
 318         elif len(os.listdir(self.tmp_dir())) != 0:
 319             raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})")
 320
 321         os.mkdir(self.ssh_dir())
 322         os.mkdir(self.output_dir())
 323
 324         # Find libstd and libtest
 325         libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
 326         libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
 327
 328         if not libstd_paths:
 329             raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
 330
 331         if not libtest_paths:
 332             raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
 333
 334         self.libstd_name = os.path.basename(libstd_paths[0])
 335         self.libtest_name = os.path.basename(libtest_paths[0])
 336
 337         # Generate SSH keys for the emulator to use
 338         self.log_info("Generating SSH keys...")
 339         subprocess.check_call(
 340             [
 341                 "ssh-keygen",
 342                 "-N",
 343                 "",
 344                 "-t",
 345                 "ed25519",
 346                 "-f",
 347                 self.ssh_keyfile_path(),
 348                 "-C",
 349                 "Generated by fuchsia-test-runner.py",
 350             ],
 351             stdout=self.subprocess_output(),
 352             stderr=self.subprocess_output(),
 353         )
 354         authfile_contents = subprocess.check_output(
 355             [
 356                 "ssh-keygen",
 357                 "-y",
 358                 "-f",
 359                 self.ssh_keyfile_path(),
 360             ],
 361             stderr=self.subprocess_output(),
 362         )
 363         with open(self.ssh_authfile_path(), "wb") as authfile:
 364             authfile.write(authfile_contents)
 365
 366         # Start ffx isolation
 367         self.log_info("Starting ffx isolation...")
 368         self.start_ffx_isolation()
 369
 370         # Start emulator (this will generate the vdl output)
 371         self.log_info("Starting emulator...")
 372         subprocess.check_call(
 373             [
 374                 self.tool_path("fvdl"),
 375                 "--sdk",
 376                 "start",
 377                 "--tuntap",
 378                 "--headless",
 379                 "--nointeractive",
 380                 "--ssh",
 381                 self.ssh_dir(),
 382                 "--vdl-output",
 383                 self.vdl_output_path(),
 384                 "--emulator-log",
 385                 self.emulator_log_path(),
 386                 "--image-name",
 387                 self.image_name(),
 388             ],
 389             stdout=self.subprocess_output(),
 390             stderr=self.subprocess_output(),
 391         )
 392
 393         # Parse vdl output for relevant information
 394         with open(self.vdl_output_path(), encoding="utf-8") as f:
 395             vdl_content = f.read()
 396             matches = re.search(
 397                 r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"',
 398                 vdl_content,
 399             )
 400             self.emu_addr = matches.group(1)
 401
 402         # Create new package repo
 403         self.log_info("Creating package repo...")
 404         subprocess.check_call(
 405             [
 406                 self.tool_path("pm"),
 407                 "newrepo",
 408                 "-repo",
 409                 self.repo_dir(),
 410             ],
 411             stdout=self.subprocess_output(),
 412             stderr=self.subprocess_output(),
 413         )
 414
 415         # Start package server
 416         self.log_info("Starting package server...")
 417         with open(
 418             self.package_server_log_path(), "w", encoding="utf-8"
 419         ) as package_server_log:
 420             # We want this to be a long-running process that persists after the script finishes
 421             # pylint: disable=consider-using-with
 422             self.package_server_pid = subprocess.Popen(
 423                 [
 424                     self.tool_path("pm"),
 425                     "serve",
 426                     "-vt",
 427                     "-repo",
 428                     self.repo_dir(),
 429                     "-l",
 430                     ":8084",
 431                 ],
 432                 stdout=package_server_log,
 433                 stderr=package_server_log,
 434             ).pid
 435
 436         # Register package server with emulator
 437         self.log_info("Registering package server...")
 438         ssh_client = subprocess.check_output(
 439             [
 440                 "ssh",
 441                 "-i",
 442                 self.ssh_keyfile_path(),
 443                 "-o",
 444                 "StrictHostKeyChecking=accept-new",
 445                 self.emu_addr,
 446                 "-f",
 447                 "echo $SSH_CLIENT",
 448             ],
 449             text=True,
 450         )
 451         repo_addr = ssh_client.split()[0].replace("%", "%25")
 452         repo_url = f"http://[{repo_addr}]:8084/config.json"
 453         subprocess.check_call(
 454             [
 455                 "ssh",
 456                 "-i",
 457                 self.ssh_keyfile_path(),
 458                 "-o",
 459                 "StrictHostKeyChecking=accept-new",
 460                 self.emu_addr,
 461                 "-f",
 462                 f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}",
 463             ],
 464             stdout=self.subprocess_output(),
 465             stderr=self.subprocess_output(),
 466         )
 467
 468         # Write to file
 469         self.write_to_file()
 470
 471         self.log_info("Success! Your environment is ready to run tests.")
 472
 473     # FIXME: shardify this
 474     # `facet` statement required for TCP testing via
 475     # protocol `fuchsia.posix.socket.Provider`. See
 476     # https://fuchsia.dev/fuchsia-src/development/testing/components/test_runner_framework?hl=en#legacy_non-hermetic_tests
 477     CML_TEMPLATE: ClassVar[
 478         str
 479     ] = """
 480     {{
 481         program: {{
 482             runner: "elf_test_runner",
 483             binary: "bin/{exe_name}",
 484             forward_stderr_to: "log",
 485             forward_stdout_to: "log",
 486             environ: [{env_vars}
 487             ]
 488         }},
 489         capabilities: [
 490             {{ protocol: "fuchsia.test.Suite" }},
 491         ],
 492         expose: [
 493             {{
 494                 protocol: "fuchsia.test.Suite",
 495                 from: "self",
 496             }},
 497         ],
 498         use: [
 499             {{ storage: "data", path: "/data" }},
 500             {{ protocol: [ "fuchsia.process.Launcher" ] }},
 501             {{ protocol: [ "fuchsia.posix.socket.Provider" ] }}
 502         ],
 503         facets: {{
 504             "fuchsia.test": {{ type: "system" }},
 505         }},
 506     }}
 507     """
 508
 509     MANIFEST_TEMPLATE = """
 510     meta/package={package_dir}/meta/package
 511     meta/{package_name}.cm={package_dir}/meta/{package_name}.cm
 512     bin/{exe_name}={bin_path}
 513     lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name}
 514     lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name}
 515     lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/lib/libc.so
 516     lib/libzircon.so={sdk_dir}/arch/{target_arch}/sysroot/lib/libzircon.so
 517     lib/libfdio.so={sdk_dir}/arch/{target_arch}/lib/libfdio.so
 518     """
 519
 520     TEST_ENV_VARS: ClassVar[List[str]] = [
 521         "TEST_EXEC_ENV",
 522         "RUST_MIN_STACK",
 523         "RUST_BACKTRACE",
 524         "RUST_NEWRT",
 525         "RUST_LOG",
 526         "RUST_TEST_THREADS",
 527     ]
 528
 529     def run(self, args):
 530         """Runs the requested test in the testing environment.
 531
 532         Args:
 533         args: The command-line arguments to this command.
 534         Returns:
 535         The return code of the test (0 for success, else failure).
 536
 537         To run a test, this function will:
 538         - Create, compile, archive, and publish a test package
 539         - Run the test package on the emulator
 540         - Forward the test's stdout and stderr as this script's stdout and stderr
 541         """
 542
 543         bin_path = os.path.abspath(args.bin_path)
 544
 545         # Build a unique, deterministic name for the test using the name of the
 546         # binary and the last 6 hex digits of the hash of the full path
 547         def path_checksum(path):
 548             m = hashlib.sha256()
 549             m.update(path.encode("utf-8"))
 550             return m.hexdigest()[0:6]
 551
 552         base_name = os.path.basename(os.path.dirname(args.bin_path))
 553         exe_name = base_name.lower().replace(".", "_")
 554         package_name = f"{exe_name}_{path_checksum(bin_path)}"
 555
 556         package_dir = os.path.join(self.packages_dir(), package_name)
 557         cml_path = os.path.join(package_dir, "meta", f"{package_name}.cml")
 558         cm_path = os.path.join(package_dir, "meta", f"{package_name}.cm")
 559         manifest_path = os.path.join(package_dir, f"{package_name}.manifest")
 560         far_path = os.path.join(package_dir, f"{package_name}-0.far")
 561
 562         shared_libs = args.shared_libs[: args.n]
 563         arguments = args.shared_libs[args.n :]
 564
 565         test_output_dir = os.path.join(self.output_dir(), package_name)
 566
 567         # Clean and create temporary output directory
 568         if os.path.exists(test_output_dir):
 569             shutil.rmtree(test_output_dir)
 570
 571         os.mkdir(test_output_dir)
 572
 573         # Open log file
 574         log_path = os.path.join(test_output_dir, "log")
 575         with open(log_path, "w", encoding="utf-8") as log_file:
 576
 577             def log(msg):
 578                 print(msg, file=log_file)
 579                 log_file.flush()
 580
 581             log(f"Bin path: {bin_path}")
 582
 583             log("Setting up package...")
 584
 585             # Set up package
 586             subprocess.check_call(
 587                 [
 588                     self.tool_path("pm"),
 589                     "-o",
 590                     package_dir,
 591                     "-n",
 592                     package_name,
 593                     "init",
 594                 ],
 595                 stdout=log_file,
 596                 stderr=log_file,
 597             )
 598
 599             log("Writing CML...")
 600
 601             # Write and compile CML
 602             with open(cml_path, "w", encoding="utf-8") as cml:
 603                 # Collect environment variables
 604                 env_vars = ""
 605                 for var_name in self.TEST_ENV_VARS:
 606                     var_value = os.getenv(var_name)
 607                     if var_value is not None:
 608                         env_vars += f'\n            "{var_name}={var_value}",'
 609
 610                 # Default to no backtrace for test suite
 611                 if os.getenv("RUST_BACKTRACE") == None:
 612                     env_vars += f'\n            "RUST_BACKTRACE=0",'
 613
 614                 cml.write(
 615                     self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name)
 616                 )
 617
 618             log("Compiling CML...")
 619
 620             subprocess.check_call(
 621                 [
 622                     self.tool_path("cmc"),
 623                     "compile",
 624                     cml_path,
 625                     "--includepath",
 626                     ".",
 627                     "--output",
 628                     cm_path,
 629                 ],
 630                 stdout=log_file,
 631                 stderr=log_file,
 632             )
 633
 634             log("Writing manifest...")
 635
 636             # Write, build, and archive manifest
 637             with open(manifest_path, "w", encoding="utf-8") as manifest:
 638                 manifest.write(
 639                     self.MANIFEST_TEMPLATE.format(
 640                         bin_path=bin_path,
 641                         exe_name=exe_name,
 642                         package_dir=package_dir,
 643                         package_name=package_name,
 644                         rust_dir=self.rust_dir,
 645                         rustlib_dir=self.rustlib_dir(),
 646                         sdk_dir=self.sdk_dir,
 647                         libstd_name=self.libstd_name,
 648                         libtest_name=self.libtest_name,
 649                         target_arch=self.target_arch,
 650                     )
 651                 )
 652                 for shared_lib in shared_libs:
 653                     manifest.write(f"lib/{os.path.basename(shared_lib)}={shared_lib}\n")
 654
 655             log("Compiling and archiving manifest...")
 656
 657             subprocess.check_call(
 658                 [
 659                     self.tool_path("pm"),
 660                     "-o",
 661                     package_dir,
 662                     "-m",
 663                     manifest_path,
 664                     "build",
 665                 ],
 666                 stdout=log_file,
 667                 stderr=log_file,
 668             )
 669             subprocess.check_call(
 670                 [
 671                     self.tool_path("pm"),
 672                     "-o",
 673                     package_dir,
 674                     "-m",
 675                     manifest_path,
 676                     "archive",
 677                 ],
 678                 stdout=log_file,
 679                 stderr=log_file,
 680             )
 681
 682             log("Publishing package to repo...")
 683
 684             # Publish package to repo
 685             subprocess.check_call(
 686                 [
 687                     self.tool_path("pm"),
 688                     "publish",
 689                     "-a",
 690                     "-repo",
 691                     self.repo_dir(),
 692                     "-f",
 693                     far_path,
 694                 ],
 695                 stdout=log_file,
 696                 stderr=log_file,
 697             )
 698
 699             log("Running ffx test...")
 700
 701             # Run test on emulator
 702             subprocess.run(
 703                 [
 704                     self.tool_path("ffx"),
 705                     "--config",
 706                     self.ffx_user_config_path(),
 707                     "test",
 708                     "run",
 709                     f"fuchsia-pkg://{self.TEST_REPO_NAME}/{package_name}#meta/{package_name}.cm",
 710                     "--min-severity-logs",
 711                     "TRACE",
 712                     "--output-directory",
 713                     test_output_dir,
 714                     "--",
 715                 ]
 716                 + arguments,
 717                 env=self.ffx_cmd_env(),
 718                 check=False,
 719                 stdout=log_file,
 720                 stderr=log_file,
 721             )
 722
 723             log("Reporting test suite output...")
 724
 725             # Read test suite output
 726             run_summary_path = os.path.join(test_output_dir, "run_summary.json")
 727             if os.path.exists(run_summary_path):
 728                 with open(run_summary_path, encoding="utf-8") as f:
 729                     run_summary = json.loads(f.read())
 730
 731                 suite = run_summary["data"]["suites"][0]
 732                 case = suite["cases"][0]
 733
 734                 return_code = 0 if case["outcome"] == "PASSED" else 1
 735
 736                 artifacts = case["artifacts"]
 737                 artifact_dir = case["artifact_dir"]
 738                 stdout_path = None
 739                 stderr_path = None
 740
 741                 for path, artifact in artifacts.items():
 742                     artifact_path = os.path.join(test_output_dir, artifact_dir, path)
 743                     artifact_type = artifact["artifact_type"]
 744
 745                     if artifact_type == "STDERR":
 746                         stderr_path = artifact_path
 747                     elif artifact_type == "STDOUT":
 748                         stdout_path = artifact_path
 749
 750                 if stdout_path is not None and os.path.exists(stdout_path):
 751                     with open(stdout_path, encoding="utf-8") as f:
 752                         print(f.read(), file=sys.stdout, end="")
 753
 754                 if stderr_path is not None and os.path.exists(stderr_path):
 755                     with open(stderr_path, encoding="utf-8") as f:
 756                         print(f.read(), file=sys.stderr, end="")
 757             else:
 758                 log("Failed to open test run summary")
 759                 return_code = 254
 760
 761             log("Done!")
 762
 763         return return_code
 764
 765     def stop(self):
 766         """Shuts down and cleans up the testing environment.
 767
 768         Args:
 769         args: The command-line arguments to this command.
 770         Returns:
 771         The return code of the test (0 for success, else failure).
 772
 773         During cleanup, this function will stop the emulator, package server, and
 774         update server, then delete all temporary files. If an error is encountered
 775         while stopping any running processes, the temporary files will not be deleted.
 776         Passing --delete-tmp will force the process to delete the files anyway.
 777         """
 778
 779         self.log_debug("Reporting logs...")
 780
 781         # Print test log files
 782         for test_dir in os.listdir(self.output_dir()):
 783             log_path = os.path.join(self.output_dir(), test_dir, "log")
 784             self.log_debug(f"\n---- Logs for test '{test_dir}' ----\n")
 785             if os.path.exists(log_path):
 786                 with open(log_path, encoding="utf-8") as log:
 787                     self.log_debug(log.read())
 788             else:
 789                 self.log_debug("No logs found")
 790
 791         # Print the emulator log
 792         self.log_debug("\n---- Emulator logs ----\n")
 793         if os.path.exists(self.emulator_log_path()):
 794             with open(self.emulator_log_path(), encoding="utf-8") as log:
 795                 self.log_debug(log.read())
 796         else:
 797             self.log_debug("No emulator logs found")
 798
 799         # Print the package server log
 800         self.log_debug("\n---- Package server log ----\n")
 801         if os.path.exists(self.package_server_log_path()):
 802             with open(self.package_server_log_path(), encoding="utf-8") as log:
 803                 self.log_debug(log.read())
 804         else:
 805             self.log_debug("No package server log found")
 806
 807         # Print the ffx daemon log
 808         self.log_debug("\n---- ffx daemon log ----\n")
 809         if os.path.exists(self.ffx_daemon_log_path()):
 810             with open(self.ffx_daemon_log_path(), encoding="utf-8") as log:
 811                 self.log_debug(log.read())
 812         else:
 813             self.log_debug("No ffx daemon log found")
 814
 815         # Stop package server
 816         self.log_info("Stopping package server...")
 817         os.kill(self.package_server_pid, signal.SIGTERM)
 818
 819         # Shut down the emulator
 820         self.log_info("Stopping emulator...")
 821         subprocess.check_call(
 822             [
 823                 self.tool_path("fvdl"),
 824                 "--sdk",
 825                 "kill",
 826                 "--launched-proto",
 827                 self.vdl_output_path(),
 828             ],
 829             stdout=self.subprocess_output(),
 830             stderr=self.subprocess_output(),
 831         )
 832
 833         # Stop ffx isolation
 834         self.log_info("Stopping ffx isolation...")
 835         self.stop_ffx_isolation()
 836
 837     def delete_tmp(self):
 838         # Remove temporary files
 839         self.log_info("Deleting temporary files...")
 840         shutil.rmtree(self.tmp_dir(), ignore_errors=True)
 841
 842     def debug(self, args):
 843         command = [
 844             self.tool_path("ffx"),
 845             "--config",
 846             self.ffx_user_config_path(),
 847             "debug",
 848             "connect",
 849             "--",
 850             "--build-id-dir",
 851             os.path.join(self.sdk_dir, ".build-id"),
 852             "--build-id-dir",
 853             os.path.join(self.libs_dir(), ".build-id"),
 854         ]
 855
 856         # Add rust source if it's available
 857         if args.rust_src is not None:
 858             command += [
 859                 "--build-dir",
 860                 args.rust_src,
 861             ]
 862
 863         # Add fuchsia source if it's available
 864         if args.fuchsia_src is not None:
 865             command += [
 866                 "--build-dir",
 867                 os.path.join(args.fuchsia_src, "out", "default"),
 868             ]
 869
 870         # Load debug symbols for the test binary and automatically attach
 871         if args.test is not None:
 872             if args.rust_src is None:
 873                 raise Exception(
 874                     "A Rust source path is required with the `test` argument"
 875                 )
 876
 877             test_name = os.path.splitext(os.path.basename(args.test))[0]
 878
 879             build_dir = os.path.join(
 880                 args.rust_src,
 881                 "fuchsia-build",
 882                 self.host_arch_triple(),
 883             )
 884             test_dir = os.path.join(
 885                 build_dir,
 886                 "test",
 887                 os.path.dirname(args.test),
 888                 test_name,
 889             )
 890
 891             with open(self.zxdb_script_path(), mode="w", encoding="utf-8") as f:
 892                 print(f"attach {test_name[:31]}", file=f)
 893
 894             command += [
 895                 "--symbol-path",
 896                 test_dir,
 897                 "-S",
 898                 self.zxdb_script_path(),
 899             ]
 900
 901         # Add any other zxdb arguments the user passed
 902         if args.zxdb_args is not None:
 903             command += args.zxdb_args
 904
 905         # Connect to the running emulator with zxdb
 906         subprocess.run(command, env=self.ffx_cmd_env(), check=False)
 907
 908
 909 def start(args):
 910     test_env = TestEnvironment.from_args(args)
 911     test_env.start()
 912     return 0
 913
 914
 915 def run(args):
 916     test_env = TestEnvironment.read_from_file()
 917     return test_env.run(args)
 918
 919
 920 def stop(args):
 921     test_env = TestEnvironment.read_from_file()
 922     test_env.stop()
 923     if not args.no_delete:
 924         test_env.delete_tmp()
 925     return 0
 926
 927
 928 def delete_tmp(args):
 929     del args
 930     test_env = TestEnvironment.read_from_file()
 931     test_env.delete_tmp()
 932     return 0
 933
 934
 935 def debug(args):
 936     test_env = TestEnvironment.read_from_file()
 937     test_env.debug(args)
 938     return 0
 939
 940
 941 def main():
 942     parser = argparse.ArgumentParser()
 943
 944     def print_help(args):
 945         del args
 946         parser.print_help()
 947         return 0
 948
 949     parser.set_defaults(func=print_help)
 950
 951     subparsers = parser.add_subparsers(help="valid sub-commands")
 952
 953     start_parser = subparsers.add_parser(
 954         "start", help="initializes the testing environment"
 955     )
 956     start_parser.add_argument(
 957         "--rust",
 958         help="the directory of the installed Rust compiler for Fuchsia",
 959         required=True,
 960     )
 961     start_parser.add_argument(
 962         "--sdk",
 963         help="the directory of the fuchsia SDK",
 964         required=True,
 965     )
 966     start_parser.add_argument(
 967         "--verbose",
 968         help="prints more output from executed processes",
 969         action="store_true",
 970     )
 971     start_parser.add_argument(
 972         "--target-arch",
 973         help="the architecture of the image to test",
 974         required=True,
 975     )
 976     start_parser.set_defaults(func=start)
 977
 978     run_parser = subparsers.add_parser(
 979         "run", help="run a test in the testing environment"
 980     )
 981     run_parser.add_argument(
 982         "n", help="the number of shared libs passed along with the executable", type=int
 983     )
 984     run_parser.add_argument("bin_path", help="path to the binary to run")
 985     run_parser.add_argument(
 986         "shared_libs",
 987         help="the shared libs passed along with the binary",
 988         nargs=argparse.REMAINDER,
 989     )
 990     run_parser.set_defaults(func=run)
 991
 992     stop_parser = subparsers.add_parser(
 993         "stop", help="shuts down and cleans up the testing environment"
 994     )
 995     stop_parser.add_argument(
 996         "--no-delete",
 997         default=False,
 998         action="store_true",
 999         help="don't delete temporary files after stopping",
1000     )
1001     stop_parser.set_defaults(func=stop)
1002
1003     delete_parser = subparsers.add_parser(
1004         "delete-tmp",
1005         help="deletes temporary files after the testing environment has been manually cleaned up",
1006     )
1007     delete_parser.set_defaults(func=delete_tmp)
1008
1009     debug_parser = subparsers.add_parser(
1010         "debug",
1011         help="connect to the active testing environment with zxdb",
1012     )
1013     debug_parser.add_argument(
1014         "--rust-src",
1015         default=None,
1016         help="the path to the Rust source being tested",
1017     )
1018     debug_parser.add_argument(
1019         "--fuchsia-src",
1020         default=None,
1021         help="the path to the Fuchsia source",
1022     )
1023     debug_parser.add_argument(
1024         "--test",
1025         default=None,
1026         help="the path to the test to debug (e.g. ui/box/new.rs)",
1027     )
1028     debug_parser.add_argument(
1029         "zxdb_args",
1030         default=None,
1031         nargs=argparse.REMAINDER,
1032         help="any additional arguments to pass to zxdb",
1033     )
1034     debug_parser.set_defaults(func=debug)
1035
1036     args = parser.parse_args()
1037     return args.func(args)
1038
1039
1040 if __name__ == "__main__":
1041     sys.exit(main())