Replace fvdl with ffx, allow test without install

Along with replacing fvdl uses with the equivalent ffx commands, this
also switches from using the install path for libstd-*.so and
libtest-*.so to using the build directory (now passed on the command
line). The user no longer needs to run x.py install before running tests
now, and the correct libstd and libtest are detected on run instead of
startup so the test runner can handle recompilations after starting the
testing environment.

src/ci/docker/scripts/fuchsia-test-runner.py

@@ -25,13 +25,9 @@ from typing import ClassVar, List, Optional
 
 @dataclass
 class TestEnvironment:
-    rust_dir: str
+    rust_build_dir: str
     sdk_dir: str
     target: str
-    package_server_pid: Optional[int] = None
-    emu_addr: Optional[str] = None
-    libstd_name: Optional[str] = None
-    libtest_name: Optional[str] = None
     verbose: bool = False
 
     @staticmethod
@@ -57,7 +53,7 @@ class TestEnvironment:
     @classmethod
     def from_args(cls, args):
         return cls(
-            os.path.abspath(args.rust),
+            os.path.abspath(args.rust_build),
             os.path.abspath(args.sdk),
             args.target,
             verbose=args.verbose,
@@ -68,13 +64,9 @@ class TestEnvironment:
         with open(cls.env_file_path(), encoding="utf-8") as f:
             test_env = json.loads(f.read())
             return cls(
-                test_env["rust_dir"],
+                test_env["rust_build_dir"],
                 test_env["sdk_dir"],
                 test_env["target"],
-                libstd_name=test_env["libstd_name"],
-                libtest_name=test_env["libtest_name"],
-                emu_addr=test_env["emu_addr"],
-                package_server_pid=test_env["package_server_pid"],
                 verbose=test_env["verbose"],
             )
 
@@ -82,18 +74,6 @@ class TestEnvironment:
         with open(self.env_file_path(), "w", encoding="utf-8") as f:
             f.write(json.dumps(self.__dict__))
 
-    def ssh_dir(self):
-        return os.path.join(self.tmp_dir(), "ssh")
-
-    def ssh_keyfile_path(self):
-        return os.path.join(self.ssh_dir(), "fuchsia_ed25519")
-
-    def ssh_authfile_path(self):
-        return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys")
-
-    def vdl_output_path(self):
-        return os.path.join(self.tmp_dir(), "vdl_output")
-
     def package_server_log_path(self):
         return os.path.join(self.tmp_dir(), "package_server_log")
 
@@ -113,7 +93,9 @@ class TestEnvironment:
 
     def libs_dir(self):
         return os.path.join(
-            self.rust_dir,
+            self.rust_build_dir,
+            "host",
+            "stage2",
             "lib",
         )
 
@@ -212,21 +194,19 @@ class TestEnvironment:
         # Set configs
         configs = {
             "log.enabled": "true",
-            "ssh.pub": self.ssh_authfile_path(),
-            "ssh.priv": self.ssh_keyfile_path(),
             "test.is_isolated": "true",
             "test.experimental_structured_output": "true",
         }
         for key, value in configs.items():
             subprocess.check_call(
                 [
-                    self.tool_path("ffx"),
+                    ffx_path,
                     "config",
                     "set",
                     key,
                     value,
                 ],
-                env=self.ffx_cmd_env(),
+                env=ffx_env,
                 stdout=self.subprocess_output(),
                 stderr=self.subprocess_output(),
             )
@@ -248,6 +228,7 @@ class TestEnvironment:
                 self.tool_path("ffx"),
                 "daemon",
                 "stop",
+                "-w",
             ],
             env=self.ffx_cmd_env(),
             stdout=self.subprocess_output(),
@@ -275,86 +256,61 @@ class TestEnvironment:
         elif len(os.listdir(self.tmp_dir())) != 0:
             raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})")
 
-        os.mkdir(self.ssh_dir())
         os.mkdir(self.output_dir())
 
-        # Find libstd and libtest
+        ffx_path = self.tool_path("ffx")
-        libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
+        ffx_env = self.ffx_cmd_env()
-        libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
-
-        if not libstd_paths:
-            raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
-
-        if not libtest_paths:
-            raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
-
-        self.libstd_name = os.path.basename(libstd_paths[0])
-        self.libtest_name = os.path.basename(libtest_paths[0])
-
-        # Generate SSH keys for the emulator to use
-        self.log_info("Generating SSH keys...")
-        subprocess.check_call(
-            [
-                "ssh-keygen",
-                "-N",
-                "",
-                "-t",
-                "ed25519",
-                "-f",
-                self.ssh_keyfile_path(),
-                "-C",
-                "Generated by fuchsia-test-runner.py",
-            ],
-            stdout=self.subprocess_output(),
-            stderr=self.subprocess_output(),
-        )
-        authfile_contents = subprocess.check_output(
-            [
-                "ssh-keygen",
-                "-y",
-                "-f",
-                self.ssh_keyfile_path(),
-            ],
-            stderr=self.subprocess_output(),
-        )
-        with open(self.ssh_authfile_path(), "wb") as authfile:
-            authfile.write(authfile_contents)
 
         # Start ffx isolation
         self.log_info("Starting ffx isolation...")
         self.start_ffx_isolation()
 
-        # Start emulator (this will generate the vdl output)
+        # Stop any running emulators (there shouldn't be any)
-        self.log_info("Starting emulator...")
         subprocess.check_call(
             [
-                self.tool_path("fvdl"),
+                ffx_path,
-                "--sdk",
+                "emu",
-                "start",
+                "stop",
-                "--tuntap",
+                "--all",
-                "--headless",
-                "--nointeractive",
-                "--ssh",
-                self.ssh_dir(),
-                "--vdl-output",
-                self.vdl_output_path(),
-                "--emulator-log",
-                self.emulator_log_path(),
-                "--image-name",
-                "qemu-" + self.triple_to_arch(self.target),
             ],
+            env=ffx_env,
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
 
-        # Parse vdl output for relevant information
+        # Start emulator
-        with open(self.vdl_output_path(), encoding="utf-8") as f:
+        self.log_info("Starting emulator...")
-            vdl_content = f.read()
+        product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target)
-            matches = re.search(
+        subprocess.check_call(
-                r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"',
+            [
-                vdl_content,
+                ffx_path,
-            )
+                "product-bundle",
-            self.emu_addr = matches.group(1)
+                "get",
+                product_bundle,
+            ],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
+        )
+        # FIXME: condition --accel hyper on target arch matching host arch
+        subprocess.check_call(
+            [
+                ffx_path,
+                "emu",
+                "start",
+                product_bundle,
+                "--headless",
+                "--log",
+                self.emulator_log_path(),
+                "--net",
+                "tap",
+                "--accel",
+                "hyper",
+            ],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
+        )
 
         # Create new package repo
         self.log_info("Creating package repo...")
@@ -369,55 +325,40 @@ class TestEnvironment:
             stderr=self.subprocess_output(),
         )
 
-        # Start package server
+        # Add repo
-        self.log_info("Starting package server...")
-        with open(
-            self.package_server_log_path(), "w", encoding="utf-8"
-        ) as package_server_log:
-            # We want this to be a long-running process that persists after the script finishes
-            # pylint: disable=consider-using-with
-            self.package_server_pid = subprocess.Popen(
-                [
-                    self.tool_path("pm"),
-                    "serve",
-                    "-vt",
-                    "-repo",
-                    self.repo_dir(),
-                    "-l",
-                    ":8084",
-                ],
-                stdout=package_server_log,
-                stderr=package_server_log,
-            ).pid
-
-        # Register package server with emulator
-        self.log_info("Registering package server...")
-        ssh_client = subprocess.check_output(
-            [
-                "ssh",
-                "-i",
-                self.ssh_keyfile_path(),
-                "-o",
-                "StrictHostKeyChecking=accept-new",
-                self.emu_addr,
-                "-f",
-                "echo $SSH_CLIENT",
-            ],
-            text=True,
-        )
-        repo_addr = ssh_client.split()[0].replace("%", "%25")
-        repo_url = f"http://[{repo_addr}]:8084/config.json"
         subprocess.check_call(
             [
-                "ssh",
+                ffx_path,
-                "-i",
+                "repository",
-                self.ssh_keyfile_path(),
+                "add-from-pm",
-                "-o",
+                self.repo_dir(),
-                "StrictHostKeyChecking=accept-new",
+                "--repository",
-                self.emu_addr,
+                self.TEST_REPO_NAME,
-                "-f",
-                f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}",
             ],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
+        )
+
+        # Start repository server
+        subprocess.check_call(
+            [ffx_path, "repository", "server", "start", "--address", "[::]:0"],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
+        )
+
+        # Register with newly-started emulator
+        subprocess.check_call(
+            [
+                ffx_path,
+                "target",
+                "repository",
+                "register",
+                "--repository",
+                self.TEST_REPO_NAME,
+            ],
+            env=ffx_env,
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
@@ -471,8 +412,8 @@ class TestEnvironment:
     meta/package={package_dir}/meta/package
     meta/{package_name}.cm={package_dir}/meta/{package_name}.cm
     bin/{exe_name}={bin_path}
-    lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name}
+    lib/{libstd_name}={libstd_path}
-    lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name}
+    lib/{libtest_name}={libtest_path}
     lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/dist/lib/ld.so.1
     lib/libfdio.so={sdk_dir}/arch/{target_arch}/dist/libfdio.so
     """
@@ -502,6 +443,16 @@ class TestEnvironment:
 
         bin_path = os.path.abspath(args.bin_path)
 
+        # Find libstd and libtest
+        libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
+        libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
+
+        if not libstd_paths:
+            raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
+
+        if not libtest_paths:
+            raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
+
         # Build a unique, deterministic name for the test using the name of the
         # binary and the last 6 hex digits of the hash of the full path
         def path_checksum(path):
@@ -604,11 +555,12 @@ class TestEnvironment:
                         exe_name=exe_name,
                         package_dir=package_dir,
                         package_name=package_name,
-                        rust_dir=self.rust_dir,
+                        target=self.target,
-                        rustlib_dir=self.target,
                         sdk_dir=self.sdk_dir,
-                        libstd_name=self.libstd_name,
+                        libstd_name=os.path.basename(libstd_paths[0]),
-                        libtest_name=self.libtest_name,
+                        libtest_name=os.path.basename(libtest_paths[0]),
+                        libstd_path=libstd_paths[0],
+                        libtest_path=libtest_paths[0],
                         target_arch=self.triple_to_arch(self.target),
                     )
                 )
@@ -779,20 +731,15 @@ class TestEnvironment:
         else:
             self.log_debug("No ffx daemon log found")
 
-        # Stop package server
-        self.log_info("Stopping package server...")
-        os.kill(self.package_server_pid, signal.SIGTERM)
-
         # Shut down the emulator
         self.log_info("Stopping emulator...")
         subprocess.check_call(
             [
-                self.tool_path("fvdl"),
+                self.tool_path("ffx"),
-                "--sdk",
+                "emu",
-                "kill",
+                "stop",
-                "--launched-proto",
-                self.vdl_output_path(),
             ],
+            env=self.ffx_cmd_env(),
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
@@ -969,8 +916,8 @@ def main():
         "start", help="initializes the testing environment"
     )
     start_parser.add_argument(
-        "--rust",
+        "--rust-build",
-        help="the directory of the installed Rust compiler for Fuchsia",
+        help="the current compiler build directory (`$RUST_SRC/build` by default)",
         required=True,
     )
     start_parser.add_argument(

src/doc/rustc/src/platform-support/fuchsia.md

@@ -681,12 +681,9 @@ local Rust source checkout:
 cd ${RUST_SRC_PATH}
 ```
 
-To run the Rust test suite on an emulated Fuchsia device, you must install the
+To run the Rust test suite on an emulated Fuchsia device, you'll also need to
-Rust compiler locally. See "[Targeting Fuchsia with a compiler built from source](#targeting-fuchsia-with-a-compiler-built-from-source)"
+download a copy of the Fuchsia SDK. The current minimum supported SDK version is
-for the steps to build locally.
+[10.20221207.2.89][minimum_supported_sdk_version].
-
-You'll also need to download a copy of the Fuchsia SDK. The current minimum
-supported SDK version is [10.20221207.2.89][minimum_supported_sdk_version].
 
 [minimum_supported_sdk_version]: https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core/linux-amd64/+/version:10.20221207.2.89
 
@@ -695,13 +692,13 @@ Fuchsia's test runner interacts with the Fuchsia emulator and is located at
 test environment with:
 
 ```sh
-src/ci/docker/scripts/fuchsia-test-runner.py start
+src/ci/docker/scripts/fuchsia-test-runner.py start                            \
-    --rust ${RUST_SRC_PATH}/install
+    --rust ${RUST_SRC_PATH}/build                                             \
-    --sdk ${SDK_PATH}
+    --sdk ${SDK_PATH}                                                         \
-    --target {x86_64-unknown-fuchsia|aarch64-unknown-fuchsia}
+    --target {x86_64-unknown-fuchsia|aarch64-unknown-fuchsia}                 \
 ```
 
-Where `${RUST_SRC_PATH}/install` is the `prefix` set in `config.toml` and
+Where `${RUST_SRC_PATH}/build` is the `build-dir` set in `config.toml` and
 `${SDK_PATH}` is the path to the downloaded and unzipped SDK.
 
 Once our environment is started, we can run our tests using `x.py` as usual. The