Add some soundness checks to startup

Author: felsenhower

conftest.py

@@ -26,7 +26,7 @@
 import pytest
 
 import util
-from util import partdiff_params_tuple
+from util import ReferenceSource, partdiff_params_tuple
 
 
 def shlex_list_str(value: str) -> list[str]:
@@ -41,16 +41,16 @@ def shlex_list_str(value: str) -> list[str]:
     return shlex.split(value)
 
 
-def reference_source_param(value: str) -> util.ReferenceSource:
+def reference_source_param(value: str) -> ReferenceSource:
     """Parse a ReferenceSource from str.
 
     Args:
         value (str): The str to parse.
 
     Returns:
-        util.ReferenceSource: The parsed ReferenceSource.
+        ReferenceSource: The parsed ReferenceSource.
     """
-    return util.ReferenceSource(value)
+    return ReferenceSource(value)
 
 
 REGEX_NUM_LIST = re.compile(r"^(?:\d+(?:-\d+)?)(?:,\d+(?:-\d+)?)*$")
@@ -217,8 +217,8 @@ def pytest_addoption(parser: pytest.Parser) -> None:
             "auto == try cache and fall back to impl)."
         ),
         type=reference_source_param,
-        default=util.ReferenceSource.cache,
-        choices=util.ReferenceSource,
+        default=ReferenceSource.cache,
+        choices=ReferenceSource,
     )
     custom_options.addoption(
         "--num-threads",
@@ -315,6 +315,16 @@ def pytest_configure(config: pytest.Config) -> None:
     """
     See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_configure
     """
+    if config.getoption("reference_source") in (
+        ReferenceSource.auto,
+        ReferenceSource.impl,
+    ):
+        util.ensure_reference_implementation_exists()
+
+    util.check_executable_exists(
+        config.getoption("executable"), config.getoption("cwd")
+    )
+
     if config.getoption("valgrind"):
         if shutil.which("valgrind") is None:
             raise RuntimeError("Passed --valgrind, but valgrind could not be found.")

util.py

@@ -252,7 +252,6 @@ def get_from_cache():
         return reference_output_data[partdiff_params]
 
     def get_from_impl():
-        ensure_reference_implementation_exists()
         command_line = [REFERENCE_IMPLEMENTATION_EXEC] + list(partdiff_params)
         return subprocess.check_output(command_line).decode("utf-8")
 
@@ -296,3 +295,18 @@ def get_actual_output(
     if use_valgrind:
         command_line = ["valgrind", "--leak-check=full"] + command_line
     return subprocess.check_output(command_line, cwd=cwd).decode("utf-8")
+
+
+def check_executable_exists(executable: list[str], cwd: Path | None) -> None:
+    """Check if the executable exists and can be executed by running it.
+
+    The executable is allowed to return a non-zero exit status.
+
+    Args:
+        executable (list[str]): The executable to check
+        cwd (Path | None): The working directory of the executable.
+    """
+    try:
+        subprocess.check_output(executable, cwd=cwd)
+    except subprocess.CalledProcessError:
+        pass