Add timeout parameter

Author: felsenhower

README.md

@@ -82,9 +82,10 @@ Custom options for test_partdiff:
                         (default: $PWD).
   --shuffle=[SEED]      Shuffle the test cases.
   --allow-extra-iterations=n
-                        For term=prec, allow more iterations than the (serial)
+                        For term=acc, allow more iterations than the (serial)
                         reference implementation would do (0 == disallow; n ==
                         allow n more; -1 == unlimited)
+  --timeout=n           Stop EXECUTABLE after n seconds (can be float).
 ```
 
 The custom options are explained below.
@@ -225,3 +226,7 @@ Allowed values for this parameter are:
 
 > [!IMPORTANT]
 > Since `partdiff_tester` probably needs to execute the reference implementation in the described scenario, it is best to always pass `--reference-source=auto` alongside this parameter. Otherwise, the tests will likely fail.
+
+### `timeout`
+
+A timeout in seconds for `EXECUTABLE`. Zero means no timeout.

conftest.py

@@ -234,6 +234,27 @@ def shuffle_type(value: str) -> tuple[ShuffleType, int | None]:
     return (ShuffleType.SHUFFLE_EXPL_SEED, int(value))
 
 
+def timeout(value: str) -> float | None:
+    """Parse a timeout value from a str.
+
+    Args:
+        value (str): The value to parse.
+
+    Raises:
+        ValueError: When the value cannot be parsed or is not a non-negative
+          float.
+
+    Returns:
+        float | None: The parsed timeout, or None if 0 was passed.
+    """
+    parsed_value = float(value)
+    if parsed_value < 0:
+        raise ValueError("Timeout must be positive.")
+    if parsed_value == 0.0:
+        return None
+    return parsed_value
+
+
 def pytest_addoption(parser: pytest.Parser) -> None:
     """
     See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_addoption
@@ -329,6 +350,13 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         type=extra_iterations,
         default=0,
     )
+    custom_options.addoption(
+        "--timeout",
+        help="Stop EXECUTABLE after n seconds (can be float). Zero means no timeout.",
+        metavar="n",
+        type=timeout,
+        default=None,
+    )
 
 
 @pytest.fixture

test_partdiff.py

@@ -62,9 +62,10 @@ def test_partdiff_parametrized(
     reference_source = pytestconfig.getoption("reference_source")
     cwd = pytestconfig.getoption("cwd")
     allow_extra_iterations = pytestconfig.getoption("allow_extra_iterations")
+    timeout = pytestconfig.getoption("timeout")
 
     actual_output = util.get_actual_output(
-        partdiff_params, partdiff_executable, use_valgrind, cwd
+        partdiff_params, partdiff_executable, use_valgrind, cwd, timeout
     )
     reference_output = util.get_reference_output(
         partdiff_params, reference_output_data, reference_source

util.py

@@ -230,6 +230,7 @@ def get_actual_output(
     partdiff_executable: list[str],
     use_valgrind: bool,
     cwd: Path | None,
+    timeout: float | None,
 ) -> str:
     """Get the actual output for a parameter combination.
 
@@ -238,14 +239,17 @@ def get_actual_output(
         partdiff_executable (list[str]): The executable to run.
         use_valgrind (bool): Wether valgrind shall be used.
         cwd (Path | None): The working directory of the executable.
+        timeout (float | None): The timeout in seconds, or None for no timeout.
 
     Returns:
         str: The output of the executable.
     """
     command_line = partdiff_executable + list(partdiff_params)
     if use_valgrind:
         command_line = ["valgrind", "--leak-check=full"] + command_line
-    return subprocess.check_output(command_line, cwd=cwd).decode("utf-8")
+    return subprocess.check_output(command_line, cwd=cwd, timeout=timeout).decode(
+        "utf-8"
+    )
 
 
 def check_executable_exists(executable: list[str], cwd: Path | None) -> None: