System tests: Add configurable simulation timeout per test case

changelog-entries/735.md

@@ -0,0 +1 @@
+- Add optional `timeout` field to `tests.yaml` entries for per-test timeout configuration. Overridable via `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable.

tools/tests/README.md

@@ -335,8 +335,11 @@ test_suites:
           - fluid-openfoam
           - solid-fenics
         reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
+        timeout: 1200
 ```
 
+The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable).
+
 This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run.
 
 ### Generate Reference Results

tools/tests/generate_reference_results.py

@@ -2,7 +2,7 @@
 from metadata_parser.metdata import Tutorials, ReferenceResult
 from systemtests.TestSuite import TestSuites
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT
 from pathlib import Path
 from typing import List
 from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR
@@ -144,12 +144,14 @@ def main():
         for tutorial in tutorials:
             max_times = test_suite.max_times.get(tutorial, [])
             mtw_list = test_suite.max_time_windows.get(tutorial, [])
+            timeouts = test_suite.timeouts.get(tutorial, [])
             for i, (case, reference_result) in enumerate(zip(
                     test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
                 max_time = max_times[i] if i < len(max_times) else None
                 max_time_windows = mtw_list[i] if i < len(mtw_list) else None
+                timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
                 systemtests_to_run.add(
-                    Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows))
+                    Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout))
 
     reference_result_per_tutorial = {}
     current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

tools/tests/systemtests.py

@@ -2,7 +2,7 @@
 import argparse
 from pathlib import Path
 from systemtests.SystemtestArguments import SystemtestArguments
-from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table
+from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table
 from systemtests.TestSuite import TestSuites
 from metadata_parser.metdata import Tutorials, Case
 import logging
@@ -60,12 +60,14 @@ def main():
             for tutorial in tutorials:
                 max_times = test_suite.max_times.get(tutorial, [])
                 mtw_list = test_suite.max_time_windows.get(tutorial, [])
+                timeouts = test_suite.timeouts.get(tutorial, [])
                 for i, (case, reference_result) in enumerate(zip(
                         test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])):
                     max_time = max_times[i] if i < len(max_times) else None
                     max_time_windows = mtw_list[i] if i < len(mtw_list) else None
+                    timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT
                     systemtests_to_run.append(
-                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows))
+                        Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout))
 
     if not systemtests_to_run:
         raise RuntimeError("Did not find any Systemtests to execute.")

tools/tests/systemtests/Systemtest.py

@@ -19,7 +19,7 @@
 import os
 
 
-GLOBAL_TIMEOUT = 1200
+GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 900))
 SHORT_TIMEOUT = 10
 
 
@@ -136,6 +136,7 @@ class Systemtest:
     reference_result: ReferenceResult
     max_time: float | None = None
     max_time_windows: int | None = None
+    timeout: int = GLOBAL_TIMEOUT
     params_to_use: Dict[str, str] = field(init=False)
     env: Dict[str, str] = field(init=False)
 
@@ -429,7 +430,7 @@ def _run_field_compare(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 raise KeyboardInterrupt from k
@@ -518,7 +519,7 @@ def _run_tutorial(self):
                                        cwd=self.system_test_dir)
 
             try:
-                stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT)
+                stdout, stderr = process.communicate(timeout=self.timeout)
             except KeyboardInterrupt as k:
                 process.kill()
                 # process.send_signal(9)

tools/tests/systemtests/TestSuite.py

@@ -12,6 +12,7 @@ class TestSuite:
     reference_results: Dict[Tutorial, List[ReferenceResult]]
     max_times: Dict[Tutorial, list] = field(default_factory=dict)
     max_time_windows: Dict[Tutorial, list] = field(default_factory=dict)
+    timeouts: Dict[Tutorial, List] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return_string = f"Test suite: {self.name} contains:"
@@ -52,6 +53,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                 reference_results_of_tutorial = {}
                 max_times_of_tutorial = {}
                 max_time_windows_of_tutorial = {}
+                timeouts_of_tutorial = {}
                 # iterate over tutorials:
                 for tutorial_case in test_suites_raw[test_suite_name]['tutorials']:
                     tutorial = parsed_tutorials.get_by_path(tutorial_case['path'])
@@ -63,6 +65,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         reference_results_of_tutorial[tutorial] = []
                         max_times_of_tutorial[tutorial] = []
                         max_time_windows_of_tutorial[tutorial] = []
+                        timeouts_of_tutorial[tutorial] = []
 
                     all_case_combinations = tutorial.case_combinations
                     case_combination_requested = CaseCombination.from_string_list(
@@ -80,12 +83,20 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials):
                         if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0):
                             raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}")
                         max_time_windows_of_tutorial[tutorial].append(mtw_raw)
+
+                        timeout_value = tutorial_case.get('timeout', None)
+                        if timeout_value is not None and not isinstance(timeout_value, int):
+                            raise TypeError(
+                                f"Expected 'timeout' to be an integer or None, but got {type(timeout_value).__name__} "
+                                f"(value: {timeout_value}) in tutorial '{tutorial}'."
+                            )
+                        timeouts_of_tutorial[tutorial].append(timeout_value)
                     else:
                         raise Exception(
                             f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}")
 
                 testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial,
-                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial))
+                                            reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial))
 
         return cls(testsuites)