diff --git a/changelog-entries/735.md b/changelog-entries/735.md new file mode 100644 index 000000000..56ca71a67 --- /dev/null +++ b/changelog-entries/735.md @@ -0,0 +1 @@ +- Add optional `timeout` field to `tests.yaml` entries for per-test timeout configuration. Overridable via `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable. diff --git a/tools/tests/README.md b/tools/tests/README.md index 41d406f86..1ec6f36c5 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -335,8 +335,11 @@ test_suites: - fluid-openfoam - solid-fenics reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz + timeout: 1200 ``` +The optional `timeout` field (in seconds) sets the maximum time for the solver run and fieldcompare phases of that specific case. If omitted, it defaults to `GLOBAL_TIMEOUT` (currently 900s, overridable via the `PRECICE_SYSTEMTESTS_TIMEOUT` environment variable). + This defines two test suites, namely `openfoam_adapter_pr` and `openfoam_adapter_release`. Each of them defines which case combinations of which tutorials to run. ### Generate Reference Results diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index e3ac5261e..a9bac5ac1 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -2,7 +2,7 @@ from metadata_parser.metdata import Tutorials, ReferenceResult from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT from pathlib import Path from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR @@ -144,12 +144,14 @@ def main(): for tutorial in tutorials: max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) + timeouts = test_suite.timeouts.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None + timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 4968c7486..6debe18d0 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,7 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table +from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -60,12 +60,14 @@ def main(): for tutorial in tutorials: max_times = test_suite.max_times.get(tutorial, []) mtw_list = test_suite.max_time_windows.get(tutorial, []) + timeouts = test_suite.timeouts.get(tutorial, []) for i, (case, reference_result) in enumerate(zip( test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial])): max_time = max_times[i] if i < len(max_times) else None max_time_windows = mtw_list[i] if i < len(mtw_list) else None + timeout = timeouts[i] if i < len(timeouts) and timeouts[i] is not None else GLOBAL_TIMEOUT systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time, max_time_windows=max_time_windows, timeout=timeout)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 97ad188c3..f62f0fb32 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -19,7 +19,7 @@ import os -GLOBAL_TIMEOUT = 1200 +GLOBAL_TIMEOUT = int(os.environ.get("PRECICE_SYSTEMTESTS_TIMEOUT", 900)) SHORT_TIMEOUT = 10 @@ -136,6 +136,7 @@ class Systemtest: reference_result: ReferenceResult max_time: float | None = None max_time_windows: int | None = None + timeout: int = GLOBAL_TIMEOUT params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -429,7 +430,7 @@ def _run_field_compare(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() raise KeyboardInterrupt from k @@ -518,7 +519,7 @@ def _run_tutorial(self): cwd=self.system_test_dir) try: - stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + stdout, stderr = process.communicate(timeout=self.timeout) except KeyboardInterrupt as k: process.kill() # process.send_signal(9) diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 5bb0e3738..dff317ad9 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -12,6 +12,7 @@ class TestSuite: reference_results: Dict[Tutorial, List[ReferenceResult]] max_times: Dict[Tutorial, list] = field(default_factory=dict) max_time_windows: Dict[Tutorial, list] = field(default_factory=dict) + timeouts: Dict[Tutorial, List] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -52,6 +53,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): reference_results_of_tutorial = {} max_times_of_tutorial = {} max_time_windows_of_tutorial = {} + timeouts_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -63,6 +65,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): reference_results_of_tutorial[tutorial] = [] max_times_of_tutorial[tutorial] = [] max_time_windows_of_tutorial[tutorial] = [] + timeouts_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -80,12 +83,20 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if mtw_raw is not None and (not isinstance(mtw_raw, int) or mtw_raw <= 0): raise ValueError(f"max_time_windows must be a positive integer, got {mtw_raw!r}") max_time_windows_of_tutorial[tutorial].append(mtw_raw) + + timeout_value = tutorial_case.get('timeout', None) + if timeout_value is not None and not isinstance(timeout_value, int): + raise TypeError( + f"Expected 'timeout' to be an integer or None, but got {type(timeout_value).__name__} " + f"(value: {timeout_value}) in tutorial '{tutorial}'." + ) + timeouts_of_tutorial[tutorial].append(timeout_value) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial, max_time_windows_of_tutorial, timeouts_of_tutorial)) return cls(testsuites)