Allow extra iterations

README.md

@@ -75,6 +75,10 @@ Custom options for test_partdiff:
   --cwd=CWD             Set the working directory when launching EXECUTABLE
                         (default: $PWD).
   --shuffle             Shuffle the test cases.
+  --allow-extra-iterations=n
+                        For term=prec, allow more iterations than the (serial)
+                        reference implementation would do (0 == disallow; n ==
+                        allow n more; -1 == unlimited)
 ```
 
 The custom options are explained below.
@@ -185,3 +189,25 @@ Set the working directory of `EXECUTABLE`.
 ### `shuffle`
 
 Shuffle the test cases. Might be handy if you want to quickly test 10 random cases or so (`--shuffle --max-num-tests=10`).
+
+### `allow-extra-iterations`
+
+When choosing termination by precision, an implementation of partdiff that has been parallelized with MPI might perform more iterations than the serial reference implementation. In general, this behaviour is allowed, as long as the output (matrix and residuum) is identical to the reference implementation's output when it performs the same number of iterations.
+
+Example:
+
+When started with the parameters `1 1 0 2 1 1e-4`, the reference implementation terminates after 48 iterations with a residuum of `9.154544e-05`.
+
+With the same parameters, an `mpi-partdiff` might terminate after 50 iterations with a residuum of `6.669574e-05` (which is better).
+
+When starting the reference implementation with the parameters `1 1 0 2 2 50`, it also terminates after 50 iterations with a resiuum of `6.669574e-05`.
+
+Therefore, the `mpi-partdiff` can be considered correct.
+
+Allowed values for this parameter are:
+- `--allow-extra-iterations=0`: Do not allow extra iterations (default)
+- `--allow-extra-iterations=<n>`: Allow `n` iterations more than the serial implementation does
+- `--allow-extra-iterations=-1`: Allow an unlimited number of extra iterations
+
+> [!IMPORTANT]
+> Since `partdiff_tester` probably needs to execute the reference implementation in the described scenario, it is best to always pass `--reference-source=auto` alongside this parameter. Otherwise, the tests will likely fail.

conftest.py

@@ -25,8 +25,9 @@
 
 import pytest
 
+import output_masks
 import util
-from util import ReferenceSource, partdiff_params_tuple
+from util import PartdiffParamsTuple, ReferenceSource
 
 
 def shlex_list_str(value: str) -> list[str]:
@@ -170,6 +171,17 @@ def parse_regex_str(value: str) -> re.Pattern:
 
 
 def dir_path(value: str) -> Path:
+    """Parse a directory Path from a str.
+
+    Args:
+        value (str): The str to parse.
+
+    Raises:
+        ValueError: When the given value does not exist or is not a directory.
+
+    Returns:
+        Path: The parsed path.
+    """
     p = Path(value)
     if not p.exists():
         raise ValueError(f'Path "{value}" does not exist.')
@@ -178,6 +190,26 @@ def dir_path(value: str) -> Path:
     return p
 
 
+def extra_iterations(value: str) -> int:
+    """Parse a value for the --allow-extra-iterations parameter.
+
+    Args:
+        value (str): The value to parse.
+
+    Raises:
+        ValueError: When value doesn't contain an int between -1 and +inf
+
+    Returns:
+        int: The parsed int.
+    """
+    result = int(value)
+    if result < -1:
+        raise ValueError(
+            f'Illegal value for extra-iterations "{value}", must be -1, 0, or positive.'
+        )
+    return result
+
+
 def pytest_addoption(parser: pytest.Parser) -> None:
     """
     See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_addoption
@@ -194,7 +226,7 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Strictness of the check (default: 1)",
         type=int,
         default=1,
-        choices=range(len(util.OUTPUT_MASKS)),
+        choices=range(output_masks.NUM_OUTPUT_MASKS),
     )
     custom_options.addoption(
         "--valgrind",
@@ -217,7 +249,7 @@ def pytest_addoption(parser: pytest.Parser) -> None:
             "auto == try cache and fall back to impl)."
         ),
         type=reference_source_param,
-        default=ReferenceSource.cache,
+        default=ReferenceSource.CACHE,
         choices=ReferenceSource,
     )
     custom_options.addoption(
@@ -262,10 +294,17 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Shuffle the test cases.",
         action="store_true",
     )
+    custom_options.addoption(
+        "--allow-extra-iterations",
+        help="For term=prec, allow more iterations than the (serial) reference implementation would do (0 == disallow; n == allow n more; -1 == unlimited)",
+        metavar="n",
+        type=extra_iterations,
+        default=0,
+    )
 
 
 @pytest.fixture
-def reference_output_data() -> dict[partdiff_params_tuple, str]:
+def reference_output_data() -> dict[PartdiffParamsTuple, str]:
     """
     See util.get_reference_output_data_map()
     """
@@ -292,6 +331,11 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
             ) in itertools.product(num_threads_list, test_cases)
         ]
 
+        # Interlude: Soundness check of the partdiff parameters by trying to parse each tuple.
+        for test_case in test_cases:
+            # This throws an exception if we have incorrect test cases:
+            _ = util.PartdiffParamsClass.from_tuple(test_case)
+
         # 2. Apply the filter regexes (if desired):
         test_cases = [
             test_case
@@ -315,9 +359,12 @@ def pytest_configure(config: pytest.Config) -> None:
     """
     See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_configure
     """
+    if config.getoption("executable") is None:
+        return
+
     if config.getoption("reference_source") in (
-        ReferenceSource.auto,
-        ReferenceSource.impl,
+        ReferenceSource.AUTO,
+        ReferenceSource.IMPL,
     ):
         util.ensure_reference_implementation_exists()