From 4c958fc238e292196a63cc31c9536a879645cbcc Mon Sep 17 00:00:00 2001 From: Ruben Felgenhauer Date: Wed, 12 Nov 2025 11:30:17 +0100 Subject: [PATCH 1/4] Fix some linter complaints --- conftest.py | 21 +++++++++++---- test_partdiff.py | 10 +++----- util.py | 66 +++++++++++++++++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/conftest.py b/conftest.py index 4bf726a..7281010 100644 --- a/conftest.py +++ b/conftest.py @@ -26,7 +26,7 @@ import pytest import util -from util import ReferenceSource, partdiff_params_tuple +from util import ReferenceSource, PartdiffParamsTuple def shlex_list_str(value: str) -> list[str]: @@ -170,6 +170,17 @@ def parse_regex_str(value: str) -> re.Pattern: def dir_path(value: str) -> Path: + """Parse a directory Path from a str. + + Args: + value (str): The str to parse. + + Raises: + ValueError: When the given value does not exist or is not a directory. + + Returns: + Path: The parsed path. + """ p = Path(value) if not p.exists(): raise ValueError(f'Path "{value}" does not exist.') @@ -217,7 +228,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: "auto == try cache and fall back to impl)." ), type=reference_source_param, - default=ReferenceSource.cache, + default=ReferenceSource.CACHE, choices=ReferenceSource, ) custom_options.addoption( @@ -265,7 +276,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: @pytest.fixture -def reference_output_data() -> dict[partdiff_params_tuple, str]: +def reference_output_data() -> dict[PartdiffParamsTuple, str]: """ See util.get_reference_output_data_map() """ @@ -316,8 +327,8 @@ def pytest_configure(config: pytest.Config) -> None: See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_configure """ if config.getoption("reference_source") in ( - ReferenceSource.auto, - ReferenceSource.impl, + ReferenceSource.AUTO, + ReferenceSource.IMPL, ): util.ensure_reference_implementation_exists() diff --git a/test_partdiff.py b/test_partdiff.py index 87b7866..52d3ac0 100644 --- a/test_partdiff.py +++ b/test_partdiff.py @@ -6,24 +6,22 @@ import pytest import util -from util import ReferenceSource, partdiff_params_tuple +from util import PartdiffParamsTuple def test_partdiff_parametrized( pytestconfig: pytest.Config, - reference_output_data: dict[partdiff_params_tuple, str], + reference_output_data: dict[PartdiffParamsTuple, str], test_id: str, ) -> None: """Test if the output of a partdiff implementation matches the output of the reference implementation. Args: pytestconfig (pytest.Config): See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytestconfig - reference_output_data (dict[partdiff_params_tuple, str]): The cached reference output data + reference_output_data (dict[PartdiffParamsTuple, str]): The cached reference output data test_id (str): The parameters to test as a space-separated string (not a tuple because a str prints better). """ - partdiff_params = tuple(test_id.split()) - assert len(partdiff_params) == 6 - + partdiff_params = util.params_tuple_from_str(test_id) partdiff_executable = pytestconfig.getoption("executable") strictness = pytestconfig.getoption("strictness") use_valgrind = pytestconfig.getoption("valgrind") diff --git a/util.py b/util.py index b19be0a..1aa8313 100644 --- a/util.py +++ b/util.py @@ -11,7 +11,15 @@ REFERENCE_IMPLEMENTATION_DIR = Path.cwd() / "reference_implementation" REFERENCE_IMPLEMENTATION_EXEC = REFERENCE_IMPLEMENTATION_DIR / "partdiff" -ReferenceSource = StrEnum("ReferenceSource", ["auto", "cache", "impl"]) + +class ReferenceSource(StrEnum): + AUTO = "auto" + CACHE = "cache" + IMPL = "impl" + + +PartdiffParamsTuple = tuple[str, str, str, str, str, str] + RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") @@ -155,14 +163,12 @@ re.VERBOSE | re.DOTALL, ) -partdiff_params_tuple = tuple[str, str, str, str, str, str] - -def iter_reference_output_data() -> Iterator[tuple[partdiff_params_tuple, str]]: +def iter_reference_output_data() -> Iterator[tuple[PartdiffParamsTuple, str]]: """Iterate over the reference output data. Yields: - Iterator[tuple[partdiff_params_tuple, str]]: An iterator over the partdiff params and the corresponding output + Iterator[tuple[PartdiffParamsTuple, str]]: An iterator over the partdiff params and the corresponding output of the reference implementation. """ assert REFERENCE_OUTPUT_PATH.is_dir() @@ -175,11 +181,11 @@ def iter_reference_output_data() -> Iterator[tuple[partdiff_params_tuple, str]]: yield (partdiff_params, reference_output) -def iter_test_cases() -> Iterator[partdiff_params_tuple]: +def iter_test_cases() -> Iterator[PartdiffParamsTuple]: """Iterate over the test cases. Yields: - Iterator[partdiff_params_tuple]: An iterator over the partdiff params from the test cases. + Iterator[PartdiffParamsTuple]: An iterator over the partdiff params from the test cases. """ with TEST_CASES_FILE_PATH.open() as f: for line in f: @@ -192,21 +198,21 @@ def iter_test_cases() -> Iterator[partdiff_params_tuple]: @cache -def get_test_cases() -> list[partdiff_params_tuple]: +def get_test_cases() -> list[PartdiffParamsTuple]: """Get the test cases as a list. Returns: - list[partdiff_params_tuple]: The test cases as a list of parameter tuples. + list[PartdiffParamsTuple]: The test cases as a list of parameter tuples. """ return list(iter_test_cases()) @cache -def get_reference_output_data_map() -> dict[partdiff_params_tuple, str]: +def get_reference_output_data_map() -> dict[PartdiffParamsTuple, str]: """Get the reference output as a dict. Returns: - dict[partdiff_params_tuple, str]: A dict mapping parameter combinations to the corresponding output. + dict[PartdiffParamsTuple, str]: A dict mapping parameter combinations to the corresponding output. """ return dict(iter_reference_output_data()) @@ -228,15 +234,15 @@ def is_executable(): def get_reference_output( - partdiff_params: partdiff_params_tuple, - reference_output_data: dict[partdiff_params_tuple, str], + partdiff_params: PartdiffParamsTuple, + reference_output_data: dict[PartdiffParamsTuple, str], reference_source: ReferenceSource, ) -> str: """Acquire the reference output. Args: - partdiff_params (partdiff_params_tuple): The parameter combination to get the output for. - reference_output_data (dict[partdiff_params_tuple, str]): The cached reference output. + partdiff_params (PartdiffParamsTuple): The parameter combination to get the output for. + reference_output_data (dict[PartdiffParamsTuple, str]): The cached reference output. reference_source (ReferenceSource): The source of the reference output (cache, impl, or auto). Raises: @@ -246,7 +252,8 @@ def get_reference_output( str: The reference output for the params. """ # Force the number of threads to 1: - partdiff_params = ("1",) + partdiff_params[1:6] + _num, method, lines, func, term, preciter = partdiff_params + partdiff_params = ("1", method, lines, func, term, preciter) def get_from_cache(): return reference_output_data[partdiff_params] @@ -258,11 +265,11 @@ def get_from_impl(): assert reference_source in ReferenceSource match reference_source: - case ReferenceSource.auto: + case ReferenceSource.AUTO: if partdiff_params in reference_output_data: return get_from_cache() return get_from_impl() - case ReferenceSource.cache: + case ReferenceSource.CACHE: if partdiff_params not in reference_output_data: raise RuntimeError( 'Parameter combination "{}" was not found in cache'.format( @@ -270,12 +277,14 @@ def get_from_impl(): ) ) return get_from_cache() - case ReferenceSource.impl: + case ReferenceSource.IMPL: return get_from_impl() + case other: + raise ValueError(f'Unexpected ReferenceSource "{other}"') def get_actual_output( - partdiff_params: partdiff_params_tuple, + partdiff_params: PartdiffParamsTuple, partdiff_executable: list[str], use_valgrind: bool, cwd: Path | None, @@ -283,7 +292,7 @@ def get_actual_output( """Get the actual output for a parameter combination. Args: - partdiff_params (partdiff_params_tuple): The parameter combination. + partdiff_params (PartdiffParamsTuple): The parameter combination. partdiff_executable (list[str]): The executable to run. use_valgrind (bool): Wether valgrind shall be used. cwd (Path | None): The working directory of the executable. @@ -310,3 +319,18 @@ def check_executable_exists(executable: list[str], cwd: Path | None) -> None: subprocess.check_output(executable, cwd=cwd) except subprocess.CalledProcessError: pass + + +def params_tuple_from_str(value: str) -> PartdiffParamsTuple: + """Parse a PartdiffParamsTuple from a space-separated str + + Args: + value (str): The str to parse + + Returns: + PartdiffParamsTuple: The parsed tuple + """ + l = value.split() + assert len(l) == 6 + num, method, lines, func, term, preciter = l + return (num, method, lines, func, term, preciter) From ac48821242f661d6f5053677213c73d4f3c031f7 Mon Sep 17 00:00:00 2001 From: Ruben Felgenhauer Date: Wed, 12 Nov 2025 11:57:23 +0100 Subject: [PATCH 2/4] Add soundness check for test cases --- conftest.py | 5 +++++ util.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 7281010..5cfb6a4 100644 --- a/conftest.py +++ b/conftest.py @@ -303,6 +303,11 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ) in itertools.product(num_threads_list, test_cases) ] + # Interlude: Soundness check of the partdiff parameters by trying to parse each tuple. + for test_case in test_cases: + # This throws an exception if we have incorrect test cases: + _ = util.PartdiffParamsClass.from_tuple(test_case) + # 2. Apply the filter regexes (if desired): test_cases = [ test_case diff --git a/util.py b/util.py index 1aa8313..dfc5c0f 100644 --- a/util.py +++ b/util.py @@ -4,9 +4,11 @@ import re import subprocess from collections.abc import Iterator -from enum import StrEnum +from enum import Enum, StrEnum from functools import cache from pathlib import Path +from dataclasses import dataclass +from typing import Self REFERENCE_IMPLEMENTATION_DIR = Path.cwd() / "reference_implementation" REFERENCE_IMPLEMENTATION_EXEC = REFERENCE_IMPLEMENTATION_DIR / "partdiff" @@ -21,6 +23,50 @@ class ReferenceSource(StrEnum): PartdiffParamsTuple = tuple[str, str, str, str, str, str] +class MethodParam(Enum): + GAUSS_SEIDEL = 1 + JACOBI = 2 + + +class FuncParam(Enum): + FZERO = 1 # gotta go fast + FPISIN = 2 + + +class TermParam(Enum): + PREC = 1 + ITER = 2 + + +@dataclass +class PartdiffParamsClass: + num: int + method: MethodParam + lines: int + func: FuncParam + term: TermParam + preciter: int | float + + @classmethod + def from_tuple(cls, t: PartdiffParamsTuple) -> Self: + num = int(t[0]) + assert 1 <= num <= 1024 + method = MethodParam(int(t[1])) + lines = int(t[2]) + assert 0 <= lines <= 100000 + func = FuncParam(int(t[3])) + term = TermParam(int(t[4])) + preciter: int | float = -1 + if term == TermParam.ITER: + preciter = int(t[5]) + assert 1 <= preciter <= 200000 + else: + preciter = float(t[5]) + assert 1e-20 <= preciter <= 1e-4 + assert preciter != -1 + return PartdiffParamsClass(num, method, lines, func, term, preciter) + + RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") F = RE_MATRIX_FLOAT.pattern From c6c24a0d9d167a74b073a2de1246deba0822366b Mon Sep 17 00:00:00 2001 From: Ruben Felgenhauer Date: Wed, 12 Nov 2025 12:14:07 +0100 Subject: [PATCH 3/4] Add case distinction for termination condition --- test_partdiff.py | 100 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/test_partdiff.py b/test_partdiff.py index 52d3ac0..5f4376f 100644 --- a/test_partdiff.py +++ b/test_partdiff.py @@ -3,10 +3,64 @@ The test is parametrized by `pytest_generate_tests` in `conftest.py` via its `test_id` argument. """ +from pathlib import Path + import pytest import util -from util import PartdiffParamsTuple +from util import PartdiffParamsTuple, ReferenceSource, TermParam + + +def _test_partdiff_term_iter( + reference_output_data: dict[PartdiffParamsTuple, str], + partdiff_params: PartdiffParamsTuple, + partdiff_executable: list[str], + strictness: int, + use_valgrind: bool, + reference_source: ReferenceSource, + cwd: Path | None, +): + actual_output = util.get_actual_output( + partdiff_params, partdiff_executable, use_valgrind, cwd + ) + reference_output = util.get_reference_output( + partdiff_params, reference_output_data, reference_source + ) + re_output_mask = util.OUTPUT_MASKS[strictness] + m_actual = re_output_mask.match(actual_output) + assert m_actual is not None, (actual_output,) + m_expected = re_output_mask.match(reference_output) + assert m_expected is not None, (reference_output,) + assert len(m_expected.groups()) == len(m_actual.groups()) + assert len(m_expected.groups()) > 0 + for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): + assert capture_expected == capture_actual + + +def _test_partdiff_term_prec( + reference_output_data: dict[PartdiffParamsTuple, str], + partdiff_params: PartdiffParamsTuple, + partdiff_executable: list[str], + strictness: int, + use_valgrind: bool, + reference_source: ReferenceSource, + cwd: Path | None, +): + actual_output = util.get_actual_output( + partdiff_params, partdiff_executable, use_valgrind, cwd + ) + reference_output = util.get_reference_output( + partdiff_params, reference_output_data, reference_source + ) + re_output_mask = util.OUTPUT_MASKS[strictness] + m_actual = re_output_mask.match(actual_output) + assert m_actual is not None, (actual_output,) + m_expected = re_output_mask.match(reference_output) + assert m_expected is not None, (reference_output,) + assert len(m_expected.groups()) == len(m_actual.groups()) + assert len(m_expected.groups()) > 0 + for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): + assert capture_expected == capture_actual def test_partdiff_parametrized( @@ -27,24 +81,26 @@ def test_partdiff_parametrized( use_valgrind = pytestconfig.getoption("valgrind") reference_source = pytestconfig.getoption("reference_source") cwd = pytestconfig.getoption("cwd") - - actual_output = util.get_actual_output( - partdiff_params, partdiff_executable, use_valgrind, cwd - ) - reference_output = util.get_reference_output( - partdiff_params, reference_output_data, reference_source - ) - - re_output_mask = util.OUTPUT_MASKS[strictness] - - m_expected = re_output_mask.match(reference_output) - assert m_expected is not None, (reference_output,) - - m_actual = re_output_mask.match(actual_output) - assert m_actual is not None, (actual_output,) - - assert len(m_expected.groups()) == len(m_actual.groups()) - assert len(m_expected.groups()) > 0 - - for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): - assert capture_expected == capture_actual + match util.PartdiffParamsClass.from_tuple(partdiff_params).term: + case TermParam.PREC: + _test_partdiff_term_prec( + reference_output_data, + partdiff_params, + partdiff_executable, + strictness, + use_valgrind, + reference_source, + cwd, + ) + case TermParam.ITER: + _test_partdiff_term_iter( + reference_output_data, + partdiff_params, + partdiff_executable, + strictness, + use_valgrind, + reference_source, + cwd, + ) + case other: + raise ValueError(f'Unexpected termination condition "{other}"') From 1f1d48c9fd5b0f140d1b44591951f17eca658a9b Mon Sep 17 00:00:00 2001 From: Ruben Felgenhauer Date: Wed, 12 Nov 2025 15:09:43 +0100 Subject: [PATCH 4/4] Add parameter --allow-extra-iterations --- README.md | 26 ++++ conftest.py | 35 ++++- output_masks.py | 325 +++++++++++++++++++++++++++++++++++++++++++++++ test_partdiff.py | 133 +++++++++---------- util.py | 167 ++++++------------------ 5 files changed, 491 insertions(+), 195 deletions(-) create mode 100644 output_masks.py diff --git a/README.md b/README.md index 309e75f..6a8f406 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ Custom options for test_partdiff: --cwd=CWD Set the working directory when launching EXECUTABLE (default: $PWD). --shuffle Shuffle the test cases. + --allow-extra-iterations=n + For term=prec, allow more iterations than the (serial) + reference implementation would do (0 == disallow; n == + allow n more; -1 == unlimited) ``` The custom options are explained below. @@ -185,3 +189,25 @@ Set the working directory of `EXECUTABLE`. ### `shuffle` Shuffle the test cases. Might be handy if you want to quickly test 10 random cases or so (`--shuffle --max-num-tests=10`). + +### `allow-extra-iterations` + +When choosing termination by precision, an implementation of partdiff that has been parallelized with MPI might perform more iterations than the serial reference implementation. In general, this behaviour is allowed, as long as the output (matrix and residuum) is identical to the reference implementation's output when it performs the same number of iterations. + +Example: + +When started with the parameters `1 1 0 2 1 1e-4`, the reference implementation terminates after 48 iterations with a residuum of `9.154544e-05`. + +With the same parameters, an `mpi-partdiff` might terminate after 50 iterations with a residuum of `6.669574e-05` (which is better). + +When starting the reference implementation with the parameters `1 1 0 2 2 50`, it also terminates after 50 iterations with a resiuum of `6.669574e-05`. + +Therefore, the `mpi-partdiff` can be considered correct. + +Allowed values for this parameter are: +- `--allow-extra-iterations=0`: Do not allow extra iterations (default) +- `--allow-extra-iterations=`: Allow `n` iterations more than the serial implementation does +- `--allow-extra-iterations=-1`: Allow an unlimited number of extra iterations + +> [!IMPORTANT] +> Since `partdiff_tester` probably needs to execute the reference implementation in the described scenario, it is best to always pass `--reference-source=auto` alongside this parameter. Otherwise, the tests will likely fail. diff --git a/conftest.py b/conftest.py index 5cfb6a4..f909ad4 100644 --- a/conftest.py +++ b/conftest.py @@ -25,8 +25,9 @@ import pytest +import output_masks import util -from util import ReferenceSource, PartdiffParamsTuple +from util import PartdiffParamsTuple, ReferenceSource def shlex_list_str(value: str) -> list[str]: @@ -189,6 +190,26 @@ def dir_path(value: str) -> Path: return p +def extra_iterations(value: str) -> int: + """Parse a value for the --allow-extra-iterations parameter. + + Args: + value (str): The value to parse. + + Raises: + ValueError: When value doesn't contain an int between -1 and +inf + + Returns: + int: The parsed int. + """ + result = int(value) + if result < -1: + raise ValueError( + f'Illegal value for extra-iterations "{value}", must be -1, 0, or positive.' + ) + return result + + def pytest_addoption(parser: pytest.Parser) -> None: """ See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_addoption @@ -205,7 +226,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Strictness of the check (default: 1)", type=int, default=1, - choices=range(len(util.OUTPUT_MASKS)), + choices=range(output_masks.NUM_OUTPUT_MASKS), ) custom_options.addoption( "--valgrind", @@ -273,6 +294,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Shuffle the test cases.", action="store_true", ) + custom_options.addoption( + "--allow-extra-iterations", + help="For term=prec, allow more iterations than the (serial) reference implementation would do (0 == disallow; n == allow n more; -1 == unlimited)", + metavar="n", + type=extra_iterations, + default=0, + ) @pytest.fixture @@ -331,6 +359,9 @@ def pytest_configure(config: pytest.Config) -> None: """ See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_configure """ + if config.getoption("executable") is None: + return + if config.getoption("reference_source") in ( ReferenceSource.AUTO, ReferenceSource.IMPL, diff --git a/output_masks.py b/output_masks.py new file mode 100644 index 0000000..175d58c --- /dev/null +++ b/output_masks.py @@ -0,0 +1,325 @@ +"""Output masks for partdiff. + +These are essentially regexes that (have to) match the output of a partdiff implementation. + +There are output masks for different strictness levels (see --strictness). + +In most cases OUTPUT_MASKS[strictness] is used. + +When --allow-extra-iterations is passed, OUTPUT_MASKS_ALLOW_EXTRA_ITER[strictness] +and OUTPUT_MASKS_WITH_EXTRA_ITER[strictness] may be used instead in some cases. +""" + +import re + +NUM_OUTPUT_MASKS = 5 + +RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") + +F = RE_MATRIX_FLOAT.pattern + +RE_MATRIX = re.compile( + rf""" + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s* +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_0 = re.compile( + rf""" + ^ + .* + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_1 = re.compile( + rf""" + ^ + .* + ([0-9\.e+-]+) + \s* + .+: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_2 = re.compile( + rf""" + ^ + (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time + (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage + (.+): \s+ .+ \s*\n # Calculation method + (.+): \s+ ([0-9]+) \s*\n # Interlines + (.+): \s+ .+ \s*\n # Pertubation function + (.+): \s+ .+ \s*\n # Termination + (.+): \s+ ([0-9]+) \s*\n # Number of iterations + (.+): \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + .+: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_3 = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ (.+) \s*\n # Termination + Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + Matrix: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4 = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: (.+)\n" + r"Anzahl Iterationen: ([0-9]+)\n" + r"Norm des Fehlers: ([0-9\.e+-]+)\n" + r"\n" + r"Matrix:\n" + r"(" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r")$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_STRICT_0_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + .* + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_1_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + .* + [0-9\.e+-]+ + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_2_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time + (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage + (.+): \s+ .+ \s*\n # Calculation method + (.+): \s+ ([0-9]+) \s*\n # Interlines + (.+): \s+ .+ \s*\n # Pertubation function + (.+): \s+ .+ \s*\n # Termination + (.+): \s+ [0-9]+ \s*\n # Number of iterations + (.+): \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_3_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ (.+) \s*\n # Termination + Anzahl\sIterationen: \s+ [0-9]+ \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + Matrix: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4_ALLOW_EXTRA_ITER = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: (.+)\n" + r"Anzahl Iterationen: [0-9]+\n" + r"Norm des Fehlers: [0-9\.e+-]+\n" + r"\n" + r"Matrix:\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r"$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_STRICT_0_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_0 + +RE_OUTPUT_MASK_STRICT_1_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_1 + +RE_OUTPUT_MASK_STRICT_2_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_2 + +RE_OUTPUT_MASK_STRICT_3_WITH_EXTRA_ITER = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ .+ \s*\n # Termination + Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + Matrix: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4_WITH_EXTRA_ITER = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: .+\n" + r"Anzahl Iterationen: ([0-9]+)\n" + r"Norm des Fehlers: ([0-9\.e+-]+)\n" + r"\n" + r"Matrix:\n" + r"(" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r")$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_FOR_ITERATIONS = re.compile( + rf""" + ^ + .* + .+: \s+ ([0-9]+) \s*\n # Number of iterations + .+: \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +OUTPUT_MASKS = [ + RE_OUTPUT_MASK_STRICT_0, + RE_OUTPUT_MASK_STRICT_1, + RE_OUTPUT_MASK_STRICT_2, + RE_OUTPUT_MASK_STRICT_3, + RE_OUTPUT_MASK_STRICT_4, +] + +OUTPUT_MASKS_ALLOW_EXTRA_ITER = [ + RE_OUTPUT_MASK_STRICT_0_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_1_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_2_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_3_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_4_ALLOW_EXTRA_ITER, +] + +OUTPUT_MASKS_WITH_EXTRA_ITER = [ + RE_OUTPUT_MASK_STRICT_0_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_1_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_2_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_3_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_4_WITH_EXTRA_ITER, +] + +assert len(OUTPUT_MASKS) == NUM_OUTPUT_MASKS +assert len(OUTPUT_MASKS_ALLOW_EXTRA_ITER) == NUM_OUTPUT_MASKS +assert len(OUTPUT_MASKS_WITH_EXTRA_ITER) == NUM_OUTPUT_MASKS diff --git a/test_partdiff.py b/test_partdiff.py index 5f4376f..de0f743 100644 --- a/test_partdiff.py +++ b/test_partdiff.py @@ -3,62 +3,42 @@ The test is parametrized by `pytest_generate_tests` in `conftest.py` via its `test_id` argument. """ -from pathlib import Path +import re import pytest import util -from util import PartdiffParamsTuple, ReferenceSource, TermParam +from output_masks import ( + OUTPUT_MASKS, + OUTPUT_MASKS_ALLOW_EXTRA_ITER, + OUTPUT_MASKS_WITH_EXTRA_ITER, +) +from util import PartdiffParamsTuple, TermParam -def _test_partdiff_term_iter( - reference_output_data: dict[PartdiffParamsTuple, str], - partdiff_params: PartdiffParamsTuple, - partdiff_executable: list[str], - strictness: int, - use_valgrind: bool, - reference_source: ReferenceSource, - cwd: Path | None, +def check_partdiff_output( + actual_output: str, + reference_output: str, + mask: re.Pattern, ): - actual_output = util.get_actual_output( - partdiff_params, partdiff_executable, use_valgrind, cwd - ) - reference_output = util.get_reference_output( - partdiff_params, reference_output_data, reference_source - ) - re_output_mask = util.OUTPUT_MASKS[strictness] - m_actual = re_output_mask.match(actual_output) - assert m_actual is not None, (actual_output,) - m_expected = re_output_mask.match(reference_output) - assert m_expected is not None, (reference_output,) - assert len(m_expected.groups()) == len(m_actual.groups()) - assert len(m_expected.groups()) > 0 - for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): - assert capture_expected == capture_actual + """Check the output of partdiff using an output mask. + This function asserts that... + 1. The actual output matches the selected output mask + 2. The reference output matches the selected output mask + 3. For all of the output mask's capture groups, that the + captured values of actual and reference output are identical. -def _test_partdiff_term_prec( - reference_output_data: dict[PartdiffParamsTuple, str], - partdiff_params: PartdiffParamsTuple, - partdiff_executable: list[str], - strictness: int, - use_valgrind: bool, - reference_source: ReferenceSource, - cwd: Path | None, -): - actual_output = util.get_actual_output( - partdiff_params, partdiff_executable, use_valgrind, cwd - ) - reference_output = util.get_reference_output( - partdiff_params, reference_output_data, reference_source - ) - re_output_mask = util.OUTPUT_MASKS[strictness] - m_actual = re_output_mask.match(actual_output) + Args: + actual_output (str): The output of the tested EXECUTABLE + reference_output (str): The output of the reference implementation + mask (re.Pattern): The output mask. + """ + m_actual = mask.match(actual_output) assert m_actual is not None, (actual_output,) - m_expected = re_output_mask.match(reference_output) + m_expected = mask.match(reference_output) assert m_expected is not None, (reference_output,) assert len(m_expected.groups()) == len(m_actual.groups()) - assert len(m_expected.groups()) > 0 for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): assert capture_expected == capture_actual @@ -81,26 +61,49 @@ def test_partdiff_parametrized( use_valgrind = pytestconfig.getoption("valgrind") reference_source = pytestconfig.getoption("reference_source") cwd = pytestconfig.getoption("cwd") - match util.PartdiffParamsClass.from_tuple(partdiff_params).term: - case TermParam.PREC: - _test_partdiff_term_prec( - reference_output_data, - partdiff_params, - partdiff_executable, - strictness, - use_valgrind, - reference_source, - cwd, + allow_extra_iterations = pytestconfig.getoption("allow_extra_iterations") + + actual_output = util.get_actual_output( + partdiff_params, partdiff_executable, use_valgrind, cwd + ) + reference_output = util.get_reference_output( + partdiff_params, reference_output_data, reference_source + ) + if util.PartdiffParamsClass.from_tuple(partdiff_params).term == TermParam.PREC: + if allow_extra_iterations != 0: + actual_iterations = util.parse_num_iterations_from_partdiff_output( + actual_output ) - case TermParam.ITER: - _test_partdiff_term_iter( - reference_output_data, - partdiff_params, - partdiff_executable, - strictness, - use_valgrind, - reference_source, - cwd, + reference_iterations = util.parse_num_iterations_from_partdiff_output( + reference_output ) - case other: - raise ValueError(f'Unexpected termination condition "{other}"') + if allow_extra_iterations != -1: + assert ( + actual_iterations <= reference_iterations + allow_extra_iterations + ) + if actual_iterations != reference_iterations: + check_partdiff_output( + actual_output, + reference_output, + OUTPUT_MASKS_ALLOW_EXTRA_ITER[strictness], + ) + # Force termination condition "iterations" + num, method, lines, func, _term, _preciter = partdiff_params + partdiff_params = ( + num, + method, + lines, + func, + "2", + str(actual_iterations), + ) + reference_output = util.get_reference_output( + partdiff_params, reference_output_data, reference_source + ) + check_partdiff_output( + actual_output, + reference_output, + OUTPUT_MASKS_WITH_EXTRA_ITER[strictness], + ) + return + check_partdiff_output(actual_output, reference_output, OUTPUT_MASKS[strictness]) diff --git a/util.py b/util.py index dfc5c0f..8cb0ded 100644 --- a/util.py +++ b/util.py @@ -4,17 +4,23 @@ import re import subprocess from collections.abc import Iterator +from dataclasses import dataclass from enum import Enum, StrEnum from functools import cache from pathlib import Path -from dataclasses import dataclass from typing import Self +import output_masks + REFERENCE_IMPLEMENTATION_DIR = Path.cwd() / "reference_implementation" REFERENCE_IMPLEMENTATION_EXEC = REFERENCE_IMPLEMENTATION_DIR / "partdiff" +REFERENCE_OUTPUT_PATH = Path.cwd() / "reference_output" +TEST_CASES_FILE_PATH = Path.cwd() / "test_cases.txt" class ReferenceSource(StrEnum): + """See --reference-source""" + AUTO = "auto" CACHE = "cache" IMPL = "impl" @@ -24,22 +30,30 @@ class ReferenceSource(StrEnum): class MethodParam(Enum): + """Enum for partdiff's method parameter""" + GAUSS_SEIDEL = 1 JACOBI = 2 class FuncParam(Enum): + """Enum for partdiff's func param""" + FZERO = 1 # gotta go fast FPISIN = 2 class TermParam(Enum): + """Enum for partdiff's term param""" + PREC = 1 ITER = 2 @dataclass class PartdiffParamsClass: + """Partdiff's params in a more accessible datastructure""" + num: int method: MethodParam lines: int @@ -49,6 +63,14 @@ class PartdiffParamsClass: @classmethod def from_tuple(cls, t: PartdiffParamsTuple) -> Self: + """Parse a PartdiffParamsClass from a PartdiffParamsTuple. + + Args: + t (PartdiffParamsTuple): The tuple to parse. + + Returns: + Self: The parsed PartdiffParamsClass. + """ num = int(t[0]) assert 1 <= num <= 1024 method = MethodParam(int(t[1])) @@ -67,132 +89,6 @@ def from_tuple(cls, t: PartdiffParamsTuple) -> Self: return PartdiffParamsClass(num, method, lines, func, term, preciter) -RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") - -F = RE_MATRIX_FLOAT.pattern - -RE_MATRIX = re.compile( - rf""" - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s* -""", - re.VERBOSE | re.DOTALL, -) - -RE_OUTPUT_MASK_STRICT_0 = re.compile( - rf""" - ^ - .* - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) - -RE_OUTPUT_MASK_STRICT_1 = re.compile( - rf""" - ^ - .* - ([0-9\.e+-]+) - \s* - .+: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) - -RE_OUTPUT_MASK_STRICT_2 = re.compile( - rf""" - ^ - (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time - (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage - (.+): \s+ .+ \s*\n # Calculation method - (.+): \s+ ([0-9]+) \s*\n # Interlines - (.+): \s+ .+ \s*\n # Pertubation function - (.+): \s+ .+ \s*\n # Termination - (.+): \s+ ([0-9]+) \s*\n # Number of iterations - (.+): \s+ ([0-9\.e+-]+) \s*\n # Residuum - \s* - .+: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) - -RE_OUTPUT_MASK_STRICT_3 = re.compile( - rf""" - ^ - Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) - Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) - Berechnungsmethode: \s+ (.+) \s*\n # Calculation method - Interlines: \s+ ([0-9]+) \s*\n # Interlines - Stoerfunktion: \s+ (.+) \s*\n # Pertubation function - Terminierung: \s+ (.+) \s*\n # Termination - Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations - Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum - \s* - Matrix: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) - -RE_OUTPUT_MASK_STRICT_4 = re.compile( - ( - # fmt: off - r"^" - r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" - r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" - r"Berechnungsmethode: (.+)\n" - r"Interlines: ([0-9]+)\n" - r"Stoerfunktion: (.+)\n" - r"Terminierung: (.+)\n" - r"Anzahl Iterationen: ([0-9]+)\n" - r"Norm des Fehlers: ([0-9\.e+-]+)\n" - r"\n" - r"Matrix:\n" - r"(" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - r")$" - # fmt: on - ), - re.DOTALL, -) - - -OUTPUT_MASKS = [ - RE_OUTPUT_MASK_STRICT_0, - RE_OUTPUT_MASK_STRICT_1, - RE_OUTPUT_MASK_STRICT_2, - RE_OUTPUT_MASK_STRICT_3, - RE_OUTPUT_MASK_STRICT_4, -] - -REFERENCE_OUTPUT_PATH = Path.cwd() / "reference_output" -TEST_CASES_FILE_PATH = Path.cwd() / "test_cases.txt" - RE_REF_OUTPUT_FILE = re.compile( r""" ^ @@ -318,7 +214,7 @@ def get_from_impl(): case ReferenceSource.CACHE: if partdiff_params not in reference_output_data: raise RuntimeError( - 'Parameter combination "{}" was not found in cache'.format( + 'Parameter combination "{}" was not found in cache. Run with "--reference-source=auto" to fix this.'.format( " ".join(partdiff_params) ) ) @@ -380,3 +276,18 @@ def params_tuple_from_str(value: str) -> PartdiffParamsTuple: assert len(l) == 6 num, method, lines, func, term, preciter = l return (num, method, lines, func, term, preciter) + + +def parse_num_iterations_from_partdiff_output(output: str) -> int: + """Parse the number of iterations from partdiff's output. + + Args: + output (str): The partdiff output to parse. + + Returns: + int: The parsed iterations. + """ + m = output_masks.RE_OUTPUT_MASK_FOR_ITERATIONS.match(output) + assert m is not None + assert len(m.groups()) == 1 + return int(m.groups()[0])