diff --git a/README.md b/README.md index 309e75f..6a8f406 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ Custom options for test_partdiff: --cwd=CWD Set the working directory when launching EXECUTABLE (default: $PWD). --shuffle Shuffle the test cases. + --allow-extra-iterations=n + For term=prec, allow more iterations than the (serial) + reference implementation would do (0 == disallow; n == + allow n more; -1 == unlimited) ``` The custom options are explained below. @@ -185,3 +189,25 @@ Set the working directory of `EXECUTABLE`. ### `shuffle` Shuffle the test cases. Might be handy if you want to quickly test 10 random cases or so (`--shuffle --max-num-tests=10`). + +### `allow-extra-iterations` + +When choosing termination by precision, an implementation of partdiff that has been parallelized with MPI might perform more iterations than the serial reference implementation. In general, this behaviour is allowed, as long as the output (matrix and residuum) is identical to the reference implementation's output when it performs the same number of iterations. + +Example: + +When started with the parameters `1 1 0 2 1 1e-4`, the reference implementation terminates after 48 iterations with a residuum of `9.154544e-05`. + +With the same parameters, an `mpi-partdiff` might terminate after 50 iterations with a residuum of `6.669574e-05` (which is better). + +When starting the reference implementation with the parameters `1 1 0 2 2 50`, it also terminates after 50 iterations with a resiuum of `6.669574e-05`. + +Therefore, the `mpi-partdiff` can be considered correct. + +Allowed values for this parameter are: +- `--allow-extra-iterations=0`: Do not allow extra iterations (default) +- `--allow-extra-iterations=`: Allow `n` iterations more than the serial implementation does +- `--allow-extra-iterations=-1`: Allow an unlimited number of extra iterations + +> [!IMPORTANT] +> Since `partdiff_tester` probably needs to execute the reference implementation in the described scenario, it is best to always pass `--reference-source=auto` alongside this parameter. Otherwise, the tests will likely fail. diff --git a/conftest.py b/conftest.py index 4bf726a..f909ad4 100644 --- a/conftest.py +++ b/conftest.py @@ -25,8 +25,9 @@ import pytest +import output_masks import util -from util import ReferenceSource, partdiff_params_tuple +from util import PartdiffParamsTuple, ReferenceSource def shlex_list_str(value: str) -> list[str]: @@ -170,6 +171,17 @@ def parse_regex_str(value: str) -> re.Pattern: def dir_path(value: str) -> Path: + """Parse a directory Path from a str. + + Args: + value (str): The str to parse. + + Raises: + ValueError: When the given value does not exist or is not a directory. + + Returns: + Path: The parsed path. + """ p = Path(value) if not p.exists(): raise ValueError(f'Path "{value}" does not exist.') @@ -178,6 +190,26 @@ def dir_path(value: str) -> Path: return p +def extra_iterations(value: str) -> int: + """Parse a value for the --allow-extra-iterations parameter. + + Args: + value (str): The value to parse. + + Raises: + ValueError: When value doesn't contain an int between -1 and +inf + + Returns: + int: The parsed int. + """ + result = int(value) + if result < -1: + raise ValueError( + f'Illegal value for extra-iterations "{value}", must be -1, 0, or positive.' + ) + return result + + def pytest_addoption(parser: pytest.Parser) -> None: """ See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_addoption @@ -194,7 +226,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Strictness of the check (default: 1)", type=int, default=1, - choices=range(len(util.OUTPUT_MASKS)), + choices=range(output_masks.NUM_OUTPUT_MASKS), ) custom_options.addoption( "--valgrind", @@ -217,7 +249,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: "auto == try cache and fall back to impl)." ), type=reference_source_param, - default=ReferenceSource.cache, + default=ReferenceSource.CACHE, choices=ReferenceSource, ) custom_options.addoption( @@ -262,10 +294,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Shuffle the test cases.", action="store_true", ) + custom_options.addoption( + "--allow-extra-iterations", + help="For term=prec, allow more iterations than the (serial) reference implementation would do (0 == disallow; n == allow n more; -1 == unlimited)", + metavar="n", + type=extra_iterations, + default=0, + ) @pytest.fixture -def reference_output_data() -> dict[partdiff_params_tuple, str]: +def reference_output_data() -> dict[PartdiffParamsTuple, str]: """ See util.get_reference_output_data_map() """ @@ -292,6 +331,11 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ) in itertools.product(num_threads_list, test_cases) ] + # Interlude: Soundness check of the partdiff parameters by trying to parse each tuple. + for test_case in test_cases: + # This throws an exception if we have incorrect test cases: + _ = util.PartdiffParamsClass.from_tuple(test_case) + # 2. Apply the filter regexes (if desired): test_cases = [ test_case @@ -315,9 +359,12 @@ def pytest_configure(config: pytest.Config) -> None: """ See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_configure """ + if config.getoption("executable") is None: + return + if config.getoption("reference_source") in ( - ReferenceSource.auto, - ReferenceSource.impl, + ReferenceSource.AUTO, + ReferenceSource.IMPL, ): util.ensure_reference_implementation_exists() diff --git a/output_masks.py b/output_masks.py new file mode 100644 index 0000000..175d58c --- /dev/null +++ b/output_masks.py @@ -0,0 +1,325 @@ +"""Output masks for partdiff. + +These are essentially regexes that (have to) match the output of a partdiff implementation. + +There are output masks for different strictness levels (see --strictness). + +In most cases OUTPUT_MASKS[strictness] is used. + +When --allow-extra-iterations is passed, OUTPUT_MASKS_ALLOW_EXTRA_ITER[strictness] +and OUTPUT_MASKS_WITH_EXTRA_ITER[strictness] may be used instead in some cases. +""" + +import re + +NUM_OUTPUT_MASKS = 5 + +RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") + +F = RE_MATRIX_FLOAT.pattern + +RE_MATRIX = re.compile( + rf""" + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n + \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s* +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_0 = re.compile( + rf""" + ^ + .* + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_1 = re.compile( + rf""" + ^ + .* + ([0-9\.e+-]+) + \s* + .+: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_2 = re.compile( + rf""" + ^ + (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time + (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage + (.+): \s+ .+ \s*\n # Calculation method + (.+): \s+ ([0-9]+) \s*\n # Interlines + (.+): \s+ .+ \s*\n # Pertubation function + (.+): \s+ .+ \s*\n # Termination + (.+): \s+ ([0-9]+) \s*\n # Number of iterations + (.+): \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + .+: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_3 = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ (.+) \s*\n # Termination + Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + Matrix: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4 = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: (.+)\n" + r"Anzahl Iterationen: ([0-9]+)\n" + r"Norm des Fehlers: ([0-9\.e+-]+)\n" + r"\n" + r"Matrix:\n" + r"(" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r")$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_STRICT_0_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + .* + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_1_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + .* + [0-9\.e+-]+ + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_2_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time + (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage + (.+): \s+ .+ \s*\n # Calculation method + (.+): \s+ ([0-9]+) \s*\n # Interlines + (.+): \s+ .+ \s*\n # Pertubation function + (.+): \s+ .+ \s*\n # Termination + (.+): \s+ [0-9]+ \s*\n # Number of iterations + (.+): \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_3_ALLOW_EXTRA_ITER = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ (.+) \s*\n # Termination + Anzahl\sIterationen: \s+ [0-9]+ \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + Matrix: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4_ALLOW_EXTRA_ITER = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: (.+)\n" + r"Anzahl Iterationen: [0-9]+\n" + r"Norm des Fehlers: [0-9\.e+-]+\n" + r"\n" + r"Matrix:\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r"$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_STRICT_0_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_0 + +RE_OUTPUT_MASK_STRICT_1_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_1 + +RE_OUTPUT_MASK_STRICT_2_WITH_EXTRA_ITER = RE_OUTPUT_MASK_STRICT_2 + +RE_OUTPUT_MASK_STRICT_3_WITH_EXTRA_ITER = re.compile( + rf""" + ^ + Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) + Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) + Berechnungsmethode: \s+ (.+) \s*\n # Calculation method + Interlines: \s+ ([0-9]+) \s*\n # Interlines + Stoerfunktion: \s+ (.+) \s*\n # Pertubation function + Terminierung: \s+ .+ \s*\n # Termination + Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations + Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum + \s* + Matrix: + ({RE_MATRIX.pattern}) + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +RE_OUTPUT_MASK_STRICT_4_WITH_EXTRA_ITER = re.compile( + ( + # fmt: off + r"^" + r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" + r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" + r"Berechnungsmethode: (.+)\n" + r"Interlines: ([0-9]+)\n" + r"Stoerfunktion: (.+)\n" + r"Terminierung: .+\n" + r"Anzahl Iterationen: ([0-9]+)\n" + r"Norm des Fehlers: ([0-9\.e+-]+)\n" + r"\n" + r"Matrix:\n" + r"(" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" + r")$" + # fmt: on + ), + re.DOTALL, +) + + +RE_OUTPUT_MASK_FOR_ITERATIONS = re.compile( + rf""" + ^ + .* + .+: \s+ ([0-9]+) \s*\n # Number of iterations + .+: \s+ [0-9\.e+-]+ \s*\n # Residuum + \s* + .+: + {RE_MATRIX.pattern} + .* + $ +""", + re.VERBOSE | re.DOTALL, +) + +OUTPUT_MASKS = [ + RE_OUTPUT_MASK_STRICT_0, + RE_OUTPUT_MASK_STRICT_1, + RE_OUTPUT_MASK_STRICT_2, + RE_OUTPUT_MASK_STRICT_3, + RE_OUTPUT_MASK_STRICT_4, +] + +OUTPUT_MASKS_ALLOW_EXTRA_ITER = [ + RE_OUTPUT_MASK_STRICT_0_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_1_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_2_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_3_ALLOW_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_4_ALLOW_EXTRA_ITER, +] + +OUTPUT_MASKS_WITH_EXTRA_ITER = [ + RE_OUTPUT_MASK_STRICT_0_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_1_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_2_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_3_WITH_EXTRA_ITER, + RE_OUTPUT_MASK_STRICT_4_WITH_EXTRA_ITER, +] + +assert len(OUTPUT_MASKS) == NUM_OUTPUT_MASKS +assert len(OUTPUT_MASKS_ALLOW_EXTRA_ITER) == NUM_OUTPUT_MASKS +assert len(OUTPUT_MASKS_WITH_EXTRA_ITER) == NUM_OUTPUT_MASKS diff --git a/test_partdiff.py b/test_partdiff.py index 87b7866..de0f743 100644 --- a/test_partdiff.py +++ b/test_partdiff.py @@ -3,32 +3,65 @@ The test is parametrized by `pytest_generate_tests` in `conftest.py` via its `test_id` argument. """ +import re + import pytest import util -from util import ReferenceSource, partdiff_params_tuple +from output_masks import ( + OUTPUT_MASKS, + OUTPUT_MASKS_ALLOW_EXTRA_ITER, + OUTPUT_MASKS_WITH_EXTRA_ITER, +) +from util import PartdiffParamsTuple, TermParam + + +def check_partdiff_output( + actual_output: str, + reference_output: str, + mask: re.Pattern, +): + """Check the output of partdiff using an output mask. + + This function asserts that... + 1. The actual output matches the selected output mask + 2. The reference output matches the selected output mask + 3. For all of the output mask's capture groups, that the + captured values of actual and reference output are identical. + + Args: + actual_output (str): The output of the tested EXECUTABLE + reference_output (str): The output of the reference implementation + mask (re.Pattern): The output mask. + """ + m_actual = mask.match(actual_output) + assert m_actual is not None, (actual_output,) + m_expected = mask.match(reference_output) + assert m_expected is not None, (reference_output,) + assert len(m_expected.groups()) == len(m_actual.groups()) + for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): + assert capture_expected == capture_actual def test_partdiff_parametrized( pytestconfig: pytest.Config, - reference_output_data: dict[partdiff_params_tuple, str], + reference_output_data: dict[PartdiffParamsTuple, str], test_id: str, ) -> None: """Test if the output of a partdiff implementation matches the output of the reference implementation. Args: pytestconfig (pytest.Config): See https://docs.pytest.org/en/7.1.x/reference/reference.html#pytestconfig - reference_output_data (dict[partdiff_params_tuple, str]): The cached reference output data + reference_output_data (dict[PartdiffParamsTuple, str]): The cached reference output data test_id (str): The parameters to test as a space-separated string (not a tuple because a str prints better). """ - partdiff_params = tuple(test_id.split()) - assert len(partdiff_params) == 6 - + partdiff_params = util.params_tuple_from_str(test_id) partdiff_executable = pytestconfig.getoption("executable") strictness = pytestconfig.getoption("strictness") use_valgrind = pytestconfig.getoption("valgrind") reference_source = pytestconfig.getoption("reference_source") cwd = pytestconfig.getoption("cwd") + allow_extra_iterations = pytestconfig.getoption("allow_extra_iterations") actual_output = util.get_actual_output( partdiff_params, partdiff_executable, use_valgrind, cwd @@ -36,17 +69,41 @@ def test_partdiff_parametrized( reference_output = util.get_reference_output( partdiff_params, reference_output_data, reference_source ) - - re_output_mask = util.OUTPUT_MASKS[strictness] - - m_expected = re_output_mask.match(reference_output) - assert m_expected is not None, (reference_output,) - - m_actual = re_output_mask.match(actual_output) - assert m_actual is not None, (actual_output,) - - assert len(m_expected.groups()) == len(m_actual.groups()) - assert len(m_expected.groups()) > 0 - - for capture_expected, capture_actual in zip(m_expected.groups(), m_actual.groups()): - assert capture_expected == capture_actual + if util.PartdiffParamsClass.from_tuple(partdiff_params).term == TermParam.PREC: + if allow_extra_iterations != 0: + actual_iterations = util.parse_num_iterations_from_partdiff_output( + actual_output + ) + reference_iterations = util.parse_num_iterations_from_partdiff_output( + reference_output + ) + if allow_extra_iterations != -1: + assert ( + actual_iterations <= reference_iterations + allow_extra_iterations + ) + if actual_iterations != reference_iterations: + check_partdiff_output( + actual_output, + reference_output, + OUTPUT_MASKS_ALLOW_EXTRA_ITER[strictness], + ) + # Force termination condition "iterations" + num, method, lines, func, _term, _preciter = partdiff_params + partdiff_params = ( + num, + method, + lines, + func, + "2", + str(actual_iterations), + ) + reference_output = util.get_reference_output( + partdiff_params, reference_output_data, reference_source + ) + check_partdiff_output( + actual_output, + reference_output, + OUTPUT_MASKS_WITH_EXTRA_ITER[strictness], + ) + return + check_partdiff_output(actual_output, reference_output, OUTPUT_MASKS[strictness]) diff --git a/util.py b/util.py index b19be0a..8cb0ded 100644 --- a/util.py +++ b/util.py @@ -4,140 +4,90 @@ import re import subprocess from collections.abc import Iterator -from enum import StrEnum +from dataclasses import dataclass +from enum import Enum, StrEnum from functools import cache from pathlib import Path +from typing import Self + +import output_masks REFERENCE_IMPLEMENTATION_DIR = Path.cwd() / "reference_implementation" REFERENCE_IMPLEMENTATION_EXEC = REFERENCE_IMPLEMENTATION_DIR / "partdiff" +REFERENCE_OUTPUT_PATH = Path.cwd() / "reference_output" +TEST_CASES_FILE_PATH = Path.cwd() / "test_cases.txt" -ReferenceSource = StrEnum("ReferenceSource", ["auto", "cache", "impl"]) -RE_MATRIX_FLOAT = re.compile(r"[01]\.[0-9]{4}") +class ReferenceSource(StrEnum): + """See --reference-source""" -F = RE_MATRIX_FLOAT.pattern + AUTO = "auto" + CACHE = "cache" + IMPL = "impl" -RE_MATRIX = re.compile( - rf""" - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s*\n - \s*{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s+{F}\s* -""", - re.VERBOSE | re.DOTALL, -) -RE_OUTPUT_MASK_STRICT_0 = re.compile( - rf""" - ^ - .* - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) +PartdiffParamsTuple = tuple[str, str, str, str, str, str] -RE_OUTPUT_MASK_STRICT_1 = re.compile( - rf""" - ^ - .* - ([0-9\.e+-]+) - \s* - .+: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) -RE_OUTPUT_MASK_STRICT_2 = re.compile( - rf""" - ^ - (.+): \s+ [0-9\.]+ \s+ s \s*\n # Calculation time - (.+): \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage - (.+): \s+ .+ \s*\n # Calculation method - (.+): \s+ ([0-9]+) \s*\n # Interlines - (.+): \s+ .+ \s*\n # Pertubation function - (.+): \s+ .+ \s*\n # Termination - (.+): \s+ ([0-9]+) \s*\n # Number of iterations - (.+): \s+ ([0-9\.e+-]+) \s*\n # Residuum - \s* - .+: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) +class MethodParam(Enum): + """Enum for partdiff's method parameter""" -RE_OUTPUT_MASK_STRICT_3 = re.compile( - rf""" - ^ - Berechnungszeit: \s+ [0-9\.]+ \s+ s \s*\n # Calculation time (not captured!) - Speicherbedarf: \s+ [0-9\.]+ \s+ MiB \s*\n # Memory usage (not captured!) - Berechnungsmethode: \s+ (.+) \s*\n # Calculation method - Interlines: \s+ ([0-9]+) \s*\n # Interlines - Stoerfunktion: \s+ (.+) \s*\n # Pertubation function - Terminierung: \s+ (.+) \s*\n # Termination - Anzahl\sIterationen: \s+ ([0-9]+) \s*\n # Number of iterations - Norm\sdes\sFehlers: \s+ ([0-9\.e+-]+) \s*\n # Residuum - \s* - Matrix: - ({RE_MATRIX.pattern}) - .* - $ -""", - re.VERBOSE | re.DOTALL, -) + GAUSS_SEIDEL = 1 + JACOBI = 2 -RE_OUTPUT_MASK_STRICT_4 = re.compile( - ( - # fmt: off - r"^" - r"Berechnungszeit: [0-9]+\.[0-9]{6} s\n" - r"Speicherbedarf: [0-9]+\.[0-9]{6} MiB\n" - r"Berechnungsmethode: (.+)\n" - r"Interlines: ([0-9]+)\n" - r"Stoerfunktion: (.+)\n" - r"Terminierung: (.+)\n" - r"Anzahl Iterationen: ([0-9]+)\n" - r"Norm des Fehlers: ([0-9\.e+-]+)\n" - r"\n" - r"Matrix:\n" - r"(" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - rf" {F} {F} {F} {F} {F} {F} {F} {F} {F}\n" - r")$" - # fmt: on - ), - re.DOTALL, -) +class FuncParam(Enum): + """Enum for partdiff's func param""" -OUTPUT_MASKS = [ - RE_OUTPUT_MASK_STRICT_0, - RE_OUTPUT_MASK_STRICT_1, - RE_OUTPUT_MASK_STRICT_2, - RE_OUTPUT_MASK_STRICT_3, - RE_OUTPUT_MASK_STRICT_4, -] + FZERO = 1 # gotta go fast + FPISIN = 2 + + +class TermParam(Enum): + """Enum for partdiff's term param""" + + PREC = 1 + ITER = 2 + + +@dataclass +class PartdiffParamsClass: + """Partdiff's params in a more accessible datastructure""" + + num: int + method: MethodParam + lines: int + func: FuncParam + term: TermParam + preciter: int | float + + @classmethod + def from_tuple(cls, t: PartdiffParamsTuple) -> Self: + """Parse a PartdiffParamsClass from a PartdiffParamsTuple. + + Args: + t (PartdiffParamsTuple): The tuple to parse. + + Returns: + Self: The parsed PartdiffParamsClass. + """ + num = int(t[0]) + assert 1 <= num <= 1024 + method = MethodParam(int(t[1])) + lines = int(t[2]) + assert 0 <= lines <= 100000 + func = FuncParam(int(t[3])) + term = TermParam(int(t[4])) + preciter: int | float = -1 + if term == TermParam.ITER: + preciter = int(t[5]) + assert 1 <= preciter <= 200000 + else: + preciter = float(t[5]) + assert 1e-20 <= preciter <= 1e-4 + assert preciter != -1 + return PartdiffParamsClass(num, method, lines, func, term, preciter) -REFERENCE_OUTPUT_PATH = Path.cwd() / "reference_output" -TEST_CASES_FILE_PATH = Path.cwd() / "test_cases.txt" RE_REF_OUTPUT_FILE = re.compile( r""" @@ -155,14 +105,12 @@ re.VERBOSE | re.DOTALL, ) -partdiff_params_tuple = tuple[str, str, str, str, str, str] - -def iter_reference_output_data() -> Iterator[tuple[partdiff_params_tuple, str]]: +def iter_reference_output_data() -> Iterator[tuple[PartdiffParamsTuple, str]]: """Iterate over the reference output data. Yields: - Iterator[tuple[partdiff_params_tuple, str]]: An iterator over the partdiff params and the corresponding output + Iterator[tuple[PartdiffParamsTuple, str]]: An iterator over the partdiff params and the corresponding output of the reference implementation. """ assert REFERENCE_OUTPUT_PATH.is_dir() @@ -175,11 +123,11 @@ def iter_reference_output_data() -> Iterator[tuple[partdiff_params_tuple, str]]: yield (partdiff_params, reference_output) -def iter_test_cases() -> Iterator[partdiff_params_tuple]: +def iter_test_cases() -> Iterator[PartdiffParamsTuple]: """Iterate over the test cases. Yields: - Iterator[partdiff_params_tuple]: An iterator over the partdiff params from the test cases. + Iterator[PartdiffParamsTuple]: An iterator over the partdiff params from the test cases. """ with TEST_CASES_FILE_PATH.open() as f: for line in f: @@ -192,21 +140,21 @@ def iter_test_cases() -> Iterator[partdiff_params_tuple]: @cache -def get_test_cases() -> list[partdiff_params_tuple]: +def get_test_cases() -> list[PartdiffParamsTuple]: """Get the test cases as a list. Returns: - list[partdiff_params_tuple]: The test cases as a list of parameter tuples. + list[PartdiffParamsTuple]: The test cases as a list of parameter tuples. """ return list(iter_test_cases()) @cache -def get_reference_output_data_map() -> dict[partdiff_params_tuple, str]: +def get_reference_output_data_map() -> dict[PartdiffParamsTuple, str]: """Get the reference output as a dict. Returns: - dict[partdiff_params_tuple, str]: A dict mapping parameter combinations to the corresponding output. + dict[PartdiffParamsTuple, str]: A dict mapping parameter combinations to the corresponding output. """ return dict(iter_reference_output_data()) @@ -228,15 +176,15 @@ def is_executable(): def get_reference_output( - partdiff_params: partdiff_params_tuple, - reference_output_data: dict[partdiff_params_tuple, str], + partdiff_params: PartdiffParamsTuple, + reference_output_data: dict[PartdiffParamsTuple, str], reference_source: ReferenceSource, ) -> str: """Acquire the reference output. Args: - partdiff_params (partdiff_params_tuple): The parameter combination to get the output for. - reference_output_data (dict[partdiff_params_tuple, str]): The cached reference output. + partdiff_params (PartdiffParamsTuple): The parameter combination to get the output for. + reference_output_data (dict[PartdiffParamsTuple, str]): The cached reference output. reference_source (ReferenceSource): The source of the reference output (cache, impl, or auto). Raises: @@ -246,7 +194,8 @@ def get_reference_output( str: The reference output for the params. """ # Force the number of threads to 1: - partdiff_params = ("1",) + partdiff_params[1:6] + _num, method, lines, func, term, preciter = partdiff_params + partdiff_params = ("1", method, lines, func, term, preciter) def get_from_cache(): return reference_output_data[partdiff_params] @@ -258,24 +207,26 @@ def get_from_impl(): assert reference_source in ReferenceSource match reference_source: - case ReferenceSource.auto: + case ReferenceSource.AUTO: if partdiff_params in reference_output_data: return get_from_cache() return get_from_impl() - case ReferenceSource.cache: + case ReferenceSource.CACHE: if partdiff_params not in reference_output_data: raise RuntimeError( - 'Parameter combination "{}" was not found in cache'.format( + 'Parameter combination "{}" was not found in cache. Run with "--reference-source=auto" to fix this.'.format( " ".join(partdiff_params) ) ) return get_from_cache() - case ReferenceSource.impl: + case ReferenceSource.IMPL: return get_from_impl() + case other: + raise ValueError(f'Unexpected ReferenceSource "{other}"') def get_actual_output( - partdiff_params: partdiff_params_tuple, + partdiff_params: PartdiffParamsTuple, partdiff_executable: list[str], use_valgrind: bool, cwd: Path | None, @@ -283,7 +234,7 @@ def get_actual_output( """Get the actual output for a parameter combination. Args: - partdiff_params (partdiff_params_tuple): The parameter combination. + partdiff_params (PartdiffParamsTuple): The parameter combination. partdiff_executable (list[str]): The executable to run. use_valgrind (bool): Wether valgrind shall be used. cwd (Path | None): The working directory of the executable. @@ -310,3 +261,33 @@ def check_executable_exists(executable: list[str], cwd: Path | None) -> None: subprocess.check_output(executable, cwd=cwd) except subprocess.CalledProcessError: pass + + +def params_tuple_from_str(value: str) -> PartdiffParamsTuple: + """Parse a PartdiffParamsTuple from a space-separated str + + Args: + value (str): The str to parse + + Returns: + PartdiffParamsTuple: The parsed tuple + """ + l = value.split() + assert len(l) == 6 + num, method, lines, func, term, preciter = l + return (num, method, lines, func, term, preciter) + + +def parse_num_iterations_from_partdiff_output(output: str) -> int: + """Parse the number of iterations from partdiff's output. + + Args: + output (str): The partdiff output to parse. + + Returns: + int: The parsed iterations. + """ + m = output_masks.RE_OUTPUT_MASK_FOR_ITERATIONS.match(output) + assert m is not None + assert len(m.groups()) == 1 + return int(m.groups()[0])