From 3da3fa464dcbce8e7bf9940be33d86644208d80d Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 12:40:46 -0700 Subject: [PATCH 01/15] Add benchmarks for incremetnally slicing a file --- src/nwb_benchmarks/__init__.py | 7 ++ src/nwb_benchmarks/benchmarks/params.py | 51 ++++++++++++++ src/nwb_benchmarks/database/_models.py | 2 + src/nwb_benchmarks/setup/_reduce_results.py | 76 ++++++++++++++------- 4 files changed, 113 insertions(+), 23 deletions(-) diff --git a/src/nwb_benchmarks/__init__.py b/src/nwb_benchmarks/__init__.py index 320fad36..779a81cb 100644 --- a/src/nwb_benchmarks/__init__.py +++ b/src/nwb_benchmarks/__init__.py @@ -10,6 +10,7 @@ TSHARK_PATH = os.environ.get("TSHARK_PATH", None) NETWORK_INTERFACE = os.environ.get("NWB_BENCHMARKS_NETWORK_INTERFACE", None) RUN_DOWNLOAD_BENCHMARKS = os.environ.get("RUN_DOWNLOAD_BENCHMARKS", None) +RUN_INCREMENTAL_SLICING_BENCHMARKS = os.environ.get("RUN_INCREMENTAL_SLICING_BENCHMARKS", None) if TSHARK_PATH is None: TSHARK_PATH = shutil.which("tshark") @@ -26,9 +27,15 @@ "RUN_DOWNLOAD_BENCHMARKS is set. Benchmarks that download the entire test file will be run, which may take a long time." ) +if RUN_INCREMENTAL_SLICING_BENCHMARKS: + warnings.warn( + "RUN_INCREMENTAL_SLICING_BENCHMARKS is set. Incremental slicing benchmarks will be run, which may take a long time." + ) + __all__ = [ "main", "TSHARK_PATH", "NETWORK_INTERFACE", "RUN_DOWNLOAD_BENCHMARKS", + "RUN_INCREMENTAL_SLICING_BENCHMARKS", ] diff --git a/src/nwb_benchmarks/benchmarks/params.py b/src/nwb_benchmarks/benchmarks/params.py index 26223700..e6668aaf 100644 --- a/src/nwb_benchmarks/benchmarks/params.py +++ b/src/nwb_benchmarks/benchmarks/params.py @@ -191,6 +191,57 @@ # icephys data has shape (N,) and chunk shape (8192,) icephys_slices = [(slice(0, 8192 * i),) for i in range(1, 6)] +#################################### INCREMENTAL SLICE PARAMETERS ################################### + +incremental_hdf5_ecephys_params = dict( + name="EcephysIncrementalSliceTestCase", + object_name="ElectricalSeries", + slice_template=ecephys_slices[0], + slice_strategy="iterate_time_axis", +) +incremental_hdf5_ophys_params = dict( + name="OphysIncrementalSliceTestCase", + object_name="TwoPhotonSeries", + slice_template=ophys_slices[0], + slice_strategy="iterate_time_axis", +) +incremental_hdf5_icephys_params = dict( + name="IcephysIncrementalSliceTestCase", + slice_strategy="iterate_icephys_timeseries", +) + +hdf5_redirected_read_incremental_slice_params = ( + dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_redirected"]), + dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_redirected"]), + dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_redirected"]), +) + +hdf5_no_redirect_download_incremental_slice_params = ( + dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_no_redirect"]), +) + +zarr_direct_read_incremental_slice_params = ( + dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_direct"]), + dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_direct"]), + dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_direct"]), +) + +zarr_no_redirect_download_incremental_slice_params = ( + dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_no_redirect"]), +) + +lindi_no_redirect_download_incremental_slice_params = ( + dict(**incremental_hdf5_ecephys_params, https_url=lindi_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=lindi_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=lindi_icephys_params["https_url_no_redirect"]), +) + +#################################### REMOTE SLICE PARAMETERS ################################### + hdf5_redirected_read_slice_params = [] for index, slice_range in enumerate(ecephys_slices): hdf5_redirected_read_slice_params.append( diff --git a/src/nwb_benchmarks/database/_models.py b/src/nwb_benchmarks/database/_models.py index 3f1560f7..0144150a 100644 --- a/src/nwb_benchmarks/database/_models.py +++ b/src/nwb_benchmarks/database/_models.py @@ -117,6 +117,8 @@ def to_dataframe(self) -> "polars.DataFrame": "parameter_case_https_url": [result.parameter_case.get("https_url") for result in self.results], "parameter_case_object_name": [result.parameter_case.get("object_name") for result in self.results], "parameter_case_slice_range": [result.parameter_case.get("slice_range") for result in self.results], + "parameter_case_slice_template": [result.parameter_case.get("slice_template") for result in self.results], + "parameter_case_slice_strategy": [result.parameter_case.get("slice_strategy") for result in self.results], "value": [result.value for result in self.results], "variable": [result.variable for result in self.results], } diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index b0aae4f0..27f63ba9 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -15,6 +15,51 @@ from ..utils import get_dictionary_checksum +def _extract_successful_results(test_case: str, raw_results_list: list, result_columns: list[str] | None) -> dict: + """Extract successful ASV benchmark results keyed by serialized parameter case. + + ASV has used multiple result serialization layouts. Older result files store a + fixed-length list where parameters are at index 1 and samples are at index 11. + Newer result files include a top-level ``result_columns`` list and each result + row follows that column order. This helper normalizes both layouts to the + reduced results format used by nwb_benchmarks. + """ + + # Older ASV layout used by the original reducer implementation. + if len(raw_results_list) == 12: + serialized_params = raw_results_list[1][0] + raw_results = raw_results_list[11] + elif result_columns is not None: + column_index = {column_name: index for index, column_name in enumerate(result_columns)} + if "result" not in column_index or "params" not in column_index: + return {} + + serialized_params = raw_results_list[column_index["params"]] + raw_results = raw_results_list[column_index["result"]] + else: + return {} + + # ASV represents one benchmark parameter axis as [[param0, param1, ...]]. + # The targeted subclasses used for incremental slicing often contain just + # one parameter case, e.g. [[param0]]. + assert len(serialized_params) == 1, "Unexpected length of serialized parameters list!" + serialized_params = serialized_params[0] + + # Skipped results in JSON are written as `null` and read back into Python as `None`. + non_skipped_results = [result for result in raw_results if result is not None] + if len(serialized_params) != len(non_skipped_results): + message = ( + f"In intermediate results for test case {test_case}: \n" + f"\tLength mismatch between parameters ({len(serialized_params)}) and " + f"result samples ({len(non_skipped_results)})!\n\n" + "Please raise an issue and share your intermediate results file." + ) + warnings.warn(message=message) + return {} + + return {params: raw_result for params, raw_result in zip(serialized_params, non_skipped_results)} + + def _parse_environment_info(raw_environment_info: List[str]) -> Dict[str, List[Dict[str, str]]]: """Turn the results of `conda list` printout to a JSON dictionary.""" header_stripped = raw_environment_info[3:] @@ -42,30 +87,15 @@ def reduce_results(machine_id: str, raw_results_file_path: pathlib.Path, raw_env timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") reduced_results = dict() + result_columns = raw_results_info.get("result_columns") for test_case, raw_results_list in raw_results_info["results"].items(): - - # Only successful runs have a results field of length 12 - if len(raw_results_list) != 12: - continue - - # This code assumes that test cases are only run with one parameter - assert len(raw_results_list[1]) == 1, "Unexpected length of serialized parameters list!" - serialized_params = raw_results_list[1][0] - - # Skipped results in JSON are writen as `null` and read back into Python as `None` - non_skipped_results = [result for result in raw_results_list[11] if result is not None] - if len(serialized_params) != len(non_skipped_results): - message = ( - f"In intermediate results for test case {test_case}: \n" - f"\tLength mismatch between parameters ({len(serialized_params)}) and " - f"result samples ({len(non_skipped_results)})!\n\n" - "Please raise an issue and share your intermediate results file." - ) - warnings.warn(message=message) - else: - reduced_results.update( - {test_case: {params: raw_result for params, raw_result in zip(serialized_params, non_skipped_results)}} - ) + extracted_results = _extract_successful_results( + test_case=test_case, + raw_results_list=raw_results_list, + result_columns=result_columns, + ) + if extracted_results: + reduced_results.update({test_case: extracted_results}) if len(reduced_results) == 0: raise ValueError( From c441814d678c18e0250b6ff0c4b17b92a9ec3a10 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 13:08:18 -0700 Subject: [PATCH 02/15] Document evironment variables --- docs/running_benchmarks.rst | 92 +++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index d9367431..8f432b86 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -52,6 +52,98 @@ Or drop the ``sudo`` if on Windows. Many of the current tests can take several minutes to complete; the entire suite will take many times that. Grab some coffee, read a book, or better yet (when the suite becomes larger) just leave it to run overnight. +Environment Variables +--------------------- + +The benchmark suite uses a few optional environment variables to configure network tracking and to opt in to +long-running benchmark families that are disabled by default. + +``NWB_BENCHMARKS_NETWORK_INTERFACE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Network tracking benchmarks monitor traffic on a specific network interface. Set ``NWB_BENCHMARKS_NETWORK_INTERFACE`` +to the interface that should be monitored, for example ``en0`` on macOS or ``eth0`` on many Linux systems: + +.. code-block:: + + export NWB_BENCHMARKS_NETWORK_INTERFACE=en0 + +On Windows PowerShell, use: + +.. code-block:: + + $env:NWB_BENCHMARKS_NETWORK_INTERFACE="Ethernet" + +If this variable is not set, the package will warn when ``tshark`` is available. Network tracking results may be +missing or invalid until a suitable interface is configured. + +``TSHARK_PATH`` +~~~~~~~~~~~~~~~ + +Network tracking benchmarks require the ``tshark`` executable. If ``tshark`` is installed on your ``PATH``, no extra +configuration is needed. Otherwise, set ``TSHARK_PATH`` to the absolute path to the executable: + +.. code-block:: + + export TSHARK_PATH=/path/to/tshark + +On Windows PowerShell, use a path to ``tshark.exe``: + +.. code-block:: + + $env:TSHARK_PATH="C:\Program Files\Wireshark\tshark.exe" + +If ``tshark`` cannot be found, network tracking benchmarks are skipped. + +``RUN_DOWNLOAD_BENCHMARKS`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Benchmarks that download entire remote test files are disabled by default because they can take a long time and consume +substantial bandwidth and disk space. Set ``RUN_DOWNLOAD_BENCHMARKS`` to any non-empty value to include them: + +.. code-block:: + + export RUN_DOWNLOAD_BENCHMARKS=true + +``RUN_INCREMENTAL_SLICING_BENCHMARKS`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Incremental slicing benchmarks are disabled by default because they can be long-running. Set +``RUN_INCREMENTAL_SLICING_BENCHMARKS`` to opt in. The value can either enable all incremental slicing benchmarks or +select specific data modalities. + +Accepted disabled values are unset, empty, ``false``, ``0``, ``no``, or ``off``. + +Accepted values to run all incremental slicing modalities are ``true``, ``1``, ``yes``, ``on``, or ``all``: + +.. code-block:: + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=true + +Accepted modality names are ``ecephys``, ``ophys``, and ``icephys``. To run only one modality: + +.. code-block:: + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys + +To run multiple modalities, provide a comma-separated list: + +.. code-block:: + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys,ophys + +Invalid values raise an error during package import so that misspellings do not accidentally run the wrong set of +benchmarks. + +Example: running only icephys incremental slicing benchmarks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys + nwb_benchmarks run --bench track_incremental_slicing --debug + + Additional Flags ---------------- From 7b7e6760f768aba390accb6a2f5c61ce8b50115b Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 13:10:02 -0700 Subject: [PATCH 03/15] Make it easy to run the incremental slice benchmark for individual modalitities via an environment variable --- src/nwb_benchmarks/__init__.py | 43 +++++++++++++++- src/nwb_benchmarks/benchmarks/params.py | 67 +++++++++++++++++-------- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/nwb_benchmarks/__init__.py b/src/nwb_benchmarks/__init__.py index 779a81cb..0bc03d28 100644 --- a/src/nwb_benchmarks/__init__.py +++ b/src/nwb_benchmarks/__init__.py @@ -10,7 +10,37 @@ TSHARK_PATH = os.environ.get("TSHARK_PATH", None) NETWORK_INTERFACE = os.environ.get("NWB_BENCHMARKS_NETWORK_INTERFACE", None) RUN_DOWNLOAD_BENCHMARKS = os.environ.get("RUN_DOWNLOAD_BENCHMARKS", None) -RUN_INCREMENTAL_SLICING_BENCHMARKS = os.environ.get("RUN_INCREMENTAL_SLICING_BENCHMARKS", None) +RUN_INCREMENTAL_SLICING_BENCHMARKS_RAW = os.environ.get("RUN_INCREMENTAL_SLICING_BENCHMARKS", None) + + +def _parse_incremental_slicing_benchmarks_config(value: str | None) -> tuple[bool, set[str] | None]: + """Parse RUN_INCREMENTAL_SLICING_BENCHMARKS into enabled state and selected modalities.""" + if value is None: + return False, None + + normalized_value = value.strip().lower() + if normalized_value in {"", "false", "0", "no", "off"}: + return False, None + + if normalized_value in {"true", "1", "yes", "on", "all"}: + return True, None + + selected_modalities = {modality.strip().lower() for modality in normalized_value.split(",") if modality.strip()} + valid_modalities = {"ecephys", "ophys", "icephys"} + invalid_modalities = selected_modalities - valid_modalities + if invalid_modalities: + raise ValueError( + f"Invalid RUN_INCREMENTAL_SLICING_BENCHMARKS value {value!r}. " + "Expected true, all, ecephys, ophys, icephys, or a comma-separated list of modalities." + ) + + return True, selected_modalities + + +( + RUN_INCREMENTAL_SLICING_BENCHMARKS, + RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES, +) = _parse_incremental_slicing_benchmarks_config(RUN_INCREMENTAL_SLICING_BENCHMARKS_RAW) if TSHARK_PATH is None: TSHARK_PATH = shutil.which("tshark") @@ -28,8 +58,16 @@ ) if RUN_INCREMENTAL_SLICING_BENCHMARKS: + if RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES is None: + incremental_slicing_benchmark_description = "all incremental slicing benchmarks" + else: + incremental_slicing_benchmark_description = ( + "incremental slicing benchmarks for modalities " + f"{', '.join(sorted(RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES))}" + ) warnings.warn( - "RUN_INCREMENTAL_SLICING_BENCHMARKS is set. Incremental slicing benchmarks will be run, which may take a long time." + "RUN_INCREMENTAL_SLICING_BENCHMARKS is set. " + f"{incremental_slicing_benchmark_description} will be run, which may take a long time." ) __all__ = [ @@ -38,4 +76,5 @@ "NETWORK_INTERFACE", "RUN_DOWNLOAD_BENCHMARKS", "RUN_INCREMENTAL_SLICING_BENCHMARKS", + "RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES", ] diff --git a/src/nwb_benchmarks/benchmarks/params.py b/src/nwb_benchmarks/benchmarks/params.py index e6668aaf..78270068 100644 --- a/src/nwb_benchmarks/benchmarks/params.py +++ b/src/nwb_benchmarks/benchmarks/params.py @@ -1,3 +1,4 @@ +from nwb_benchmarks import RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES from nwb_benchmarks.core import get_https_url ################################### BASE PARAMETERS ################################### @@ -193,51 +194,77 @@ #################################### INCREMENTAL SLICE PARAMETERS ################################### + +def filter_incremental_slice_params_by_modality(params: tuple[dict, ...]) -> tuple[dict, ...]: + """Filter incremental slicing benchmark params by RUN_INCREMENTAL_SLICING_BENCHMARKS modality selection.""" + if RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES is None: + return params + + return tuple( + parameter_set + for parameter_set in params + if parameter_set.get("modality") in RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES + ) + + incremental_hdf5_ecephys_params = dict( name="EcephysIncrementalSliceTestCase", + modality="ecephys", object_name="ElectricalSeries", slice_template=ecephys_slices[0], slice_strategy="iterate_time_axis", ) incremental_hdf5_ophys_params = dict( name="OphysIncrementalSliceTestCase", + modality="ophys", object_name="TwoPhotonSeries", slice_template=ophys_slices[0], slice_strategy="iterate_time_axis", ) incremental_hdf5_icephys_params = dict( name="IcephysIncrementalSliceTestCase", + modality="icephys", slice_strategy="iterate_icephys_timeseries", ) -hdf5_redirected_read_incremental_slice_params = ( - dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_redirected"]), - dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_redirected"]), - dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_redirected"]), +hdf5_redirected_read_incremental_slice_params = filter_incremental_slice_params_by_modality( + ( + dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_redirected"]), + dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_redirected"]), + dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_redirected"]), + ) ) -hdf5_no_redirect_download_incremental_slice_params = ( - dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_no_redirect"]), +hdf5_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( + ( + dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_no_redirect"]), + ) ) -zarr_direct_read_incremental_slice_params = ( - dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_direct"]), - dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_direct"]), - dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_direct"]), +zarr_direct_read_incremental_slice_params = filter_incremental_slice_params_by_modality( + ( + dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_direct"]), + dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_direct"]), + dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_direct"]), + ) ) -zarr_no_redirect_download_incremental_slice_params = ( - dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_no_redirect"]), +zarr_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( + ( + dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_no_redirect"]), + ) ) -lindi_no_redirect_download_incremental_slice_params = ( - dict(**incremental_hdf5_ecephys_params, https_url=lindi_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=lindi_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=lindi_icephys_params["https_url_no_redirect"]), +lindi_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( + ( + dict(**incremental_hdf5_ecephys_params, https_url=lindi_ecephys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_ophys_params, https_url=lindi_ophys_params["https_url_no_redirect"]), + dict(**incremental_hdf5_icephys_params, https_url=lindi_icephys_params["https_url_no_redirect"]), + ) ) #################################### REMOTE SLICE PARAMETERS ################################### From da1cacc6071347235da9a2277bc9dc7897038123 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 13:10:33 -0700 Subject: [PATCH 04/15] Fix but in result parser --- src/nwb_benchmarks/setup/_reduce_results.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 27f63ba9..446e131e 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -46,18 +46,22 @@ def _extract_successful_results(test_case: str, raw_results_list: list, result_c serialized_params = serialized_params[0] # Skipped results in JSON are written as `null` and read back into Python as `None`. - non_skipped_results = [result for result in raw_results if result is not None] - if len(serialized_params) != len(non_skipped_results): + if raw_results is None: + return {} + if not isinstance(raw_results, list): + raw_results = [raw_results] + + if len(serialized_params) != len(raw_results): message = ( f"In intermediate results for test case {test_case}: \n" f"\tLength mismatch between parameters ({len(serialized_params)}) and " - f"result samples ({len(non_skipped_results)})!\n\n" + f"result samples ({len(raw_results)})!\n\n" "Please raise an issue and share your intermediate results file." ) warnings.warn(message=message) return {} - return {params: raw_result for params, raw_result in zip(serialized_params, non_skipped_results)} + return {params: raw_result for params, raw_result in zip(serialized_params, raw_results) if raw_result is not None} def _parse_environment_info(raw_environment_info: List[str]) -> Dict[str, List[Dict[str, str]]]: From 3d67b5292265f6c5f4e7cf5f68a00558c02b509c Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 13:30:59 -0700 Subject: [PATCH 05/15] Add missing /track_incremental_slicing.py file --- .../benchmarks/track_incremental_slicing.py | 217 ++++++++++++++++++ src/nwb_benchmarks/setup/_reduce_results.py | 48 +++- 2 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 src/nwb_benchmarks/benchmarks/track_incremental_slicing.py diff --git a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py new file mode 100644 index 00000000..bfc9b4fa --- /dev/null +++ b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py @@ -0,0 +1,217 @@ +"""Benchmarks for cumulative open + repeated slice access patterns. + +These benchmarks explicitly measure the use case that is extrapolated in the download-vs-streaming plots: +open a file, read one slice, read the next slice, read the next slice, and so on. Each benchmark returns a dictionary +of cumulative timings, where ``cumulative_slice_000`` is open time only and ``cumulative_slice_NNN`` includes opening +the file and reading the first N slices/items. + +The benchmarks are opt-in and are skipped unless RUN_INCREMENTAL_SLICING_BENCHMARKS is set in the environment. +""" + +import shutil +import time +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import Any, Tuple + +from asv_runner.benchmarks.mark import SkipNotImplemented, skip_benchmark_if +from pynwb import NWBHDF5IO + +from nwb_benchmarks import RUN_INCREMENTAL_SLICING_BENCHMARKS +from nwb_benchmarks.core import ( + BaseBenchmark, + download_asset_if_not_exists, + get_asset_path_from_url, + get_object_by_name, + read_hdf5_pynwb_fsspec_https_with_cache, + read_hdf5_pynwb_lindi, + read_hdf5_pynwb_remfile_with_cache, + read_hdf5_pynwb_ros3, + read_zarr_pynwb_https, + read_zarr_pynwb_s3, +) +from nwb_benchmarks.setup import get_persistent_download_directory + +from .params import ( + hdf5_no_redirect_download_incremental_slice_params, + hdf5_redirected_read_incremental_slice_params, + lindi_no_redirect_download_incremental_slice_params, + zarr_direct_read_incremental_slice_params, + zarr_no_redirect_download_incremental_slice_params, +) + + +class IncrementalSliceBenchmark(BaseBenchmark, ABC): + """Base class for cumulative open + repeated slice benchmarks.""" + + # Ensure a full incremental sequence is only performed once per ASV measurement. + number = 1 + + # Some full-axis incremental reads may be long-running. + timeout = 60 * 60 * 12 + + @abstractmethod + def _open_file(self, params: dict[str, Any]) -> None: + """Open the file and assign ``self.nwbfile`` plus any closeable resources.""" + pass + + def teardown(self, params: dict[str, Any]) -> None: + if hasattr(self, "io"): + self.io.close() + if hasattr(self, "file"): + self.file.close() + if hasattr(self, "bytestream"): + self.bytestream.close() + if hasattr(self, "client"): + self.client.close() + if hasattr(self, "tmpdir"): + shutil.rmtree(path=self.tmpdir.name, ignore_errors=True) + self.tmpdir.cleanup() + + @staticmethod + def _iter_time_axis_slices(data: Any, slice_template: Tuple[slice, ...]) -> Iterable[Tuple[slice, ...]]: + """Yield adjacent slices along axis 0 using the first slice range as a template.""" + if not hasattr(data, "shape") or len(data.shape) == 0: + raise ValueError("Expected array-like data with at least one dimension for time-axis slicing.") + + time_axis_length = data.shape[0] + first_axis_template = slice_template[0] + chunk_size = first_axis_template.stop - (first_axis_template.start or 0) + if chunk_size <= 0: + raise ValueError(f"Invalid time-axis slice template: {slice_template}") + + trailing_slices = slice_template[1:] + for start in range(0, time_axis_length, chunk_size): + stop = min(start + chunk_size, time_axis_length) + yield (slice(start, stop), *trailing_slices) + + @staticmethod + def _iter_icephys_timeseries(nwbfile: Any) -> Iterable[tuple[str, Any]]: + """Yield all TimeSeries-like objects from acquisition and stimulus/presentation with readable data.""" + containers = [("acquisition", nwbfile.acquisition), ("stimulus", nwbfile.stimulus)] + for container_name, container in containers: + for object_name, neurodata_object in container.items(): + data = getattr(neurodata_object, "data", None) + if data is None: + continue + if not hasattr(data, "__getitem__"): + continue + yield f"{container_name}/{object_name}", data + + def _track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, float]: + """Track cumulative open + slice timings for the configured slice strategy.""" + timings = {} + start_time = time.perf_counter() + + self._open_file(params=params) + timings["cumulative_slice_000"] = time.perf_counter() - start_time + + slice_strategy = params["slice_strategy"] + if slice_strategy == "iterate_time_axis": + object_name = params["object_name"] + self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) + self.data_to_slice = self.neurodata_object.data + + for slice_index, slice_range in enumerate( + self._iter_time_axis_slices(data=self.data_to_slice, slice_template=params["slice_template"]), start=1 + ): + self._temp = self.data_to_slice[slice_range] + timings[f"cumulative_slice_{slice_index:03d}"] = time.perf_counter() - start_time + elif slice_strategy == "iterate_icephys_timeseries": + for slice_index, (_, data) in enumerate(self._iter_icephys_timeseries(nwbfile=self.nwbfile), start=1): + self._temp = data[:] + timings[f"cumulative_slice_{slice_index:03d}"] = time.perf_counter() - start_time + else: + raise ValueError(f"Unsupported incremental slice strategy: {slice_strategy}") + + return timings + + @skip_benchmark_if(not RUN_INCREMENTAL_SLICING_BENCHMARKS) + def track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, float]: + """Track cumulative open + repeated slice timing.""" + return self._track_cumulative_slice_times(params=params) + + +class HDF5PyNWBRemfileWithCacheIncrementalSliceBenchmark(IncrementalSliceBenchmark): + """Track cumulative slicing of remote HDF5 NWB files using PyNWB and remfile with cache.""" + + params = hdf5_redirected_read_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io, self.file, self.bytestream, self.tmpdir = read_hdf5_pynwb_remfile_with_cache( + https_url=params["https_url"] + ) + + +class HDF5PyNWBFsspecHttpsWithCacheIncrementalSliceBenchmark(IncrementalSliceBenchmark): + """Track cumulative slicing of remote HDF5 NWB files using PyNWB and fsspec HTTPS with cache.""" + + params = hdf5_redirected_read_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io, self.file, self.bytestream, self.tmpdir = read_hdf5_pynwb_fsspec_https_with_cache( + https_url=params["https_url"] + ) + + +class HDF5PyNWBROS3IncrementalSliceBenchmark(IncrementalSliceBenchmark): + """Track cumulative slicing of remote HDF5 NWB files using PyNWB and ROS3.""" + + params = hdf5_redirected_read_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io, _ = read_hdf5_pynwb_ros3(https_url=params["https_url"]) + + +class HDF5PyNWBLindiIncrementalSliceBenchmark(IncrementalSliceBenchmark): + """Track cumulative slicing of HDF5 NWB files using PyNWB and a local LINDI reference file.""" + + params = lindi_no_redirect_download_incremental_slice_params + + def setup(self, params: dict[str, Any]) -> None: + # Downloading the small LINDI JSON is not part of the tracked open + slice timing. + self.lindi_file = download_asset_if_not_exists(https_url=params["https_url"]) + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io, self.client = read_hdf5_pynwb_lindi(rfs=self.lindi_file) + + +class ZarrPyNWBS3IncrementalSliceBenchmark(IncrementalSliceBenchmark): + """Track cumulative slicing of remote Zarr NWB files using PyNWB with S3 and consolidated metadata.""" + + params = zarr_direct_read_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io = read_zarr_pynwb_s3(https_url=params["https_url"], mode="r") + + +class LocalIncrementalSliceBenchmark(IncrementalSliceBenchmark, ABC): + """Base class for local cumulative open + repeated slice benchmarks.""" + + def setup(self, params: dict[str, Any]) -> None: + self.download_dir = get_persistent_download_directory() + self.file_name = get_asset_path_from_url(https_url=params["https_url"]) + self.file_path = self.download_dir / self.file_name + if not self.file_path.exists(): + raise SkipNotImplemented( + f"Expected file {self.file_path} to exist for local incremental slicing benchmark." + ) + + +class HDF5PyNWBLocalIncrementalSliceBenchmark(LocalIncrementalSliceBenchmark): + """Track cumulative slicing of local HDF5 NWB files using PyNWB.""" + + params = hdf5_no_redirect_download_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.io = NWBHDF5IO(str(self.file_path), mode="r") + self.nwbfile = self.io.read() + + +class ZarrPyNWBLocalIncrementalSliceBenchmark(LocalIncrementalSliceBenchmark): + """Track cumulative slicing of local Zarr NWB files using PyNWB with consolidated metadata.""" + + params = zarr_no_redirect_download_incremental_slice_params + + def _open_file(self, params: dict[str, Any]) -> None: + self.nwbfile, self.io = read_zarr_pynwb_https(https_url=self.file_path, mode="r") diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 446e131e..347a2378 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -3,6 +3,7 @@ import collections import datetime import hashlib +import itertools import json import pathlib import shutil @@ -15,6 +16,38 @@ from ..utils import get_dictionary_checksum +def _serialize_parameter_cases(serialized_params: list) -> list[str]: + """Return one serialized parameter-case key per ASV result. + + ASV stores parameters as a list of parameter axes. Preserve the historical + single-axis keys used by nwb_benchmarks results, while supporting ASV's + general zero-axis and multi-axis parameter layouts. + """ + + if len(serialized_params) == 0: + return ["()"] + if len(serialized_params) == 1: + return serialized_params[0] + + return [str(parameter_case) for parameter_case in itertools.product(*serialized_params)] + + +def _flatten_parameter_grid(values: list, number_of_parameter_axes: int) -> list: + """Flatten ASV result/sample grids to one value per parameter case.""" + + if number_of_parameter_axes == 0: + return [values] + if number_of_parameter_axes == 1: + return values + + flattened_values = [] + for value in values: + flattened_values.extend( + _flatten_parameter_grid(values=value, number_of_parameter_axes=number_of_parameter_axes - 1) + ) + return flattened_values + + def _extract_successful_results(test_case: str, raw_results_list: list, result_columns: list[str] | None) -> dict: """Extract successful ASV benchmark results keyed by serialized parameter case. @@ -27,7 +60,7 @@ def _extract_successful_results(test_case: str, raw_results_list: list, result_c # Older ASV layout used by the original reducer implementation. if len(raw_results_list) == 12: - serialized_params = raw_results_list[1][0] + serialized_params = raw_results_list[1] raw_results = raw_results_list[11] elif result_columns is not None: column_index = {column_name: index for index, column_name in enumerate(result_columns)} @@ -36,20 +69,23 @@ def _extract_successful_results(test_case: str, raw_results_list: list, result_c serialized_params = raw_results_list[column_index["params"]] raw_results = raw_results_list[column_index["result"]] + if "samples" in column_index and raw_results_list[column_index["samples"]] is not None: + raw_results = raw_results_list[column_index["samples"]] else: return {} - # ASV represents one benchmark parameter axis as [[param0, param1, ...]]. - # The targeted subclasses used for incremental slicing often contain just - # one parameter case, e.g. [[param0]]. - assert len(serialized_params) == 1, "Unexpected length of serialized parameters list!" - serialized_params = serialized_params[0] + number_of_parameter_axes = len(serialized_params) + serialized_params = _serialize_parameter_cases(serialized_params=serialized_params) # Skipped results in JSON are written as `null` and read back into Python as `None`. if raw_results is None: return {} if not isinstance(raw_results, list): raw_results = [raw_results] + raw_results = _flatten_parameter_grid( + values=raw_results, + number_of_parameter_axes=number_of_parameter_axes, + ) if len(serialized_params) != len(raw_results): message = ( From b119932ea6cfed81f58acbf73f010f48a9cbe4dc Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 15:56:15 -0700 Subject: [PATCH 06/15] Address review comments --- src/nwb_benchmarks/database/_models.py | 3 ++ src/nwb_benchmarks/setup/_reduce_results.py | 45 +++++++++++---------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/nwb_benchmarks/database/_models.py b/src/nwb_benchmarks/database/_models.py index 0144150a..f44830c6 100644 --- a/src/nwb_benchmarks/database/_models.py +++ b/src/nwb_benchmarks/database/_models.py @@ -36,6 +36,9 @@ def process_network_results(benchmark_results: dict) -> dict: """Add additional network metrics.""" results = benchmark_results.copy() + if "total_traffic_in_number_of_web_packets" not in results: + return results + if results["total_traffic_in_number_of_web_packets"] != 0: results["mean_time_per_web_packet"] = ( results["total_transfer_time_in_seconds"] / results["total_traffic_in_number_of_web_packets"] diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 347a2378..4294becf 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -32,20 +32,26 @@ def _serialize_parameter_cases(serialized_params: list) -> list[str]: return [str(parameter_case) for parameter_case in itertools.product(*serialized_params)] -def _flatten_parameter_grid(values: list, number_of_parameter_axes: int) -> list: - """Flatten ASV result/sample grids to one value per parameter case.""" +def _extract_parameter_case_values(values: list, serialized_params: list) -> list: + """Return one ASV result/sample value per serialized parameter case. - if number_of_parameter_axes == 0: + ASV stores aggregate results and recorded samples in grids matching the + parameter axes. The leaf value may itself be a list of recorded samples, so + the grid must be traversed by parameter-axis indices instead of recursively + flattening every list. + """ + + if len(serialized_params) == 0: return [values] - if number_of_parameter_axes == 1: - return values - flattened_values = [] - for value in values: - flattened_values.extend( - _flatten_parameter_grid(values=value, number_of_parameter_axes=number_of_parameter_axes - 1) - ) - return flattened_values + axis_index_ranges = [range(len(parameter_axis)) for parameter_axis in serialized_params] + extracted_values = [] + for parameter_case_indices in itertools.product(*axis_index_ranges): + parameter_case_value = values + for index in parameter_case_indices: + parameter_case_value = parameter_case_value[index] + extracted_values.append(parameter_case_value) + return extracted_values def _extract_successful_results(test_case: str, raw_results_list: list, result_columns: list[str] | None) -> dict: @@ -69,23 +75,20 @@ def _extract_successful_results(test_case: str, raw_results_list: list, result_c serialized_params = raw_results_list[column_index["params"]] raw_results = raw_results_list[column_index["result"]] - if "samples" in column_index and raw_results_list[column_index["samples"]] is not None: - raw_results = raw_results_list[column_index["samples"]] + if "samples" in column_index: + samples_index = column_index["samples"] + if samples_index < len(raw_results_list) and raw_results_list[samples_index] is not None: + raw_results = raw_results_list[samples_index] else: return {} - number_of_parameter_axes = len(serialized_params) - serialized_params = _serialize_parameter_cases(serialized_params=serialized_params) - # Skipped results in JSON are written as `null` and read back into Python as `None`. if raw_results is None: return {} - if not isinstance(raw_results, list): + if len(serialized_params) > 0 and not isinstance(raw_results, list): raw_results = [raw_results] - raw_results = _flatten_parameter_grid( - values=raw_results, - number_of_parameter_axes=number_of_parameter_axes, - ) + raw_results = _extract_parameter_case_values(values=raw_results, serialized_params=serialized_params) + serialized_params = _serialize_parameter_cases(serialized_params=serialized_params) if len(serialized_params) != len(raw_results): message = ( From 585822c021509de07f4bfd857537a33da7d14b61 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Tue, 6 Oct 2026 16:12:23 -0700 Subject: [PATCH 07/15] Address copilot feedback --- src/nwb_benchmarks/database/_models.py | 2 ++ src/nwb_benchmarks/setup/_reduce_results.py | 23 --------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/nwb_benchmarks/database/_models.py b/src/nwb_benchmarks/database/_models.py index f44830c6..3bce5fc7 100644 --- a/src/nwb_benchmarks/database/_models.py +++ b/src/nwb_benchmarks/database/_models.py @@ -64,6 +64,8 @@ def parse_parameter_case(s): # if the parsed string is not a dict (older benchmarks results), convert it to one if not isinstance(output, dict): + if output == (): + return {} output = {"https_url": output[0].strip("'")} return output diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index 4294becf..93f57cb1 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -32,28 +32,6 @@ def _serialize_parameter_cases(serialized_params: list) -> list[str]: return [str(parameter_case) for parameter_case in itertools.product(*serialized_params)] -def _extract_parameter_case_values(values: list, serialized_params: list) -> list: - """Return one ASV result/sample value per serialized parameter case. - - ASV stores aggregate results and recorded samples in grids matching the - parameter axes. The leaf value may itself be a list of recorded samples, so - the grid must be traversed by parameter-axis indices instead of recursively - flattening every list. - """ - - if len(serialized_params) == 0: - return [values] - - axis_index_ranges = [range(len(parameter_axis)) for parameter_axis in serialized_params] - extracted_values = [] - for parameter_case_indices in itertools.product(*axis_index_ranges): - parameter_case_value = values - for index in parameter_case_indices: - parameter_case_value = parameter_case_value[index] - extracted_values.append(parameter_case_value) - return extracted_values - - def _extract_successful_results(test_case: str, raw_results_list: list, result_columns: list[str] | None) -> dict: """Extract successful ASV benchmark results keyed by serialized parameter case. @@ -87,7 +65,6 @@ def _extract_successful_results(test_case: str, raw_results_list: list, result_c return {} if len(serialized_params) > 0 and not isinstance(raw_results, list): raw_results = [raw_results] - raw_results = _extract_parameter_case_values(values=raw_results, serialized_params=serialized_params) serialized_params = _serialize_parameter_cases(serialized_params=serialized_params) if len(serialized_params) != len(raw_results): From ee63956cbea63dd199a1df70a818126c42418eb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 04:29:35 +0000 Subject: [PATCH 08/15] Group platform-specific env var snippets into synced tabs Use sphinx-tabs group tabs (as in the nwb2bids docs) so each environment variable example shows a macOS / Linux and a Windows (PowerShell) tab; selecting a platform once switches every tab group on the page. Also adds PowerShell equivalents for the RUN_* examples and fixes a heading underline that was one character short. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- docs/conf.py | 1 + docs/requirements-rtd.txt | 1 + docs/running_benchmarks.rst | 103 ++++++++++++++++++++++++++++-------- 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a86e6b45..b799b7c7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,7 @@ "sphinx_search.extension", # Allows for auto search function the documentation "sphinx.ext.viewcode", # Shows source code in the documentation "sphinx.ext.extlinks", # Allows to use shorter external links defined in the extlinks variable. + "sphinx_tabs.tabs", # Groups platform-specific code blocks into synchronized tabs ] templates_path = ["_templates"] diff --git a/docs/requirements-rtd.txt b/docs/requirements-rtd.txt index e6132a7b..cc3b9041 100644 --- a/docs/requirements-rtd.txt +++ b/docs/requirements-rtd.txt @@ -3,3 +3,4 @@ sphinx==5.1.1 pydata-sphinx-theme==0.14.1 readthedocs-sphinx-search==0.3.2 sphinx-copybutton==0.5.0 +sphinx-tabs==3.4.5 diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 3455abf0..91158856 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -64,15 +64,19 @@ long-running benchmark families that are disabled by default. Network tracking benchmarks monitor traffic on a specific network interface. Set ``NWB_BENCHMARKS_NETWORK_INTERFACE`` to the interface that should be monitored, for example ``en0`` on macOS or ``eth0`` on many Linux systems: -.. code-block:: +.. tabs:: - export NWB_BENCHMARKS_NETWORK_INTERFACE=en0 + .. group-tab:: macOS / Linux -On Windows PowerShell, use: + .. code-block:: bash -.. code-block:: + export NWB_BENCHMARKS_NETWORK_INTERFACE=en0 - $env:NWB_BENCHMARKS_NETWORK_INTERFACE="Ethernet" + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell + + $env:NWB_BENCHMARKS_NETWORK_INTERFACE="Ethernet" If this variable is not set, the package will warn when ``tshark`` is available. Network tracking results may be missing or invalid until a suitable interface is configured. @@ -83,15 +87,19 @@ missing or invalid until a suitable interface is configured. Network tracking benchmarks require the ``tshark`` executable. If ``tshark`` is installed on your ``PATH``, no extra configuration is needed. Otherwise, set ``TSHARK_PATH`` to the absolute path to the executable: -.. code-block:: +.. tabs:: - export TSHARK_PATH=/path/to/tshark + .. group-tab:: macOS / Linux -On Windows PowerShell, use a path to ``tshark.exe``: + .. code-block:: bash -.. code-block:: + export TSHARK_PATH=/path/to/tshark + + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell - $env:TSHARK_PATH="C:\Program Files\Wireshark\tshark.exe" + $env:TSHARK_PATH="C:\Program Files\Wireshark\tshark.exe" If ``tshark`` cannot be found, network tracking benchmarks are skipped. @@ -101,9 +109,19 @@ If ``tshark`` cannot be found, network tracking benchmarks are skipped. Benchmarks that download entire remote test files are disabled by default because they can take a long time and consume substantial bandwidth and disk space. Set ``RUN_DOWNLOAD_BENCHMARKS`` to any non-empty value to include them: -.. code-block:: +.. tabs:: + + .. group-tab:: macOS / Linux + + .. code-block:: bash + + export RUN_DOWNLOAD_BENCHMARKS=true + + .. group-tab:: Windows (PowerShell) - export RUN_DOWNLOAD_BENCHMARKS=true + .. code-block:: powershell + + $env:RUN_DOWNLOAD_BENCHMARKS="true" ``RUN_INCREMENTAL_SLICING_BENCHMARKS`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -116,32 +134,73 @@ Accepted disabled values are unset, empty, ``false``, ``0``, ``no``, or ``off``. Accepted values to run all incremental slicing modalities are ``true``, ``1``, ``yes``, ``on``, or ``all``: -.. code-block:: +.. tabs:: + + .. group-tab:: macOS / Linux + + .. code-block:: bash + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=true - export RUN_INCREMENTAL_SLICING_BENCHMARKS=true + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell + + $env:RUN_INCREMENTAL_SLICING_BENCHMARKS="true" Accepted modality names are ``ecephys``, ``ophys``, and ``icephys``. To run only one modality: -.. code-block:: +.. tabs:: + + .. group-tab:: macOS / Linux + + .. code-block:: bash - export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys + + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell + + $env:RUN_INCREMENTAL_SLICING_BENCHMARKS="icephys" To run multiple modalities, provide a comma-separated list: -.. code-block:: +.. tabs:: + + .. group-tab:: macOS / Linux - export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys,ophys + .. code-block:: bash + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys,ophys + + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell + + $env:RUN_INCREMENTAL_SLICING_BENCHMARKS="icephys,ophys" Invalid values raise an error during package import so that misspellings do not accidentally run the wrong set of benchmarks. Example: running only icephys incremental slicing benchmarks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: +.. tabs:: + + .. group-tab:: macOS / Linux + + .. code-block:: bash + + export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys + nwb_benchmarks run --bench track_incremental_slicing --debug + + .. group-tab:: Windows (PowerShell) + + .. code-block:: powershell - export RUN_INCREMENTAL_SLICING_BENCHMARKS=icephys - nwb_benchmarks run --bench track_incremental_slicing --debug + $env:RUN_INCREMENTAL_SLICING_BENCHMARKS="icephys" + nwb_benchmarks run --bench track_incremental_slicing --debug Additional Flags From 3b6af3bd62c5b4acda862a515d32c17175a6b03e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 04:41:44 +0000 Subject: [PATCH 09/15] Bump package version to 0.2.0 The incremental slicing benchmarks are a new benchmark family, which AGENTS.md treats as a minor bump. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 02e7d6e3..feb70dec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ packages = ["src/nwb_benchmarks"] [project] name = "nwb_benchmarks" -version="0.1.2" +version="0.2.0" authors = [ { name="Cody Baker", email="cody.baker@catalystneuro.com" }, ] From 2a2e5d93a2c455454650e07ebe7a7e0ebaa9a10b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 05:04:50 +0000 Subject: [PATCH 10/15] Record incremental slicing timings as a single list of cumulative times `track_cumulative_slice_times` now returns `{"cumulative_time_in_seconds": [...]}` instead of one `cumulative_slice_NNN` key per step. The keys stopped sorting correctly past 999 steps (zero-padded to 3 digits), and each one became its own variable in the database. The list keeps the steps in read order: the first entry is the open time and entry N includes reading the first N slices. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- .../benchmarks/track_incremental_slicing.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py index bfc9b4fa..0b1db36d 100644 --- a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py +++ b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py @@ -1,9 +1,9 @@ """Benchmarks for cumulative open + repeated slice access patterns. These benchmarks explicitly measure the use case that is extrapolated in the download-vs-streaming plots: -open a file, read one slice, read the next slice, read the next slice, and so on. Each benchmark returns a dictionary -of cumulative timings, where ``cumulative_slice_000`` is open time only and ``cumulative_slice_NNN`` includes opening -the file and reading the first N slices/items. +open a file, read one slice, read the next slice, read the next slice, and so on. Each benchmark returns +``{"cumulative_time_in_seconds": [...]}``, a single list of cumulative timings in read order: the first entry is the +open time only and entry ``N`` includes opening the file and reading the first ``N`` slices/items. The benchmarks are opt-in and are skipped unless RUN_INCREMENTAL_SLICING_BENCHMARKS is set in the environment. """ @@ -98,13 +98,12 @@ def _iter_icephys_timeseries(nwbfile: Any) -> Iterable[tuple[str, Any]]: continue yield f"{container_name}/{object_name}", data - def _track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, float]: + def _track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, list[float]]: """Track cumulative open + slice timings for the configured slice strategy.""" - timings = {} start_time = time.perf_counter() self._open_file(params=params) - timings["cumulative_slice_000"] = time.perf_counter() - start_time + cumulative_times = [time.perf_counter() - start_time] slice_strategy = params["slice_strategy"] if slice_strategy == "iterate_time_axis": @@ -112,22 +111,22 @@ def _track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, flo self.neurodata_object = get_object_by_name(nwbfile=self.nwbfile, object_name=object_name) self.data_to_slice = self.neurodata_object.data - for slice_index, slice_range in enumerate( - self._iter_time_axis_slices(data=self.data_to_slice, slice_template=params["slice_template"]), start=1 + for slice_range in self._iter_time_axis_slices( + data=self.data_to_slice, slice_template=params["slice_template"] ): self._temp = self.data_to_slice[slice_range] - timings[f"cumulative_slice_{slice_index:03d}"] = time.perf_counter() - start_time + cumulative_times.append(time.perf_counter() - start_time) elif slice_strategy == "iterate_icephys_timeseries": - for slice_index, (_, data) in enumerate(self._iter_icephys_timeseries(nwbfile=self.nwbfile), start=1): + for _, data in self._iter_icephys_timeseries(nwbfile=self.nwbfile): self._temp = data[:] - timings[f"cumulative_slice_{slice_index:03d}"] = time.perf_counter() - start_time + cumulative_times.append(time.perf_counter() - start_time) else: raise ValueError(f"Unsupported incremental slice strategy: {slice_strategy}") - return timings + return {"cumulative_time_in_seconds": cumulative_times} @skip_benchmark_if(not RUN_INCREMENTAL_SLICING_BENCHMARKS) - def track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, float]: + def track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, list[float]]: """Track cumulative open + repeated slice timing.""" return self._track_cumulative_slice_times(params=params) From d03707eae24cc03f7f89f46c9e758918aa5cc0a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 05:04:50 +0000 Subject: [PATCH 11/15] Add unit tests and run them in CI - `RUN_INCREMENTAL_SLICING_BENCHMARKS` parsing and the modality filter of the incremental slicing parameters. - The incremental slicing helpers and `_track_cumulative_slice_times` on an in-memory NWB file, for both slice strategies. - `_extract_successful_results`, `_serialize_parameter_cases` and `reduce_results` on a raw results file written by asv 0.6.1 with `--record-samples`, holding an incremental, a network and a time benchmark, including a failed and an unselected parameter set; and the database reader on the reduced output. The DANDI URL lookup that importing the benchmark parameters does is stubbed in `conftest.py`, so the tests run offline. A `test` dependency group installs pytest with the database and figure dependencies, and the CI runs the tests before the benchmark smoke test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- .github/workflows/testing.yml | 8 ++ docs/development.rst | 14 ++ pyproject.toml | 4 +- tests/conftest.py | 7 + tests/data/asv_0.6.1_raw_results.json | 120 +++++++++++++++++ tests/test_incremental_slicing_config.py | 32 +++++ tests/test_reduce_results.py | 164 +++++++++++++++++++++++ tests/test_track_incremental_slicing.py | 119 ++++++++++++++++ 8 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py create mode 100644 tests/data/asv_0.6.1_raw_results.json create mode 100644 tests/test_incremental_slicing_config.py create mode 100644 tests/test_reduce_results.py create mode 100644 tests/test_track_incremental_slicing.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 042cdd86..5313c848 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -65,6 +65,14 @@ jobs: working-directory: nwb_benchmarks run: python -m nwb_benchmarks.scripts.check_dandi_urls + # `--group` installs a dependency group from `pyproject.toml` and needs pip 25.1 or newer. + - name: Run unit tests + working-directory: nwb_benchmarks + run: | + python -m pip install --upgrade "pip>=25.1" + python -m pip install --group test + python -m pytest tests + # This run is a smoke test of the benchmark machinery, not a measurement, so each benchmark is restricted to a # single parameter set. ASV matches the `--bench` pattern against `module.Class.method({parameters})`, which # lets the pattern pick one parametrization. Every parameter set opens the full remote NWB file in `setup`, diff --git a/docs/development.rst b/docs/development.rst index a6154c46..5a684fc9 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -29,6 +29,20 @@ Writing thorough docstrings is encouraged; please follow the Numpy style. Import and submodule structure follows ``scikit-learn`` standard. +Unit Tests +---------- + +The ``tests`` directory holds unit tests for the code around the benchmarks, such as the reduction of ASV results and +the helpers of the incremental slicing benchmarks. They run offline and do not run any benchmark. To run them... + +.. code-block:: + + pip install -e . --group test + pytest tests + +The CI runs them on every pull request, before the benchmark smoke test. + + Customized Machine Header ------------------------- diff --git a/pyproject.toml b/pyproject.toml index feb70dec..f6a78c75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,10 +58,12 @@ nwb_benchmarks = "nwb_benchmarks.command_line_interface:main" database = ["polars"] dev = ["ipython", "pre-commit"] figures = ["seaborn", "matplotlib", "pyarrow"] +test = ["pytest", {include-group = "database"}, {include-group = "figures"}] all = [ {include-group = "database"}, {include-group = "dev"}, - {include-group = "figures"} + {include-group = "figures"}, + {include-group = "test"} ] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..17a9ee9e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +import nwb_benchmarks.core + +# `nwb_benchmarks.benchmarks.params` resolves the redirected DANDI URLs through the DANDI API when it is imported. Stub +# the lookup before any test imports the benchmark suite, so the unit tests run offline and do not depend on the API. +nwb_benchmarks.core.get_https_url = lambda dandiset_id, dandi_path, follow_redirects=1: ( + f"https://example.invalid/{dandiset_id}/{dandi_path}" +) diff --git a/tests/data/asv_0.6.1_raw_results.json b/tests/data/asv_0.6.1_raw_results.json new file mode 100644 index 00000000..b4987d9c --- /dev/null +++ b/tests/data/asv_0.6.1_raw_results.json @@ -0,0 +1,120 @@ +{ + "commit_hash": "0123456789abcdef0123456789abcdef01234567", + "result_columns": [ + "result", + "params", + "version", + "started_at", + "duration", + "stats_ci_99_a", + "stats_ci_99_b", + "stats_q_25", + "stats_q_75", + "stats_number", + "stats_repeat", + "samples", + "profile" + ], + "results": { + "bench.Incremental.track_cumulative_slice_times": [ + [ + { + "cumulative_time_in_seconds": [ + 0.5, + 1.0, + 1.5 + ] + }, + null + ], + [ + [ + "{'name': 'A'}", + "{'name': 'B'}" + ] + ], + "b3e7fa9dfc40a9240d5f4dc9d6e3e552a4fbbe90b8b7e12ee43c6aff1c810eef", + 1791349309228, + 0.023083 + ], + "bench.Network.track_network": [ + [ + true, + true + ], + [ + [ + "{'name': 'A'}", + "{'name': 'B'}" + ] + ], + "26e3a7e66482e998b12768ed82102e95f8770e4e2f3737de978309feb7d94f69", + 1791349309252, + 0.012449, + null, + null, + null, + null, + null, + null, + [ + { + "total_transfer_in_bytes": 100, + "total_traffic_in_number_of_web_packets": 4, + "total_transfer_time_in_seconds": 2.0 + }, + { + "total_transfer_in_bytes": 100, + "total_traffic_in_number_of_web_packets": 4, + "total_transfer_time_in_seconds": 2.0 + } + ] + ], + "bench.Timed.time_read": [ + [ + 1.2534999996205443e-05, + NaN + ], + [ + [ + "{'name': 'A'}", + "{'name': 'B'}" + ] + ], + "267b65a446ea0bd2b4564e2747f943211b97455e145f7675e0abcc11247da3ad", + 1791349309265, + 0.006144, + [ + -Infinity, + null + ], + [ + Infinity, + null + ], + [ + 1.2535e-05, + null + ], + [ + 1.2535e-05, + null + ], + [ + 1, + null + ], + [ + 1, + null + ], + [ + [ + 1.2534999996205443e-05 + ], + null + ] + ] + }, + "version": 2 +} diff --git a/tests/test_incremental_slicing_config.py b/tests/test_incremental_slicing_config.py new file mode 100644 index 00000000..d73da18b --- /dev/null +++ b/tests/test_incremental_slicing_config.py @@ -0,0 +1,32 @@ +import pytest + +from nwb_benchmarks import _parse_incremental_slicing_benchmarks_config + + +@pytest.mark.parametrize("value", [None, "", " ", "false", "0", "no", "OFF"]) +def test_disabled_values(value): + assert _parse_incremental_slicing_benchmarks_config(value) == (False, None) + + +@pytest.mark.parametrize("value", ["true", "1", "yes", "on", "all", " ALL "]) +def test_values_enabling_all_modalities(value): + assert _parse_incremental_slicing_benchmarks_config(value) == (True, None) + + +@pytest.mark.parametrize( + ("value", "expected_modalities"), + [ + ("icephys", {"icephys"}), + ("icephys,ophys", {"icephys", "ophys"}), + (" Ecephys , OPHYS ", {"ecephys", "ophys"}), + ("ecephys,,icephys,", {"ecephys", "icephys"}), + ], +) +def test_modality_selection(value, expected_modalities): + assert _parse_incremental_slicing_benchmarks_config(value) == (True, expected_modalities) + + +@pytest.mark.parametrize("value", ["spikes", "icephys,spikes", "true,icephys"]) +def test_invalid_values_raise(value): + with pytest.raises(ValueError, match="Invalid RUN_INCREMENTAL_SLICING_BENCHMARKS value"): + _parse_incremental_slicing_benchmarks_config(value) diff --git a/tests/test_reduce_results.py b/tests/test_reduce_results.py new file mode 100644 index 00000000..ce1be40e --- /dev/null +++ b/tests/test_reduce_results.py @@ -0,0 +1,164 @@ +import json +import math +import pathlib +import shutil + +import pytest + +from nwb_benchmarks.setup import _reduce_results +from nwb_benchmarks.setup._reduce_results import ( + _extract_successful_results, + _serialize_parameter_cases, + reduce_results, +) + +# Written by `asv run --record-samples` (as `nwb_benchmarks run` calls it) with asv 0.6.1, trimmed to the keys the +# reducer reads. It holds one benchmark of each result shape the suite produces: +# - `Incremental`: a `track_` benchmark returning a plain dict, as `track_incremental_slicing` does. ASV stores it in +# the `result` column without samples. Parameter set B raised, so its result is `null`. +# - `Network`: a `track_` benchmark returning `dict(samples=..., number=None)`, as the network tracking benchmarks do. +# - `Timed`: a `time_` benchmark run with a `--bench` pattern selecting only parameter set A, so B's result is NaN. +RAW_RESULTS_FILE_PATH = pathlib.Path(__file__).parent / "data" / "asv_0.6.1_raw_results.json" + +INCREMENTAL = "bench.Incremental.track_cumulative_slice_times" +NETWORK = "bench.Network.track_network" +TIMED = "bench.Timed.time_read" + + +@pytest.fixture +def raw_results_info() -> dict: + with open(RAW_RESULTS_FILE_PATH) as file_stream: + return json.load(fp=file_stream) + + +def extract(raw_results_info: dict, test_case: str, result_columns="from file") -> dict: + return _extract_successful_results( + test_case=test_case, + raw_results_list=raw_results_info["results"][test_case], + result_columns=raw_results_info["result_columns"] if result_columns == "from file" else result_columns, + ) + + +def test_incremental_results_keep_the_successful_parameter_set(raw_results_info): + assert extract(raw_results_info, INCREMENTAL) == {"{'name': 'A'}": {"cumulative_time_in_seconds": [0.5, 1.0, 1.5]}} + + +def test_network_results_are_read_from_the_samples(raw_results_info): + network_statistics = { + "total_transfer_in_bytes": 100, + "total_traffic_in_number_of_web_packets": 4, + "total_transfer_time_in_seconds": 2.0, + } + + assert extract(raw_results_info, NETWORK) == { + "{'name': 'A'}": network_statistics, + "{'name': 'B'}": network_statistics, + } + + +def test_time_results_leave_out_parameter_sets_that_did_not_run(raw_results_info): + timed_samples = raw_results_info["results"][TIMED][11] + + assert extract(raw_results_info, TIMED) == {"{'name': 'A'}": timed_samples[0]} + + +def test_rows_without_result_columns_use_the_fixed_layout(raw_results_info): + # Rows with samples have 12 entries and are read the same way; the incremental row has none, so it is left out. + assert extract(raw_results_info, NETWORK, result_columns=None) == extract(raw_results_info, NETWORK) + assert extract(raw_results_info, TIMED, result_columns=None) == extract(raw_results_info, TIMED) + assert extract(raw_results_info, INCREMENTAL, result_columns=None) == {} + + +def test_missing_and_skipped_rows_are_left_out(raw_results_info): + result_columns = raw_results_info["result_columns"] + + assert _extract_successful_results(test_case="x", raw_results_list=None, result_columns=result_columns) == {} + assert ( + _extract_successful_results(test_case="x", raw_results_list=[None, [["a"]]], result_columns=result_columns) + == {} + ) + + +def test_length_mismatch_warns_and_is_left_out(raw_results_info): + raw_results_list = [[1.0], [["{'name': 'A'}", "{'name': 'B'}"]]] + + with pytest.warns(UserWarning, match="Length mismatch"): + result = _extract_successful_results( + test_case="x", raw_results_list=raw_results_list, result_columns=raw_results_info["result_columns"] + ) + assert result == {} + + +@pytest.mark.parametrize( + ("serialized_params", "expected"), + [ + ([], ["()"]), + ([["'a'", "'b'"]], ["'a'", "'b'"]), + ([["'a'", "'b'"], ["1"]], ["(\"'a'\", '1')", "(\"'b'\", '1')"]), + ], +) +def test_serialize_parameter_cases(serialized_params, expected): + assert _serialize_parameter_cases(serialized_params=serialized_params) == expected + + +@pytest.fixture +def reduced_results_file_path(tmp_path, monkeypatch) -> pathlib.Path: + """Run `reduce_results` on a copy of the raw results, writing into a temporary home directory.""" + for name in ("RESULTS_DIR", "ENVIRONMENTS_DIR", "MACHINES_DIR"): + directory = tmp_path / name.lower() + directory.mkdir() + monkeypatch.setattr(_reduce_results, name, directory) + (tmp_path / "machines_dir" / "machine-test.json").write_text("{}") + + raw_results_file_path = tmp_path / "raw_results.json" + shutil.copy(RAW_RESULTS_FILE_PATH, raw_results_file_path) + raw_environment_info_file_path = tmp_path / "environment.txt" + raw_environment_info_file_path.write_text( + "# packages in environment at /conda/envs/nwb_benchmarks:\n" + "#\n" + "# Name Version Build Channel\n" + "pynwb 3.1.0 pypi_0 pypi\n" + ) + + reduce_results( + machine_id="test", + raw_results_file_path=raw_results_file_path, + raw_environment_info_file_path=raw_environment_info_file_path, + ) + + assert not raw_results_file_path.exists() + (reduced_results_file_path,) = (tmp_path / "results_dir").glob("*_machine-test_results.json") + return reduced_results_file_path + + +def test_reduce_results_writes_every_successful_result(reduced_results_file_path): + with open(reduced_results_file_path) as file_stream: + reduced_results_info = json.load(fp=file_stream) + + assert reduced_results_info["commit_hash"] == "0123456789abcdef0123456789abcdef01234567" + assert reduced_results_info["machine_id"] == "test" + assert sorted(reduced_results_info["results"]) == [INCREMENTAL, NETWORK, TIMED] + assert reduced_results_info["results"][INCREMENTAL] == { + "{'name': 'A'}": {"cumulative_time_in_seconds": [0.5, 1.0, 1.5]} + } + + +def test_database_reads_the_reduced_results(reduced_results_file_path): + pytest.importorskip("polars") + pytest.importorskip("seaborn") + from nwb_benchmarks.database import Results + + results = Results.safe_load_from_json(file_path=reduced_results_file_path) + data_frame = results.to_dataframe() + + incremental = data_frame.filter(data_frame["benchmark_name"] == INCREMENTAL) + assert incremental["variable"].to_list() == ["cumulative_time_in_seconds"] * 3 + assert incremental["value"].to_list() == [0.5, 1.0, 1.5] + + network = data_frame.filter(data_frame["benchmark_name"] == NETWORK) + mean_time_per_web_packet = network.filter(network["variable"] == "mean_time_per_web_packet")["value"].to_list() + assert mean_time_per_web_packet == [0.5, 0.5] + + timed = data_frame.filter(data_frame["benchmark_name"] == TIMED) + assert timed["variable"].to_list() == ["time"] + assert not math.isnan(timed["value"][0]) diff --git a/tests/test_track_incremental_slicing.py b/tests/test_track_incremental_slicing.py new file mode 100644 index 00000000..615ab0d5 --- /dev/null +++ b/tests/test_track_incremental_slicing.py @@ -0,0 +1,119 @@ +import datetime + +import numpy as np +import pytest +from pynwb import NWBFile, TimeSeries + +from nwb_benchmarks.benchmarks import params +from nwb_benchmarks.benchmarks.track_incremental_slicing import ( + IncrementalSliceBenchmark, +) + + +def make_nwbfile() -> NWBFile: + nwbfile = NWBFile( + session_description="incremental slicing test", + identifier="incremental-slicing-test", + session_start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + ) + nwbfile.add_acquisition(TimeSeries(name="series", data=np.arange(40).reshape(10, 4), unit="V", rate=1.0)) + nwbfile.add_acquisition(TimeSeries(name="response", data=np.arange(5), unit="V", rate=1.0)) + nwbfile.add_stimulus(TimeSeries(name="stimulus", data=np.arange(3), unit="A", rate=1.0)) + return nwbfile + + +class InMemoryIncrementalSliceBenchmark(IncrementalSliceBenchmark): + def __init__(self, nwbfile: NWBFile): + self._nwbfile_to_open = nwbfile + + def _open_file(self, params): + self.nwbfile = self._nwbfile_to_open + + +def test_time_axis_slices_cover_the_axis_and_keep_trailing_slices(): + data = np.zeros((10, 4)) + slices = list(IncrementalSliceBenchmark._iter_time_axis_slices(data, (slice(0, 4), slice(0, 2)))) + + assert slices == [ + (slice(0, 4), slice(0, 2)), + (slice(4, 8), slice(0, 2)), + (slice(8, 10), slice(0, 2)), + ] + + +def test_time_axis_slices_use_the_template_width_from_the_start_of_the_axis(): + slices = list(IncrementalSliceBenchmark._iter_time_axis_slices(np.zeros(7), (slice(2, 5),))) + + assert slices == [(slice(0, 3),), (slice(3, 6),), (slice(6, 7),)] + + +def test_time_axis_slices_reject_an_empty_template(): + with pytest.raises(ValueError, match="Invalid time-axis slice template"): + list(IncrementalSliceBenchmark._iter_time_axis_slices(np.zeros(7), (slice(3, 3),))) + + +def test_time_axis_slices_reject_scalar_data(): + with pytest.raises(ValueError, match="at least one dimension"): + list(IncrementalSliceBenchmark._iter_time_axis_slices(np.float64(1.0), (slice(0, 1),))) + + +def test_icephys_timeseries_iterates_acquisition_then_stimulus(): + names = [name for name, _ in IncrementalSliceBenchmark._iter_icephys_timeseries(make_nwbfile())] + + assert names == ["acquisition/series", "acquisition/response", "stimulus/stimulus"] + + +def assert_cumulative_times(result, expected_length): + assert list(result) == ["cumulative_time_in_seconds"] + cumulative_times = result["cumulative_time_in_seconds"] + assert len(cumulative_times) == expected_length + assert all(isinstance(value, float) for value in cumulative_times) + assert cumulative_times == sorted(cumulative_times) + + +def test_cumulative_times_for_time_axis_strategy(): + benchmark = InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()) + result = benchmark._track_cumulative_slice_times( + params=dict(object_name="series", slice_template=(slice(0, 4), slice(0, 4)), slice_strategy="iterate_time_axis") + ) + + # The open time, then one entry for each of the 3 slices covering the 10 time points. + assert_cumulative_times(result=result, expected_length=1 + 3) + np.testing.assert_array_equal(benchmark._temp, np.arange(32, 40).reshape(2, 4)) + + +def test_cumulative_times_for_icephys_strategy(): + benchmark = InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()) + result = benchmark._track_cumulative_slice_times(params=dict(slice_strategy="iterate_icephys_timeseries")) + + # The open time, then one entry for each of the 3 time series. + assert_cumulative_times(result=result, expected_length=1 + 3) + np.testing.assert_array_equal(benchmark._temp, np.arange(3)) + + +def test_unsupported_strategy_raises(): + benchmark = InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()) + + with pytest.raises(ValueError, match="Unsupported incremental slice strategy"): + benchmark._track_cumulative_slice_times(params=dict(slice_strategy="iterate_backwards")) + + +def test_teardown_without_open_resources(): + InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()).teardown(params=dict()) + + +def test_filter_keeps_all_parameter_sets_without_a_modality_selection(monkeypatch): + monkeypatch.setattr(params, "RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES", None) + parameter_sets = (dict(modality="ecephys"), dict(modality="ophys"), dict(modality="icephys")) + + assert params.filter_incremental_slice_params_by_modality(parameter_sets) == parameter_sets + + +def test_filter_keeps_only_selected_modalities(monkeypatch): + monkeypatch.setattr(params, "RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES", {"icephys", "ophys"}) + parameter_sets = (dict(modality="ecephys"), dict(modality="ophys"), dict(modality="icephys")) + + assert params.filter_incremental_slice_params_by_modality(parameter_sets) == ( + dict(modality="ophys"), + dict(modality="icephys"), + ) From 57c75e5c5581dc35c8cfa07a68338a9f617f4d94 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 05:17:24 +0000 Subject: [PATCH 12/15] Drop the no-op `number` override and the hdf5 prefix of the shared params - `number` only applies to `time_` benchmarks; ASV calls a `track_` benchmark once per sample regardless, so the override on `IncrementalSliceBenchmark` did nothing. - The `incremental_hdf5_*_params` dicts are shared by the HDF5, Zarr and LINDI parameter sets, so they are now `incremental_*_params`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- src/nwb_benchmarks/benchmarks/params.py | 36 +++++++++---------- .../benchmarks/track_incremental_slicing.py | 3 -- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/params.py b/src/nwb_benchmarks/benchmarks/params.py index 237457b3..8ce2f21f 100644 --- a/src/nwb_benchmarks/benchmarks/params.py +++ b/src/nwb_benchmarks/benchmarks/params.py @@ -215,21 +215,21 @@ def filter_incremental_slice_params_by_modality(params: tuple[dict, ...]) -> tup ) -incremental_hdf5_ecephys_params = dict( +incremental_ecephys_params = dict( name="EcephysIncrementalSliceTestCase", modality="ecephys", object_name="ElectricalSeries", slice_template=ecephys_slices[0], slice_strategy="iterate_time_axis", ) -incremental_hdf5_ophys_params = dict( +incremental_ophys_params = dict( name="OphysIncrementalSliceTestCase", modality="ophys", object_name="TwoPhotonSeries", slice_template=ophys_slices[0], slice_strategy="iterate_time_axis", ) -incremental_hdf5_icephys_params = dict( +incremental_icephys_params = dict( name="IcephysIncrementalSliceTestCase", modality="icephys", slice_strategy="iterate_icephys_timeseries", @@ -237,41 +237,41 @@ def filter_incremental_slice_params_by_modality(params: tuple[dict, ...]) -> tup hdf5_redirected_read_incremental_slice_params = filter_incremental_slice_params_by_modality( ( - dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_redirected"]), - dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_redirected"]), - dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_redirected"]), + dict(**incremental_ecephys_params, https_url=hdf5_ecephys_params["https_url_redirected"]), + dict(**incremental_ophys_params, https_url=hdf5_ophys_params["https_url_redirected"]), + dict(**incremental_icephys_params, https_url=hdf5_icephys_params["https_url_redirected"]), ) ) hdf5_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( ( - dict(**incremental_hdf5_ecephys_params, https_url=hdf5_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=hdf5_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=hdf5_icephys_params["https_url_no_redirect"]), + dict(**incremental_ecephys_params, https_url=hdf5_ecephys_params["https_url_no_redirect"]), + dict(**incremental_ophys_params, https_url=hdf5_ophys_params["https_url_no_redirect"]), + dict(**incremental_icephys_params, https_url=hdf5_icephys_params["https_url_no_redirect"]), ) ) zarr_direct_read_incremental_slice_params = filter_incremental_slice_params_by_modality( ( - dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_direct"]), - dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_direct"]), - dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_direct"]), + dict(**incremental_ecephys_params, https_url=zarr_ecephys_params["https_url_direct"]), + dict(**incremental_ophys_params, https_url=zarr_ophys_params["https_url_direct"]), + dict(**incremental_icephys_params, https_url=zarr_icephys_params["https_url_direct"]), ) ) zarr_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( ( - dict(**incremental_hdf5_ecephys_params, https_url=zarr_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=zarr_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=zarr_icephys_params["https_url_no_redirect"]), + dict(**incremental_ecephys_params, https_url=zarr_ecephys_params["https_url_no_redirect"]), + dict(**incremental_ophys_params, https_url=zarr_ophys_params["https_url_no_redirect"]), + dict(**incremental_icephys_params, https_url=zarr_icephys_params["https_url_no_redirect"]), ) ) lindi_no_redirect_download_incremental_slice_params = filter_incremental_slice_params_by_modality( ( - dict(**incremental_hdf5_ecephys_params, https_url=lindi_ecephys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_ophys_params, https_url=lindi_ophys_params["https_url_no_redirect"]), - dict(**incremental_hdf5_icephys_params, https_url=lindi_icephys_params["https_url_no_redirect"]), + dict(**incremental_ecephys_params, https_url=lindi_ecephys_params["https_url_no_redirect"]), + dict(**incremental_ophys_params, https_url=lindi_ophys_params["https_url_no_redirect"]), + dict(**incremental_icephys_params, https_url=lindi_icephys_params["https_url_no_redirect"]), ) ) diff --git a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py index 0b1db36d..1c15639b 100644 --- a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py +++ b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py @@ -44,9 +44,6 @@ class IncrementalSliceBenchmark(BaseBenchmark, ABC): """Base class for cumulative open + repeated slice benchmarks.""" - # Ensure a full incremental sequence is only performed once per ASV measurement. - number = 1 - # Some full-axis incremental reads may be long-running. timeout = 60 * 60 * 12 From e9af9fbe3b84df5675098fe4d4d5b096a2923ce0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 05:27:00 +0000 Subject: [PATCH 13/15] Wrap incremental slicing results as ASV samples; restore the existing reducer `track_cumulative_slice_times` now returns `dict(samples=..., number=None)`, like the network tracking benchmarks. With the `--record-samples` flag `nwb_benchmarks run` always passes, ASV then writes the cumulative times to the samples column, which the existing `reduce_results` reads, so the reducer rewrite is no longer needed: - `_reduce_results.py` is back to its version on `main`, with one fix: a parameter set that failed (`null` samples) no longer drops the successful parameter sets of the same benchmark through the length-mismatch warning. The warning now covers only a structural mismatch between the parameter, result and samples lists. - `parse_parameter_case` no longer special-cases `()`, which only zero-parameter benchmarks produce and the suite has none of. - The guard in `normalize_time_and_network_results` stays: the database reader still treats every dict result as network statistics otherwise. The reducer tests now run `reduce_results` on a raw results file regenerated with asv 0.6.1, holding the wrapped incremental result, a network, a time, and an unwrapped `track_` benchmark. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- .../benchmarks/track_incremental_slicing.py | 7 +- src/nwb_benchmarks/database/_models.py | 2 - src/nwb_benchmarks/setup/_reduce_results.py | 120 ++++--------- tests/data/asv_0.6.1_raw_results.json | 73 ++++++-- tests/test_reduce_results.py | 164 ++++++++---------- tests/test_track_incremental_slicing.py | 10 ++ 6 files changed, 173 insertions(+), 203 deletions(-) diff --git a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py index 1c15639b..5625709d 100644 --- a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py +++ b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py @@ -123,9 +123,12 @@ def _track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, lis return {"cumulative_time_in_seconds": cumulative_times} @skip_benchmark_if(not RUN_INCREMENTAL_SLICING_BENCHMARKS) - def track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, list[float]]: + def track_cumulative_slice_times(self, params: dict[str, Any]) -> dict[str, Any]: """Track cumulative open + repeated slice timing.""" - return self._track_cumulative_slice_times(params=params) + # Same structure as `NetworkTracker.asv_network_statistics`: ASV stores 'samples' in the samples column of + # the raw results (with `--record-samples`), which is where `reduce_results` reads every benchmark from. + # 'number' is simply required, but needs to be None for custom track_ functions. + return dict(samples=self._track_cumulative_slice_times(params=params), number=None) class HDF5PyNWBRemfileWithCacheIncrementalSliceBenchmark(IncrementalSliceBenchmark): diff --git a/src/nwb_benchmarks/database/_models.py b/src/nwb_benchmarks/database/_models.py index 3bce5fc7..f44830c6 100644 --- a/src/nwb_benchmarks/database/_models.py +++ b/src/nwb_benchmarks/database/_models.py @@ -64,8 +64,6 @@ def parse_parameter_case(s): # if the parsed string is not a dict (older benchmarks results), convert it to one if not isinstance(output, dict): - if output == (): - return {} output = {"https_url": output[0].strip("'")} return output diff --git a/src/nwb_benchmarks/setup/_reduce_results.py b/src/nwb_benchmarks/setup/_reduce_results.py index cf2207ba..291186aa 100644 --- a/src/nwb_benchmarks/setup/_reduce_results.py +++ b/src/nwb_benchmarks/setup/_reduce_results.py @@ -3,7 +3,6 @@ import collections import datetime import hashlib -import itertools import json import math import pathlib @@ -17,86 +16,6 @@ from ..utils import get_dictionary_checksum -def _serialize_parameter_cases(serialized_params: list) -> list[str]: - """Return one serialized parameter-case key per ASV result. - - ASV stores parameters as a list of parameter axes. Preserve the historical - single-axis keys used by nwb_benchmarks results, while supporting ASV's - general zero-axis and multi-axis parameter layouts. - """ - - if len(serialized_params) == 0: - return ["()"] - if len(serialized_params) == 1: - return serialized_params[0] - - return [str(parameter_case) for parameter_case in itertools.product(*serialized_params)] - - -def _extract_successful_results(test_case: str, raw_results_list: list, result_columns: list[str] | None) -> dict: - """Extract successful ASV benchmark results keyed by serialized parameter case. - - ASV has used multiple result serialization layouts. Older result files store a - fixed-length list where parameters are at index 1 and samples are at index 11. - Newer result files include a top-level ``result_columns`` list and each result - row follows that column order. This helper normalizes both layouts to the - reduced results format used by nwb_benchmarks. - """ - - if raw_results_list is None: - return {} - - if result_columns is not None: - column_index = {column_name: index for index, column_name in enumerate(result_columns)} - if "result" not in column_index or "params" not in column_index: - return {} - - params_index = column_index["params"] - result_index = column_index["result"] - if params_index >= len(raw_results_list) or result_index >= len(raw_results_list): - return {} - - serialized_params = raw_results_list[params_index] - aggregate_results = raw_results_list[result_index] - raw_results = aggregate_results - if "samples" in column_index: - samples_index = column_index["samples"] - if samples_index < len(raw_results_list) and raw_results_list[samples_index] is not None: - raw_results = raw_results_list[samples_index] - # Older ASV layout used by the original reducer implementation. - elif len(raw_results_list) == 12: - aggregate_results = raw_results_list[0] - serialized_params = raw_results_list[1] - raw_results = raw_results_list[11] - else: - return {} - - # Skipped results in JSON are written as `null` and read back into Python as `None`. - if aggregate_results is None or raw_results is None: - return {} - if not isinstance(aggregate_results, list): - aggregate_results = [aggregate_results] - if not isinstance(raw_results, list): - raw_results = [raw_results] - serialized_params = _serialize_parameter_cases(serialized_params=serialized_params) - - if len(serialized_params) != len(aggregate_results) or len(serialized_params) != len(raw_results): - message = ( - f"In intermediate results for test case {test_case}: \n" - f"\tLength mismatch between parameters ({len(serialized_params)}) and " - f"results ({len(aggregate_results)}) or result samples ({len(raw_results)})!\n\n" - "Please raise an issue and share your intermediate results file." - ) - warnings.warn(message=message) - return {} - - return { - params: raw_result - for params, aggregate_result, raw_result in zip(serialized_params, aggregate_results, raw_results) - if not (isinstance(aggregate_result, float) and math.isnan(aggregate_result)) and raw_result is not None - } - - def _parse_environment_info(raw_environment_info: List[str]) -> Dict[str, List[Dict[str, str]]]: """Turn the results of `conda list` printout to a JSON dictionary.""" header_stripped = raw_environment_info[3:] @@ -124,15 +43,38 @@ def reduce_results(machine_id: str, raw_results_file_path: pathlib.Path, raw_env timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") reduced_results = dict() - result_columns = raw_results_info.get("result_columns") for test_case, raw_results_list in raw_results_info["results"].items(): - extracted_results = _extract_successful_results( - test_case=test_case, - raw_results_list=raw_results_list, - result_columns=result_columns, - ) - if extracted_results: - reduced_results.update({test_case: extracted_results}) + + # Only successful runs have a results field of length 12 + if len(raw_results_list) != 12: + continue + + # This code assumes that test cases are only run with one parameter + assert len(raw_results_list[1]) == 1, "Unexpected length of serialized parameters list!" + serialized_params = raw_results_list[1][0] + + aggregate_results = raw_results_list[0] + samples_per_parameter_set = raw_results_list[11] + if not len(serialized_params) == len(aggregate_results) == len(samples_per_parameter_set): + message = ( + f"In intermediate results for test case {test_case}: \n" + f"\tLength mismatch between parameters ({len(serialized_params)}), results " + f"({len(aggregate_results)}) and result samples ({len(samples_per_parameter_set)})!\n\n" + "Please raise an issue and share your intermediate results file." + ) + warnings.warn(message=message) + continue + + # ASV records every parameter set of a benchmark, including those a `--bench` pattern did not select. + # Those were never run and have a NaN result. Samples of parameter sets that failed are written as `null` + # and read back into Python as `None`. Leave both out, keeping the parameter sets that succeeded. + successful_results = { + params: samples + for params, result, samples in zip(serialized_params, aggregate_results, samples_per_parameter_set) + if not (isinstance(result, float) and math.isnan(result)) and samples is not None + } + if len(successful_results) > 0: + reduced_results.update({test_case: successful_results}) if len(reduced_results) == 0: raise ValueError( diff --git a/tests/data/asv_0.6.1_raw_results.json b/tests/data/asv_0.6.1_raw_results.json index b4987d9c..7f4c59d4 100644 --- a/tests/data/asv_0.6.1_raw_results.json +++ b/tests/data/asv_0.6.1_raw_results.json @@ -18,13 +18,7 @@ "results": { "bench.Incremental.track_cumulative_slice_times": [ [ - { - "cumulative_time_in_seconds": [ - 0.5, - 1.0, - 1.5 - ] - }, + true, null ], [ @@ -33,9 +27,25 @@ "{'name': 'B'}" ] ], - "b3e7fa9dfc40a9240d5f4dc9d6e3e552a4fbbe90b8b7e12ee43c6aff1c810eef", - 1791349309228, - 0.023083 + "baffb96bc63f705ae883e56b15ebc74070d05406a779dc4ec4f382bfe6d84b36", + 1791350742012, + 0.017311, + null, + null, + null, + null, + null, + null, + [ + { + "cumulative_time_in_seconds": [ + 0.5, + 1.0, + 1.5 + ] + }, + null + ] ], "bench.Network.track_network": [ [ @@ -49,8 +59,8 @@ ] ], "26e3a7e66482e998b12768ed82102e95f8770e4e2f3737de978309feb7d94f69", - 1791349309252, - 0.012449, + 1791350742031, + 0.013156, null, null, null, @@ -72,7 +82,7 @@ ], "bench.Timed.time_read": [ [ - 1.2534999996205443e-05, + 1.3246999998273168e-05, NaN ], [ @@ -82,8 +92,8 @@ ] ], "267b65a446ea0bd2b4564e2747f943211b97455e145f7675e0abcc11247da3ad", - 1791349309265, - 0.006144, + 1791350742044, + 0.007133, [ -Infinity, null @@ -93,11 +103,11 @@ null ], [ - 1.2535e-05, + 1.3247e-05, null ], [ - 1.2535e-05, + 1.3247e-05, null ], [ @@ -110,10 +120,37 @@ ], [ [ - 1.2534999996205443e-05 + 1.3246999998273168e-05 ], null ] + ], + "bench.Unwrapped.track_unwrapped": [ + [ + { + "cumulative_time_in_seconds": [ + 0.5, + 1.0, + 1.5 + ] + }, + { + "cumulative_time_in_seconds": [ + 0.5, + 1.0, + 1.5 + ] + } + ], + [ + [ + "{'name': 'A'}", + "{'name': 'B'}" + ] + ], + "c89cd8f48cba96f3dc897029408daf80651d6bfa6c5477abd567a34b6c5f0c37", + 1791350742052, + 0.011107 ] }, "version": 2 diff --git a/tests/test_reduce_results.py b/tests/test_reduce_results.py index ce1be40e..0932cacd 100644 --- a/tests/test_reduce_results.py +++ b/tests/test_reduce_results.py @@ -1,28 +1,31 @@ import json import math import pathlib -import shutil import pytest from nwb_benchmarks.setup import _reduce_results -from nwb_benchmarks.setup._reduce_results import ( - _extract_successful_results, - _serialize_parameter_cases, - reduce_results, -) +from nwb_benchmarks.setup._reduce_results import reduce_results # Written by `asv run --record-samples` (as `nwb_benchmarks run` calls it) with asv 0.6.1, trimmed to the keys the -# reducer reads. It holds one benchmark of each result shape the suite produces: -# - `Incremental`: a `track_` benchmark returning a plain dict, as `track_incremental_slicing` does. ASV stores it in -# the `result` column without samples. Parameter set B raised, so its result is `null`. +# reducer reads. It holds one benchmark of each result shape below: +# - `Incremental`: a `track_` benchmark returning `dict(samples=..., number=None)` with a list of cumulative times, as +# `track_incremental_slicing` does. Parameter set B raised, so its samples are `null`. # - `Network`: a `track_` benchmark returning `dict(samples=..., number=None)`, as the network tracking benchmarks do. # - `Timed`: a `time_` benchmark run with a `--bench` pattern selecting only parameter set A, so B's result is NaN. +# - `Unwrapped`: a `track_` benchmark returning a plain dict, which ASV stores in the `result` column without samples. RAW_RESULTS_FILE_PATH = pathlib.Path(__file__).parent / "data" / "asv_0.6.1_raw_results.json" INCREMENTAL = "bench.Incremental.track_cumulative_slice_times" NETWORK = "bench.Network.track_network" TIMED = "bench.Timed.time_read" +UNWRAPPED = "bench.Unwrapped.track_unwrapped" + +NETWORK_STATISTICS = { + "total_transfer_in_bytes": 100, + "total_traffic_in_number_of_web_packets": 4, + "total_transfer_time_in_seconds": 2.0, +} @pytest.fixture @@ -31,125 +34,102 @@ def raw_results_info() -> dict: return json.load(fp=file_stream) -def extract(raw_results_info: dict, test_case: str, result_columns="from file") -> dict: - return _extract_successful_results( - test_case=test_case, - raw_results_list=raw_results_info["results"][test_case], - result_columns=raw_results_info["result_columns"] if result_columns == "from file" else result_columns, +@pytest.fixture +def reduce(tmp_path, monkeypatch): + """Return a function that runs `reduce_results` on raw results, writing into a temporary home directory.""" + for name in ("RESULTS_DIR", "ENVIRONMENTS_DIR", "MACHINES_DIR"): + directory = tmp_path / name.lower() + directory.mkdir() + monkeypatch.setattr(_reduce_results, name, directory) + (tmp_path / "machines_dir" / "machine-test.json").write_text("{}") + + raw_environment_info_file_path = tmp_path / "environment.txt" + raw_environment_info_file_path.write_text( + "# packages in environment at /conda/envs/nwb_benchmarks:\n" + "#\n" + "# Name Version Build Channel\n" + "pynwb 3.1.0 pypi_0 pypi\n" ) + def _reduce(raw_results_info: dict) -> pathlib.Path: + raw_results_file_path = tmp_path / "raw_results.json" + with open(raw_results_file_path, mode="w") as file_stream: + json.dump(obj=raw_results_info, fp=file_stream) -def test_incremental_results_keep_the_successful_parameter_set(raw_results_info): - assert extract(raw_results_info, INCREMENTAL) == {"{'name': 'A'}": {"cumulative_time_in_seconds": [0.5, 1.0, 1.5]}} + reduce_results( + machine_id="test", + raw_results_file_path=raw_results_file_path, + raw_environment_info_file_path=raw_environment_info_file_path, + ) + assert not raw_results_file_path.exists() + (reduced_results_file_path,) = (tmp_path / "results_dir").glob("*_machine-test_results.json") + return reduced_results_file_path -def test_network_results_are_read_from_the_samples(raw_results_info): - network_statistics = { - "total_transfer_in_bytes": 100, - "total_traffic_in_number_of_web_packets": 4, - "total_transfer_time_in_seconds": 2.0, - } + return _reduce - assert extract(raw_results_info, NETWORK) == { - "{'name': 'A'}": network_statistics, - "{'name': 'B'}": network_statistics, - } +def reduced_results(reduced_results_file_path: pathlib.Path) -> dict: + with open(reduced_results_file_path) as file_stream: + return json.load(fp=file_stream)["results"] -def test_time_results_leave_out_parameter_sets_that_did_not_run(raw_results_info): - timed_samples = raw_results_info["results"][TIMED][11] - assert extract(raw_results_info, TIMED) == {"{'name': 'A'}": timed_samples[0]} +def test_incremental_results_keep_the_successful_parameter_set(reduce, raw_results_info): + results = reduced_results(reduce(raw_results_info)) + assert results[INCREMENTAL] == {"{'name': 'A'}": {"cumulative_time_in_seconds": [0.5, 1.0, 1.5]}} -def test_rows_without_result_columns_use_the_fixed_layout(raw_results_info): - # Rows with samples have 12 entries and are read the same way; the incremental row has none, so it is left out. - assert extract(raw_results_info, NETWORK, result_columns=None) == extract(raw_results_info, NETWORK) - assert extract(raw_results_info, TIMED, result_columns=None) == extract(raw_results_info, TIMED) - assert extract(raw_results_info, INCREMENTAL, result_columns=None) == {} +def test_network_results(reduce, raw_results_info): + results = reduced_results(reduce(raw_results_info)) -def test_missing_and_skipped_rows_are_left_out(raw_results_info): - result_columns = raw_results_info["result_columns"] + assert results[NETWORK] == {"{'name': 'A'}": NETWORK_STATISTICS, "{'name': 'B'}": NETWORK_STATISTICS} - assert _extract_successful_results(test_case="x", raw_results_list=None, result_columns=result_columns) == {} - assert ( - _extract_successful_results(test_case="x", raw_results_list=[None, [["a"]]], result_columns=result_columns) - == {} - ) +def test_time_results_leave_out_parameter_sets_that_did_not_run(reduce, raw_results_info): + timed_samples = raw_results_info["results"][TIMED][11] + + results = reduced_results(reduce(raw_results_info)) -def test_length_mismatch_warns_and_is_left_out(raw_results_info): - raw_results_list = [[1.0], [["{'name': 'A'}", "{'name': 'B'}"]]] + assert results[TIMED] == {"{'name': 'A'}": timed_samples[0]} - with pytest.warns(UserWarning, match="Length mismatch"): - result = _extract_successful_results( - test_case="x", raw_results_list=raw_results_list, result_columns=raw_results_info["result_columns"] - ) - assert result == {} +def test_track_results_without_samples_are_left_out(reduce, raw_results_info): + results = reduced_results(reduce(raw_results_info)) -@pytest.mark.parametrize( - ("serialized_params", "expected"), - [ - ([], ["()"]), - ([["'a'", "'b'"]], ["'a'", "'b'"]), - ([["'a'", "'b'"], ["1"]], ["(\"'a'\", '1')", "(\"'b'\", '1')"]), - ], -) -def test_serialize_parameter_cases(serialized_params, expected): - assert _serialize_parameter_cases(serialized_params=serialized_params) == expected + assert sorted(results) == [INCREMENTAL, NETWORK, TIMED] -@pytest.fixture -def reduced_results_file_path(tmp_path, monkeypatch) -> pathlib.Path: - """Run `reduce_results` on a copy of the raw results, writing into a temporary home directory.""" - for name in ("RESULTS_DIR", "ENVIRONMENTS_DIR", "MACHINES_DIR"): - directory = tmp_path / name.lower() - directory.mkdir() - monkeypatch.setattr(_reduce_results, name, directory) - (tmp_path / "machines_dir" / "machine-test.json").write_text("{}") +def test_length_mismatch_warns_and_leaves_out_only_that_benchmark(reduce, raw_results_info): + raw_results_info["results"][NETWORK][11] = raw_results_info["results"][NETWORK][11][:1] - raw_results_file_path = tmp_path / "raw_results.json" - shutil.copy(RAW_RESULTS_FILE_PATH, raw_results_file_path) - raw_environment_info_file_path = tmp_path / "environment.txt" - raw_environment_info_file_path.write_text( - "# packages in environment at /conda/envs/nwb_benchmarks:\n" - "#\n" - "# Name Version Build Channel\n" - "pynwb 3.1.0 pypi_0 pypi\n" - ) + with pytest.warns(UserWarning, match=f"test case {NETWORK}: \n\tLength mismatch"): + results = reduced_results(reduce(raw_results_info)) - reduce_results( - machine_id="test", - raw_results_file_path=raw_results_file_path, - raw_environment_info_file_path=raw_environment_info_file_path, - ) + assert sorted(results) == [INCREMENTAL, TIMED] - assert not raw_results_file_path.exists() - (reduced_results_file_path,) = (tmp_path / "results_dir").glob("*_machine-test_results.json") - return reduced_results_file_path +def test_no_successful_results_raises(reduce, raw_results_info): + raw_results_info["results"] = {UNWRAPPED: raw_results_info["results"][UNWRAPPED]} -def test_reduce_results_writes_every_successful_result(reduced_results_file_path): - with open(reduced_results_file_path) as file_stream: + with pytest.raises(ValueError, match="failed to find any successful results"): + reduce(raw_results_info) + + +def test_reduced_results_file(reduce, raw_results_info): + with open(reduce(raw_results_info)) as file_stream: reduced_results_info = json.load(fp=file_stream) assert reduced_results_info["commit_hash"] == "0123456789abcdef0123456789abcdef01234567" assert reduced_results_info["machine_id"] == "test" - assert sorted(reduced_results_info["results"]) == [INCREMENTAL, NETWORK, TIMED] - assert reduced_results_info["results"][INCREMENTAL] == { - "{'name': 'A'}": {"cumulative_time_in_seconds": [0.5, 1.0, 1.5]} - } -def test_database_reads_the_reduced_results(reduced_results_file_path): +def test_database_reads_the_reduced_results(reduce, raw_results_info): pytest.importorskip("polars") pytest.importorskip("seaborn") from nwb_benchmarks.database import Results - results = Results.safe_load_from_json(file_path=reduced_results_file_path) - data_frame = results.to_dataframe() + data_frame = Results.safe_load_from_json(file_path=reduce(raw_results_info)).to_dataframe() incremental = data_frame.filter(data_frame["benchmark_name"] == INCREMENTAL) assert incremental["variable"].to_list() == ["cumulative_time_in_seconds"] * 3 diff --git a/tests/test_track_incremental_slicing.py b/tests/test_track_incremental_slicing.py index 615ab0d5..14fe39b1 100644 --- a/tests/test_track_incremental_slicing.py +++ b/tests/test_track_incremental_slicing.py @@ -91,6 +91,16 @@ def test_cumulative_times_for_icephys_strategy(): np.testing.assert_array_equal(benchmark._temp, np.arange(3)) +def test_tracked_result_is_wrapped_as_asv_samples(): + # ASV only writes a `track_` result to the samples column, which `reduce_results` reads, in this structure. + benchmark = InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()) + result = benchmark.track_cumulative_slice_times(params=dict(slice_strategy="iterate_icephys_timeseries")) + + assert list(result) == ["samples", "number"] + assert result["number"] is None + assert_cumulative_times(result=result["samples"], expected_length=1 + 3) + + def test_unsupported_strategy_raises(): benchmark = InMemoryIncrementalSliceBenchmark(nwbfile=make_nwbfile()) From 952da087f74d12be5eae296e1bfb5566a9e69e95 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:50:14 -0400 Subject: [PATCH 14/15] Regenerate the reducer test fixture with asv 0.6.6 (#208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to NeurodataWithoutBorders/nwb-benchmarks#203 (targets its branch). The raw results fixture behind `tests/test_reduce_results.py` was written with asv 0.6.1. Main now pins asv 0.6.6 and asv-runner 0.3.1 (NeurodataWithoutBorders/nwb-benchmarks#205), so this PR regenerates the fixture with those versions. ## Changes - **Merges `main` into the branch.** It merges cleanly and brings in the #205 pins, so the tests in this PR run against the asv version that wrote the fixture. - **Regenerates the fixture.** It comes from the same toy suite as before: `Incremental` (B raises), `Network`, `Timed` (`--bench` selects only parameter set A) and `Unwrapped`. The suite was run with `asv run --python=same --record-samples` and trimmed to the keys the reducer reads, with the same fixed commit hash. - **Renames the fixture** `tests/data/asv_0.6.1_raw_results.json` → `asv_0.6.6_raw_results.json`, and updates the path and comment in the test. ## What changed in the fixture The structure is the same as under 0.6.1: same `result_columns`, same row lengths (12 for the wrapped benchmarks, 5 for `Unwrapped`), NaN result for the unselected `Timed` B, `null` samples for the failed `Incremental` B, and the same `true` results for the wrapped `track_` benchmarks. Only the `Timed` timing, the benchmark version hashes, `started_at` and `duration` differ. No version bump: nothing under `src/nwb_benchmarks/` changes. ## Testing - `python -m pytest tests`: 40 passed, none skipped (polars and seaborn installed, so the database-reader test ran). - `pre-commit run` on the changed files passes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01At337MLAynHQXz3TZBF3Vv --- _Generated by [Claude Code](https://claude.ai/code/session_01At337MLAynHQXz3TZBF3Vv)_ Co-authored-by: Claude --- ...esults.json => asv_0.6.6_raw_results.json} | 32 +++++++++---------- tests/test_reduce_results.py | 6 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) rename tests/data/{asv_0.6.1_raw_results.json => asv_0.6.6_raw_results.json} (77%) diff --git a/tests/data/asv_0.6.1_raw_results.json b/tests/data/asv_0.6.6_raw_results.json similarity index 77% rename from tests/data/asv_0.6.1_raw_results.json rename to tests/data/asv_0.6.6_raw_results.json index 7f4c59d4..93acfe5e 100644 --- a/tests/data/asv_0.6.1_raw_results.json +++ b/tests/data/asv_0.6.6_raw_results.json @@ -27,9 +27,9 @@ "{'name': 'B'}" ] ], - "baffb96bc63f705ae883e56b15ebc74070d05406a779dc4ec4f382bfe6d84b36", - 1791350742012, - 0.017311, + "4933c92c8671f70cbc6f12096f43626d1a35ee18b927d65b2776eb866ab757a7", + 1791387880190, + 0.19855, null, null, null, @@ -58,9 +58,9 @@ "{'name': 'B'}" ] ], - "26e3a7e66482e998b12768ed82102e95f8770e4e2f3737de978309feb7d94f69", - 1791350742031, - 0.013156, + "2001c3f48d2b4c732d8ba1ae5b21815f0e6483e2fb2ed78cc86e3a8658f11bb4", + 1791387880390, + 0.20317, null, null, null, @@ -82,7 +82,7 @@ ], "bench.Timed.time_read": [ [ - 1.3246999998273168e-05, + 3.129999996076549e-06, NaN ], [ @@ -91,9 +91,9 @@ "{'name': 'B'}" ] ], - "267b65a446ea0bd2b4564e2747f943211b97455e145f7675e0abcc11247da3ad", - 1791350742044, - 0.007133, + "4ec591f3893044fe1b377f3e1e7ba67e837b75d5522d4151dd4f60b426e8186e", + 1791387880594, + 0.097666, [ -Infinity, null @@ -103,11 +103,11 @@ null ], [ - 1.3247e-05, + 3.13e-06, null ], [ - 1.3247e-05, + 3.13e-06, null ], [ @@ -120,7 +120,7 @@ ], [ [ - 1.3246999998273168e-05 + 3.129999996076549e-06 ], null ] @@ -148,9 +148,9 @@ "{'name': 'B'}" ] ], - "c89cd8f48cba96f3dc897029408daf80651d6bfa6c5477abd567a34b6c5f0c37", - 1791350742052, - 0.011107 + "9d2350fd9f55c4dcab83774838a0cac7a82b1928ff1ffee3bafb465713adef23", + 1791387880693, + 0.20363 ] }, "version": 2 diff --git a/tests/test_reduce_results.py b/tests/test_reduce_results.py index 0932cacd..fe963b19 100644 --- a/tests/test_reduce_results.py +++ b/tests/test_reduce_results.py @@ -7,14 +7,14 @@ from nwb_benchmarks.setup import _reduce_results from nwb_benchmarks.setup._reduce_results import reduce_results -# Written by `asv run --record-samples` (as `nwb_benchmarks run` calls it) with asv 0.6.1, trimmed to the keys the -# reducer reads. It holds one benchmark of each result shape below: +# Written by `asv run --record-samples` (as `nwb_benchmarks run` calls it) with asv 0.6.6 and asv-runner 0.3.1, the +# versions the environment pins, trimmed to the keys the reducer reads. It holds one benchmark of each result shape: # - `Incremental`: a `track_` benchmark returning `dict(samples=..., number=None)` with a list of cumulative times, as # `track_incremental_slicing` does. Parameter set B raised, so its samples are `null`. # - `Network`: a `track_` benchmark returning `dict(samples=..., number=None)`, as the network tracking benchmarks do. # - `Timed`: a `time_` benchmark run with a `--bench` pattern selecting only parameter set A, so B's result is NaN. # - `Unwrapped`: a `track_` benchmark returning a plain dict, which ASV stores in the `result` column without samples. -RAW_RESULTS_FILE_PATH = pathlib.Path(__file__).parent / "data" / "asv_0.6.1_raw_results.json" +RAW_RESULTS_FILE_PATH = pathlib.Path(__file__).parent / "data" / "asv_0.6.6_raw_results.json" INCREMENTAL = "bench.Incremental.track_cumulative_slice_times" NETWORK = "bench.Network.track_network" From e1abf5fb951001bc2e0ea9091902750b0d2a09b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 22:31:40 +0000 Subject: [PATCH 15/15] Move RUN_INCREMENTAL_SLICING_BENCHMARKS handling into the benchmark params Only the benchmark suite reads it (`params.py` filters the parameter sets by modality, `track_incremental_slicing.py` gates the benchmark), but the parser and its warning ran in the package `__init__`, so on every import of `nwb_benchmarks`: an invalid value broke every CLI command and the warning printed on all of them. They now live in `benchmarks/params.py`, next to the modality filter, and `__init__.py` is back to its version on `main`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015K2S7gEkSTfhpnfbXVD5YG --- docs/running_benchmarks.rst | 4 +- src/nwb_benchmarks/__init__.py | 46 ------------------- src/nwb_benchmarks/benchmarks/params.py | 46 ++++++++++++++++++- .../benchmarks/track_incremental_slicing.py | 2 +- tests/test_incremental_slicing_config.py | 4 +- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/running_benchmarks.rst b/docs/running_benchmarks.rst index 91158856..8d0b341d 100644 --- a/docs/running_benchmarks.rst +++ b/docs/running_benchmarks.rst @@ -180,8 +180,8 @@ To run multiple modalities, provide a comma-separated list: $env:RUN_INCREMENTAL_SLICING_BENCHMARKS="icephys,ophys" -Invalid values raise an error during package import so that misspellings do not accidentally run the wrong set of -benchmarks. +Invalid values raise an error when the benchmark suite is imported so that misspellings do not accidentally run the +wrong set of benchmarks. Example: running only icephys incremental slicing benchmarks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/nwb_benchmarks/__init__.py b/src/nwb_benchmarks/__init__.py index 0bc03d28..320fad36 100644 --- a/src/nwb_benchmarks/__init__.py +++ b/src/nwb_benchmarks/__init__.py @@ -10,37 +10,6 @@ TSHARK_PATH = os.environ.get("TSHARK_PATH", None) NETWORK_INTERFACE = os.environ.get("NWB_BENCHMARKS_NETWORK_INTERFACE", None) RUN_DOWNLOAD_BENCHMARKS = os.environ.get("RUN_DOWNLOAD_BENCHMARKS", None) -RUN_INCREMENTAL_SLICING_BENCHMARKS_RAW = os.environ.get("RUN_INCREMENTAL_SLICING_BENCHMARKS", None) - - -def _parse_incremental_slicing_benchmarks_config(value: str | None) -> tuple[bool, set[str] | None]: - """Parse RUN_INCREMENTAL_SLICING_BENCHMARKS into enabled state and selected modalities.""" - if value is None: - return False, None - - normalized_value = value.strip().lower() - if normalized_value in {"", "false", "0", "no", "off"}: - return False, None - - if normalized_value in {"true", "1", "yes", "on", "all"}: - return True, None - - selected_modalities = {modality.strip().lower() for modality in normalized_value.split(",") if modality.strip()} - valid_modalities = {"ecephys", "ophys", "icephys"} - invalid_modalities = selected_modalities - valid_modalities - if invalid_modalities: - raise ValueError( - f"Invalid RUN_INCREMENTAL_SLICING_BENCHMARKS value {value!r}. " - "Expected true, all, ecephys, ophys, icephys, or a comma-separated list of modalities." - ) - - return True, selected_modalities - - -( - RUN_INCREMENTAL_SLICING_BENCHMARKS, - RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES, -) = _parse_incremental_slicing_benchmarks_config(RUN_INCREMENTAL_SLICING_BENCHMARKS_RAW) if TSHARK_PATH is None: TSHARK_PATH = shutil.which("tshark") @@ -57,24 +26,9 @@ def _parse_incremental_slicing_benchmarks_config(value: str | None) -> tuple[boo "RUN_DOWNLOAD_BENCHMARKS is set. Benchmarks that download the entire test file will be run, which may take a long time." ) -if RUN_INCREMENTAL_SLICING_BENCHMARKS: - if RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES is None: - incremental_slicing_benchmark_description = "all incremental slicing benchmarks" - else: - incremental_slicing_benchmark_description = ( - "incremental slicing benchmarks for modalities " - f"{', '.join(sorted(RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES))}" - ) - warnings.warn( - "RUN_INCREMENTAL_SLICING_BENCHMARKS is set. " - f"{incremental_slicing_benchmark_description} will be run, which may take a long time." - ) - __all__ = [ "main", "TSHARK_PATH", "NETWORK_INTERFACE", "RUN_DOWNLOAD_BENCHMARKS", - "RUN_INCREMENTAL_SLICING_BENCHMARKS", - "RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES", ] diff --git a/src/nwb_benchmarks/benchmarks/params.py b/src/nwb_benchmarks/benchmarks/params.py index 8ce2f21f..978cc2ad 100644 --- a/src/nwb_benchmarks/benchmarks/params.py +++ b/src/nwb_benchmarks/benchmarks/params.py @@ -1,4 +1,5 @@ -from nwb_benchmarks import RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES +import os +import warnings # The DANDI API download URLs in this module (`https_url_no_redirect`) are literals, resolved once with # `nwb_benchmarks.core.get_https_url(..., follow_redirects=False)`, so that importing the benchmark suite, which ASV does @@ -203,6 +204,49 @@ #################################### INCREMENTAL SLICE PARAMETERS ################################### +def _parse_incremental_slicing_benchmarks_config(value: str | None) -> tuple[bool, set[str] | None]: + """Parse RUN_INCREMENTAL_SLICING_BENCHMARKS into enabled state and selected modalities.""" + if value is None: + return False, None + + normalized_value = value.strip().lower() + if normalized_value in {"", "false", "0", "no", "off"}: + return False, None + + if normalized_value in {"true", "1", "yes", "on", "all"}: + return True, None + + selected_modalities = {modality.strip().lower() for modality in normalized_value.split(",") if modality.strip()} + valid_modalities = {"ecephys", "ophys", "icephys"} + invalid_modalities = selected_modalities - valid_modalities + if invalid_modalities: + raise ValueError( + f"Invalid RUN_INCREMENTAL_SLICING_BENCHMARKS value {value!r}. " + "Expected true, all, ecephys, ophys, icephys, or a comma-separated list of modalities." + ) + + return True, selected_modalities + + +( + RUN_INCREMENTAL_SLICING_BENCHMARKS, + RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES, +) = _parse_incremental_slicing_benchmarks_config(os.environ.get("RUN_INCREMENTAL_SLICING_BENCHMARKS", None)) + +if RUN_INCREMENTAL_SLICING_BENCHMARKS: + if RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES is None: + incremental_slicing_benchmark_description = "all incremental slicing benchmarks" + else: + incremental_slicing_benchmark_description = ( + "incremental slicing benchmarks for modalities " + f"{', '.join(sorted(RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES))}" + ) + warnings.warn( + "RUN_INCREMENTAL_SLICING_BENCHMARKS is set. " + f"{incremental_slicing_benchmark_description} will be run, which may take a long time." + ) + + def filter_incremental_slice_params_by_modality(params: tuple[dict, ...]) -> tuple[dict, ...]: """Filter incremental slicing benchmark params by RUN_INCREMENTAL_SLICING_BENCHMARKS modality selection.""" if RUN_INCREMENTAL_SLICING_BENCHMARK_MODALITIES is None: diff --git a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py index 5625709d..1861fefe 100644 --- a/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py +++ b/src/nwb_benchmarks/benchmarks/track_incremental_slicing.py @@ -17,7 +17,6 @@ from asv_runner.benchmarks.mark import SkipNotImplemented, skip_benchmark_if from pynwb import NWBHDF5IO -from nwb_benchmarks import RUN_INCREMENTAL_SLICING_BENCHMARKS from nwb_benchmarks.core import ( BaseBenchmark, download_asset_if_not_exists, @@ -33,6 +32,7 @@ from nwb_benchmarks.setup import get_persistent_download_directory from .params import ( + RUN_INCREMENTAL_SLICING_BENCHMARKS, hdf5_no_redirect_download_incremental_slice_params, hdf5_redirected_read_incremental_slice_params, lindi_no_redirect_download_incremental_slice_params, diff --git a/tests/test_incremental_slicing_config.py b/tests/test_incremental_slicing_config.py index d73da18b..9d020273 100644 --- a/tests/test_incremental_slicing_config.py +++ b/tests/test_incremental_slicing_config.py @@ -1,6 +1,8 @@ import pytest -from nwb_benchmarks import _parse_incremental_slicing_benchmarks_config +from nwb_benchmarks.benchmarks.params import ( + _parse_incremental_slicing_benchmarks_config, +) @pytest.mark.parametrize("value", [None, "", " ", "false", "0", "no", "OFF"])