diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 09d29be66..64af7abc0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,5 +20,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Install clang-format + run: pipx install clang-format - name: Lint run: cmake -D FORMAT_COMMAND=clang-format -P cmake/lint.cmake diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2b973a0bd..3ec0ddcdf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -144,7 +144,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install nwbinspector - nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION --ignore=check_subject_exists --json-file-path out.json + nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION --ignore=check_electrodes_location_allen_ccf --json-file-path out.json if ! grep -q '"messages": \[\]' out.json; then echo "NWBInspector found issues in the NWB files" exit 1 diff --git a/.github/workflows/upgrade_schema.yml b/.github/workflows/upgrade_schema.yml index 0f24dedb9..d32ed0804 100644 --- a/.github/workflows/upgrade_schema.yml +++ b/.github/workflows/upgrade_schema.yml @@ -58,7 +58,7 @@ jobs: run: | mkdir -p nwb_files cp build/tests/data/*.nwb nwb_files/ - nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION --ignore=check_subject_exists --json-file-path out.json + nwbinspector nwb_files --threshold BEST_PRACTICE_VIOLATION --ignore=check_electrodes_location_allen_ccf --json-file-path out.json if ! grep -q '"messages": \[\]' out.json; then echo "NWBInspector found issues in the NWB files" exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e738604..8a27ea841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,26 +6,80 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Upcoming (~August 2026) + +## Upcoming [0.4.0] (~August 2026) ### Added * **Added support for streaming data read of remote NWB files:** - * Added `HDF5IO::openS3(...)` method to support opening an existing remote file in S3 in read-only mode using the HDF5 ROS3 driver. (@oruebel, [#307](https://github.com/NeurodataWithoutBorders/aqnwb/pull/307)) - * Added `HDF5IO::openRemote()` method to read remote NWB files over HTTP(S) using the [remfile-cpp](https://github.com/catalystneuro/remfile-cpp) virtual file driver (a C++ port of the Python [remfile](https://github.com/magland/remfile) package), imported as an optional CMake dependency (`AQNWB_USE_REMFILE`, requires `libcurl`). Unlike ROS3, remfile does not require HDF5 to be built with ROS3 support and works with any HTTP(S) server that supports byte-range requests. (@bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) - * Added demo for benchmarking ROS3 and remfile performance and comparing with PyNWB S3 reads (`demo/remote_read_benchmark`). (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) - * Added tutorial on using the ROS3 and remfile drivers to read NWB files in S3 (`docs/pages/userdocs/reads3.dox`) (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) -* Added `ElectricalSeries::writeAllChannels` method and `IO::writeElectricalSeriesData` overload to simplify zero-copy interleaved multichannel writes. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) -* Added `ElectricalSeries::channelsAtSameSampleOffset` method to check if all channels are at the same sample offset, which is a requirement for using `writeAllChannels`. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) -* Added new `BaseIO::findObject` and `RegisteredType::findOwnedObject` methods to simplify searching for objects by name. Added `HDF5IO::findObject` override method to optimize the search for HDF5 objects. (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308)) + * Added `HDF5IO::openS3(...)` method to support opening an existing remote file in S3 in read-only mode using the HDF5 ROS3 driver. (@oruebel, [#307](https://github.com/NeurodataWithoutBorders/aqnwb/pull/307)) + * Added `HDF5IO::openRemote()` method to read remote NWB files over HTTP(S) using the [remfile-cpp](https://github.com/catalystneuro/remfile-cpp) virtual file driver (a C++ port of the Python [remfile](https://github.com/magland/remfile) package), imported as an optional CMake dependency (`AQNWB_USE_REMFILE`, requires `libcurl`). Unlike ROS3, remfile does not require HDF5 to be built with ROS3 support and works with any HTTP(S) server that supports byte-range requests. (@bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) + * Added new `BaseIO::findObject` and `RegisteredType::findOwnedObject` methods to simplify searching for objects by name. Added `HDF5IO::findObject` override method to optimize the search for HDF5 objects. (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308)) + * Added demo for benchmarking ROS3 and remfile performance and comparing with PyNWB S3 reads (`demo/remote_read_benchmark`). (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) + * Added tutorial on using the ROS3 and remfile drivers to read NWB files in S3 (`docs/pages/userdocs/reads3.dox`) (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) +* **Added support for NWB schema 2.11 and EventTable, TimeIntervals, MeaningsTable, VectorIndex, and Subject:** + * Updated schema in the `spec/` module to the latest NWB releases: (i) NWB 2.11, (ii) HDMF Common 1.10, and (iii) HDMF Experimental 0.6 (@oruebel, [#300](https://github.com/NeurodataWithoutBorders/aqnwb/pull/300)) + * Added `TimestampVectorData` and `DurationVectorData` types to support event timing data. (@cline, @oruebel, [#301](https://github.com/NeurodataWithoutBorders/aqnwb/pull/301)) + * Updated `DynamicTable::readColumn` using type traits to support reading both `VectorData` columns with a specific data type and columns that are a subtype of `VectorData` (e.g., the new `TimestampVectorData` and `DurationVectorData`). (@cline, @oruebel, [#301](https://github.com/NeurodataWithoutBorders/aqnwb/pull/301)) + * Added new `EventsTable` type to support recording of event data and updated `NWBFile` to `NWBFile::initialize` to create the new top-level `events` group and add create and read methods to support events (e.g., `NWBFile::createEventsTable`). (@cline, @oruebel, [#304](https://github.com/NeurodataWithoutBorders/aqnwb/pull/304)) + * Added `TimeIntervals` type for annotating time intervals and updated `NWBFile` to support the standard `intervals` tables (e.g., `trials`, `invalid_times`, `epochs`). (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added new `MeaningsTable` type to support adding meanings to `DynamicTable` columns. (@cline, @oruebel, [#302](https://github.com/NeurodataWithoutBorders/aqnwb/pull/302)) + * Added `DynamicTable::createMeaningsTable` and `DynamicTable::readMeaningsTable` methods to support creating and reading `MeaningsTable` objects associated with a `DynamicTable`. (@cline, @oruebel, [#302](https://github.com/NeurodataWithoutBorders/aqnwb/pull/302)) + * Added tutorials for the new `MeaningsTable` and `EventsTable` types in `docs/pages/userdocs/events.dox`. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) + * Added tutorials for the new `TimeIntervals` type in `docs/pages/userdocs/time_intervals.dox`. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `Subject` class to represent the `/general/subject` group in NWB files. Added corresponding `SubjectSpec` to simplify configuration and initialization of `Subject`. Updated `NWBFile::initialize` to accept a `SubjectSpec` argument for subject metadata initialization (@copilot, @oruebel, [#320](https://github.com/NeurodataWithoutBorders/aqnwb/pull/320)) + * Added `isPathOrDescendant` utility functoin to simplify path initialization checks for root groups, e.g, `/events` and `/intervals` (@copilot, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) +* **Added support for single-ragged array columns in `DynamicTable`:** + * Added `VectorIndex` type for storing index columns. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `VectorData::appendData` and `VectorIndex::appendBuffer` to simplify appending values to columns. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Updated `DynamicTable::addRows` and related methods to support ragged array column writes. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) +* **Added support for row-based read and write of `DynamicTable`:** + * Added `DynamicTable::addRow` and `DynamicTable::addRows` methods to support row-based data insertion into a `DynamicTable`. Rows are specified as `RowData` (an `unordered_map` from column name to `CellValue` variant), enabling type-safe, column-keyed writes without requiring callers to manage per-column buffers directly. Row IDs are auto-generated if not provided. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)). + * Added new `Types::CellValue` variant type to support single-cell values for `DynamicTable` rows. `CellValue` supports all scalar types, as well as `std::vector` for ragged array columns. Also defined `Types::RowData` as an `unordered_map` to represent a single row of data in a `DynamicTable`. (@cline, @oruebel, Originally defined in [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305) and then subsequently enhanced and moved in [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `DynamicTable::readRows` to support reading of full rows, along with `VectorData::readCellValues` and `VectorIndex::readIndexedCellValues`. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `DynamicTable::toString` to support converting rows to a string representation. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `DynamicTable::getNumberOfRows` to support retrieving the number of rows in a `DynamicTable`. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Updated the user tutorial on reading data to discuss reading from `DynamicTable`. (@cline, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Added `BaseDataType::isCompatibleCellValue()`  and  `BaseDataType::isCompatibleVector()` for efficient runtime storage-type/variant compatibility check used in `VectorData::appendData()`  and  `appendBuffer()` to safeguard mismatching data types on write. (@copilot, @oruebel, [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) +* **Added support for `DataSpec`-based column configuration in `DynamicTable`:** + * **[BREAKING]** Updated `DynamicTable::initialize`, `ElectrodesTable::initialize`, `EventsTable::initialize`, and `MeaningsTable::initialize` to accept a `std::vector` column-spec list instead of individual `rowChunkSize` / resolution / flag parameters. This approach separated configuration from initialization, allowing users to fully configure all columns (including custom columns) beforehand such that `initialize` can then handle the full setup of the table. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) + * **Migration Note:** Callers should use the corresponding `createDefaultDataSpecs(...)` static factory to obtain the default spec vector optionally modify it, and pass it to `initialize()`. `NWBFile::createElectrodesTable` and `NWBFile::createEventsTable` have been updated accordingly. + * Added `DynamicTable::addColumn(DataSpecPtr)` overload consistent with the `DataSpec`-based API used by `initialize()` and `createDefaultDataSpecs()`.(@cline, @oruebel, [#301](https://github.com/NeurodataWithoutBorders/aqnwb/pull/301)) + * Added `Data::DataSpecBase` and `Data::DataSpec` to support runtime configuration of `DynamicTable` column layouts and to decouple table schema definition from object initialization. Each `Data` subclass (e.g., `VectorData`, `ElementIdentifiers`, `TimestampVectorData`, `DurationVectorData`) now exposes a nested `DataSpec` struct that bundles the dataset configuration, description, and any type-specific parameters needed to create and initialize the column. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) + * Added `DynamicTable::createDefaultDataSpecs` static factory to return the default ordered list of `DataSpec` objects for a table (currently just the `id` column). Subclasses (`ElectrodesTable`, `EventsTable`, `MeaningsTable`) override this to append their own required columns, allowing callers to retrieve, inspect, and customize the default column configuration before passing it to `initialize()`. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) + * Added `BaseDataType::BaseDataVariant` scalar variant type and `BaseDataType::createEmptyVectorVariant` helper to support runtime-typed single-cell and buffer operations needed by `addRow`/`addRows`. (@cline, @oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) + * Added `DyanmicTable::validateDataSpecs` (and `DynamicTable::checkRequiredColumnNames` helper function) to validate column specifications as part of `DyanmicTable::initialize` before initialization. Also updated subclasses of `DynamicTable` (e.g., `ElectrodesTable`, `EventsTable`, `MeaningsTable`) to override `validateDataSpecs` to provide their specific validation logic. (@oruebel, [#305](https://github.com/NeurodataWithoutBorders/aqnwb/pull/305)) +* **Added support for zero-copy interleaved multichannel writes:** + * Added `ElectricalSeries::writeAllChannels` method and `IO::writeElectricalSeriesData` overload to simplify zero-copy interleaved multichannel writes. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) + * Added `ElectricalSeries::channelsAtSameSampleOffset` method to check if all channels are at the same sample offset, which is a requirement for using `writeAllChannels`. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) ### Changed +* **Enhanced caching of `RegisteredType` (types) and `BaseRecordingData` (datasets) to centralize recording state and avoid duplicate object creation.** + * **[BREAKING]** Updated `RegisteredType::create()` to cache and reuse one canonical registered object per path. Factory methods resolve the concrete on-disk registered type before reusing a cached object and return `nullptr` for incompatible cached types rather than creating a duplicate or returning the wrong wrapper. `DataTyped`, `VectorDataTyped`, and `NWBDataTyped` remain transient typed facades and are not registered in `RecordingObjects`. (@cline, @oruebel, @copilot, [#328](https://github.com/NeurodataWithoutBorders/aqnwb/pull/328), [#320](https://github.com/NeurodataWithoutBorders/aqnwb/pull/320)) + * **Migration Note:** This change is mostly transparent to users, but it centralizes recording state. Code using `RegisteredType::create()` must handle a `nullptr` result when an incompatible type is already cached at the requested path; use the specialized factories for typed facades. + * **[BREAKING]** Updated the caching of `BaseRecordingData` to move from a per-object cache as part of `RegisteredType` objects (e.g., `TimeSeries`) to a central cache managed by the `BaseIO` I/O object. This change ensures that we have one `BaseRecordingData` object per dataset in a file, and that all `RegisteredType` objects that reference the same dataset will share the same `BaseRecordingData` object. This change centralizes the recording state. + * **Migration Note:** This change mainly affects users who were manually accessing or clearing the cache via `RegisteredType::getCacheRecordingData()` or `RegisteredType::clearRecordingDataCache()`. Instead, access the cache using the I/O object using `BaseIO::getRecordingDataCache()` and `BaseIO::clearRecordingDataCache()`. (@cline, @oruebel, [#328](https://github.com/NeurodataWithoutBorders/aqnwb/pull/328)) * **[BREAKING]** Moved `disableSWMRMode` option from `HDF5IO` constructor to a new `HDF5IO::startRecording(bool disableSWMRMode)` overload. The `BaseIO`-compliant `startRecording()` override is preserved and defaults to SWMR enabled. * **Migration Note**: Code using `HDF5IO(path, true)` must be updated to `HDF5IO(path)` followed by `startRecording(true)`. When the `HDF5IO` object is held as a `std::shared_ptr` (e.g., from `createIO`), downcast with `std::dynamic_pointer_cast` to access the overload. (@oruebel [#297](https://github.com/NeurodataWithoutBorders/aqnwb/pull/297)) -* Updated `build-demo.yml` and `tests.yml` CI to support testing of the new ROS3 and remfile features and scripts (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308)) +* **[BREAKING]** Updated `Types.hpp` to use a `namespace` instead of `class` to group types. (@oruebel, [#325]((https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * **Migration Note:** For most users this change should not require any code changes since the syntax for accessing the types is the same (e.g., `AQNWB::Types::Status`), but for users who were using `using` declarations to import the type can now use `using namespace AQNWB::Types`. +* **[BREAKING]** Refactored `DynamicTable` so it creates and initializes its `ElementIdentifiers` `id` column internally during `initialize()`. As part of this change, `setRowIDs()` now takes only the row ID values and writes them directly to the table's built-in `id` column instead of accepting a separate `ElementIdentifiers` object. (@oruebel, [#302](https://github.com/NeurodataWithoutBorders/aqnwb/pull/302)) + * **Migration Notes:** In practice this aligns the API with the intended usage, since callers should have always used the table's own `id` dataset. The main change is in the signature of `DynamicTable::setRowIDs()`. The change in initialization behavior is part of the move to `DataSpec`-based table configurations such that migration is covered by the changes to `DynamicTable::initialize`. +* **[BREAKING]** Updated `DynamicTable` column-name management to be immediate and table-driven. `addColumnName()` now flushes new column names to the file as columns are added, `setColNames()` is now restricted to reordering the existing set of columns (it no longer acts as a general overwrite mechanism), and `finalize()` no longer writes the `colnames` attribute because column-name updates are persisted when they occur. This ensures that the `colnames` attribute is always consistent with the actual columns in the table and helps avoid creation of invalid files. (@oruebel, [#304](https://github.com/NeurodataWithoutBorders/aqnwb/pull/304)) + * **Migration Note:** For most users this should not require code changes. The main change is that `setColNames()` now enforces that the new column names are consistent with the existing columns in the tables and, hence, cannot be used to arbitrarily overwrite the column names, but the new column names must be a reordering of the existing column names. If you need to add new columns, use `addColumn()` instead. +* **Updated CI test workflows:** + * Updated `build-demo.yml` and `tests.yml` CI to support testing of the new ROS3 and remfile features and scripts (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308)) + * Updated `lint.yml` workflow to install `clang-format` via `pipx` to ensure use of the latest version of `clang-format` (@copilot, @oruebel [#325](https://github.com/NeurodataWithoutBorders/aqnwb/pull/325)) + * Updated nwbinspector validation tests to ignore the Allen CCF electrode location check when validating mock electrode locations. (@oruebel, [#320](https://github.com/NeurodataWithoutBorders/aqnwb/pull/320)) + * Updated nwbinspector validation tests in the CI to remove dependency on `sanitizer` tests to speed up CI (@oruebel, [#289](https://github.com/NeurodataWithoutBorders/aqnwb/pull/289)) + * Updated unit tests to create `Subject` for all NWB files generated in the test suite to comply with NWB best practices. `nwbinspector` tests require subject by default now. (@oruebel, [#289](https://github.com/NeurodataWithoutBorders/aqnwb/pull/289) removed this requirements by specifying `--ignore=check_subject_exists` and [#320](https://github.com/NeurodataWithoutBorders/aqnwb/pull/320) removed the ignore to ensure that subject is always present in the test files) ### Fixed -* Updated nwbinspector validation tests in the CI to: 1) `--ignore=check_subject_exists` and 2) remove dependency on `sanitizer` tests to speed up CI (@oruebel, [#289](https://github.com/NeurodataWithoutBorders/aqnwb/pull/289)) * Fixed `get_utc_offset_seconds` to correctly account for daylight saving time using platform-specific APIs (`tm_gmtoff` on Unix/macOS; `_get_timezone` + `_get_dstbias` on Windows), preventing `session_start_time` from being written ~1 hour ahead of UTC during DST (@cboulay, [#295](https://github.com/NeurodataWithoutBorders/aqnwb/pull/295)) +* Fixed `resources/utils/schematype_to_aqnwb.py` to facilitate generation of new types: + * Emit forward declarations rather than full header includes for referenced generated types, avoiding circular header dependencies. This issue surfaced due to the update to the latest HDMF Common schema types where `DynamicTable` references `MeaningsTable`, which is itself a `DynamicTable` (@copilot, @oruebel, [#300](https://github.com/NeurodataWithoutBorders/aqnwb/pull/300)) + * Updated minimum Python version for `resources/utils` to ensure use of the latest versions of PyNWB with uv (@cline, @oruebel, [#301](https://github.com/NeurodataWithoutBorders/aqnwb/pull/301)) + * Updated cpp code generation to use new `std::weak_ptr` via `getIO` instead of the previous `m_io` shared pointer. (@cline, @oruebel, [#301](https://github.com/NeurodataWithoutBorders/aqnwb/pull/301)) + * Various indentation and formatting fixes in generated cpp code to improve readability (across multiple PRs, e.g., #300, #301). * Fixed bug in `HDF5IO::canModifyObjects` returning true when a file is opened in read-only. (@oruebel, [#307](https://github.com/NeurodataWithoutBorders/aqnwb/pull/307)) * Fixed HDF5 string type creation to explicitly use UTF-8 character set for fixed-length and variable-length strings in datasets and attributes, improving compatibility with hdmf/PyNWB string decoding (@copilot, @oruebel [#319](https://github.com/NeurodataWithoutBorders/aqnwb/pull/319)) * Fixed Windows CI by updating the CMake generator in `CMakePresets.json` from `"Visual Studio 17 2022"` to `"Visual Studio 18 2026"` to match the updated `windows-latest` runner (@copilot, @oruebel [#319](https://github.com/NeurodataWithoutBorders/aqnwb/pull/319)) diff --git a/CMakeLists.txt b/CMakeLists.txt index 483a3ce51..6e7160d36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,14 +36,21 @@ add_library( src/nwb/device/Device.cpp src/nwb/ecephys/ElectricalSeries.cpp src/nwb/ecephys/SpikeEventSeries.cpp + src/nwb/epoch/TimeIntervals.cpp + src/nwb/event/EventsTable.cpp + src/nwb/event/TimestampVectorData.cpp + src/nwb/event/DurationVectorData.cpp src/nwb/file/ElectrodeGroup.cpp src/nwb/file/ElectrodesTable.cpp + src/nwb/file/Subject.cpp src/nwb/misc/AnnotationSeries.cpp src/nwb/hdmf/base/Container.cpp src/nwb/hdmf/base/Data.cpp src/nwb/hdmf/table/DynamicTable.cpp - src/nwb/hdmf/table/VectorData.cpp src/nwb/hdmf/table/ElementIdentifiers.cpp + src/nwb/hdmf/table/MeaningsTable.cpp + src/nwb/hdmf/table/VectorData.cpp + src/nwb/hdmf/table/VectorIndex.cpp src/spec/NamespaceRegistry.cpp ) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 968daf8d7..502b596c0 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -81,3 +81,4 @@ PROJECT_LOGO = "@PROJECT_SOURCE_DIR@/resources/logo/aqnwb-logo.png" # Fail on warnings WARN_AS_ERROR = FAIL_ON_WARNINGS_PRINT +DOT_GRAPH_MAX_NODES = 100 diff --git a/docs/pages/1_userdocs.dox b/docs/pages/1_userdocs.dox index ece2140be..20d58444b 100644 --- a/docs/pages/1_userdocs.dox +++ b/docs/pages/1_userdocs.dox @@ -6,6 +6,8 @@ * * - \subpage user_install_page * - \subpage workflow + * - \subpage events + * - \subpage time_intervals * - \subpage read_page * - \subpage reads3_page * - \subpage hdf5io diff --git a/docs/pages/devdocs/read_design.dox b/docs/pages/devdocs/read_design.dox index c88157ecb..ab377e093 100644 --- a/docs/pages/devdocs/read_design.dox +++ b/docs/pages/devdocs/read_design.dox @@ -291,6 +291,12 @@ * - The \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create" method is used to create * an instance of a registered subclass by name. This method looks up the subclass name in the * registry and calls the corresponding factory function to create an instance. + * - Each I/O object caches one canonical registered object per file path. + * The path-only \ref AQNWB::NWB::RegisteredType::create "create" overload + * resolves the object's on-disk ``namespace`` and ``neurodata_type`` before + * reusing a cache entry. A request whose registered type conflicts with an + * existing cache entry fails rather than returning a different type or + * creating a second object for the same path. * * 5. **Class Name and Namespace Retrieval**: * - The \ref AQNWB::NWB::RegisteredType::getTypeName "getTypeName" and @@ -339,6 +345,13 @@ * with the type already set. The same approach is also applied in the case of \ref AQNWB::NWB::Data "Data" and its derived * class \ref AQNWB::NWB::DataTyped "DataTyped". * + * ``DataTyped``, ``VectorDataTyped``, and + * ``NWBDataTyped`` are typed read facades, not canonical registered + * objects. They are intentionally not stored in + * \ref AQNWB::IO::RecordingObjects "RecordingObjects"; use their specialized + * ``create`` or ``from...`` methods to obtain a typed view of a dataset without + * changing the canonical object cached for its path. + * * For further details and alternative approaches for implementing templated \ref AQNWB::NWB::RegisteredType "RegisteredType" * classes, see \ref implement_templated_registered_type. * diff --git a/docs/pages/devdocs/record_design.dox b/docs/pages/devdocs/record_design.dox index ceb00b5d4..c77dd18ba 100644 --- a/docs/pages/devdocs/record_design.dox +++ b/docs/pages/devdocs/record_design.dox @@ -18,7 +18,7 @@ * AqNWB records datasets efficiently via \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" objects. The main components involved in * writing data to an NWB file via AqNWB are: * - * 1. \ref DEFINE_DATASET_FIELD Macro + * 1. \ref DEFINE_DATASET_FIELD Macro and \ref AQNWB::NWB::RegisteredType "RegisteredType" * - The \ref DEFINE_DATASET_FIELD macro not only defines methods for reading datasets for a particular neurodata_type (as described in \ref read_design_page), * but also defines methods for retrieving \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" objects that are used for recording to individual datasets. * For each dataset field defined with this macro, a corresponding method is generated that returns a \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" @@ -31,13 +31,14 @@ * - It provides methods for writing data blocks to the dataset, such as \ref AQNWB::IO::BaseRecordingData::writeDataBlock "writeDataBlock", * which can handle different data types and dimensions. * - * 3. \ref AQNWB::NWB::RegisteredType "RegisteredType" - * - \ref AQNWB::NWB::RegisteredType "RegisteredType" maintains a cache of \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" objects via the `m_recordingDataCache` member. + * 3. \ref AQNWB::IO::BaseIO::m_recordingDataCache "BaseIO::m_recordingDataCache" + * - \ref AQNWB::IO::BaseIO "BaseIO" maintains a cache of \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" objects that belong to the I/O object/file. * This cache allows reusing the same \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object when it is requested multiple times, * improving performance and retaining the recording position. The cache is essential for writing data to the dataset in a streaming fashion, * as it ensures that each write continues from where the previous write left off. The cache also avoids the need for manually maintaining * the objects and allows caching of an arbitrary number of \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object such that the - * individual neurodata_type classes do not need to worry about maintaining their recording state. + * individual RegisteredType classes (i.e., neurodata_types) do not need to worry about maintaining their recording state, but the state is maintained + * centrally in the I/O object they belong to. * * 4. \ref AQNWB::IO::BaseIO "BaseIO" * - \ref AQNWB::IO::BaseIO "BaseIO" and its implementations (e.g., \ref AQNWB::IO::HDF5::HDF5IO "HDF5IO") are responsible for @@ -64,6 +65,7 @@ * + getDataSet(): BaseRecordingData * Attributes * - m_recording_objects: std::shared_ptr<RecordingObjects> + * - m_recordingDataCache: std::unordered_map<std::string, std::shared_ptr<BaseRecordingData>> * * > * ]; @@ -91,11 +93,8 @@ * label=< * * - * - * * - * - * + * * *
RegisteredType
Functions
+ clearRecordingDataCache(): void
Attributes
+ m_recordingDataCache: std::unordered_map
+ m_io: std::shared_ptr<BaseIO>
+ m_io: std::weak_ptr<BaseIO>
+ m_path: std::string
* > @@ -119,7 +118,8 @@ * * RegisteredType -> Container [arrowhead=empty, style=dashed]; * Container -> BaseRecordingData [label="created by functions defined via \nDEFINE_DATASET_FIELD"]; - * RegisteredType -> BaseRecordingData [label="caches"]; + * HDF5IO -> BaseRecordingData [label="caches"]; + * HDF5IO -> RegisteredType [label="caches"]; * BaseRecordingData -> HDF5IO [label="uses for\nwriting data"]; * HDF5IO -> NWBFile [label="write data"]; * } @@ -197,7 +197,6 @@ * \ref AQNWB::IO::RecordingObjects::addRecordingObject "addRecordingObject" and * \ref AQNWB::IO::RecordingObjects::addRecordingObject "getRecordingObject" methods. * - **Finalization of recording objects:** via \ref AQNWB::IO::RecordingObjects::finalize "finalize" - * - **Clearing of cached BaseRecordingData objects** via \ref AQNWB::IO::RecordingObjects::clearRecordingDataCache "clearRecordingDataCache"; * * \section record_design_nwbio_utils NWB I/O convenience utilities * @@ -233,9 +232,20 @@ * facilitate reliable memory management. * * \note - * Registration with the \ref AQNWB::IO::RecordingObjects "RecordingObjects" of the I/O occurs when a - * \ref AQNWB::NWB::RegisteredType "RegisteredType" is being created via the static - * \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create" factory methods. + * Canonical \ref AQNWB::NWB::RegisteredType "RegisteredType" objects created + * through \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create" + * are registered with the I/O's \ref AQNWB::IO::RecordingObjects + * "RecordingObjects". The cache has one canonical registered object per I/O + * and path; repeated compatible factory calls return that same instance. A + * request for a conflicting registered type fails rather than creating a + * second object for the path. + * + * Typed dataset facades such as \ref AQNWB::NWB::DataTyped "DataTyped", + * \ref AQNWB::NWB::VectorDataTyped "VectorDataTyped", and + * \ref AQNWB::NWB::NWBDataTyped "NWBDataTyped" are intentionally transient + * and are not registered with RecordingObjects. They share the I/O's + * BaseRecordingData cache with the canonical object when accessing the same + * dataset. * * \section recording_design_further_reading Further Reading * diff --git a/docs/pages/devdocs/registered_types.dox b/docs/pages/devdocs/registered_types.dox index da499448e..9ce5fec9b 100644 --- a/docs/pages/devdocs/registered_types.dox +++ b/docs/pages/devdocs/registered_types.dox @@ -84,7 +84,10 @@ * DEFINE_ATTRIBUTE_FIELD(getDescription, std::string, "description", Description of the data) * @endcode * and in for datasets also a method for retrieving a \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" - * wrapper used for recording data to a dataset. + * wrapper used for recording data to a dataset. The \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" + * objects are cached by the \ref AQNWB::IO::BaseIO "BaseIO" object to ensure that the current + * state of the recording process (e.g., the \ref AQNWB::IO::BaseRecordingData::getPosition "position") + * are stored. * @code * DEFINE_DATASET_FIELD(getData, recordData, float, "data", The main data) * @endcode @@ -389,19 +392,25 @@ * the \ref DEFINE_REGISTERED_FIELD macro, we can set the data type for read at compile time, simplifying read. * * \note - * Since the templacted \ref AQNWB::NWB::VectorDataTyped "VectorDataTyped" does not call - * \ref REGISTER_SUBCLASS we must manually declare: 1) `friend class AQNWB::NWB::RegisteredType;` - * and 2) `using VectorData::VectorData;` (as protected) to make sure we have access to the - * constructors and define `static std::shared_ptr create( const std::string& path, std::shared_ptr io) { return RegisteredType::create(path, io); }` - * to make sure we have the factory method available. - * + * Since the templated \ref AQNWB::NWB::VectorDataTyped + * "VectorDataTyped" does not call \ref REGISTER_SUBCLASS, it must + * declare `friend class AQNWB::NWB::RegisteredType;` and inherit the + * protected `VectorData` constructors with `using VectorData::VectorData;` + * so its specialized factory can construct it. That factory must create a + * transient typed view rather than delegate to + * \ref AQNWB::NWB::RegisteredType::create + * "RegisteredType::create", because that factory registers canonical + * objects in \ref AQNWB::IO::RecordingObjects "RecordingObjects". + * `DataTyped` and `NWBDataTyped` follow the same transient + * typed-view pattern; only their non-templated base classes are canonical + * registered types. + * + * \par * \note - * The `std::unique_ptr<..>` template type is not covariant, i.e., `std::unique_ptr` does - * not automatically convert to `std::unique_ptr`. I.e., while \ref AQNWB::NWB::VectorDataTyped "VectorDataTyped" - * can be used anywhere \ref AQNWB::NWB::VectorData "VectorData" is being used, when using `std::unique_ptr<..>` - * we cannot rely on the compiler to automatically upcast for us, but we will need to explicitly release - * and upcast `std::unique_ptr` if `std::unique_ptr` is required. Since - * defining the `DTYPE` is primarily useful for read, we therefore typically use + * AqNWB uses `std::shared_ptr` for registered and typed-view objects, so a + * `std::shared_ptr>` converts to + * `std::shared_ptr` when needed. Since defining `DTYPE` is + * primarily useful for read, we therefore typically use * \ref AQNWB::NWB::VectorDataTyped "VectorDataTyped" on read while * using \ref AQNWB::NWB::VectorData "VectorData" otherwise. * @@ -529,6 +538,7 @@ * than the requested version. In practice, this is mainly relevant if you are generating classes for the * NWB `core` and `hdmf-common` namespaces. * + * \par * \note * When generating the test app via the `--generate-test-app` option we also * need to generate the schema headers files and save them in the `/spec` folder. diff --git a/docs/pages/userdocs/events.dox b/docs/pages/userdocs/events.dox new file mode 100644 index 000000000..d642c6df3 --- /dev/null +++ b/docs/pages/userdocs/events.dox @@ -0,0 +1,243 @@ +/** + * \page events Acquiring Event Data 📋 + * + * \tableofcontents + * + * \section events_overview Overview + * + * AqNWB provides the \ref AQNWB::NWB::EventsTable "EventsTable" type for recording + * discrete events (such as TTL pulses, licks, rewards, stimulus onsets, or other + * event-based data), where each event is anchored at a single timestamp. An optional duration + * column can be added when the length of each event is known. Additional user-defined + * columns can be appended to store any per-event metadata. + * + * Each \ref AQNWB::NWB::EventsTable "EventsTable" should hold events of a **single type or related + * types** so that all rows share the same set of columns. Event types with different metadata needs + * (e.g., behavioral events and TTL pulses) should be stored in separate + * \ref AQNWB::NWB::EventsTable "EventsTable" instances. + * + * AqNWB supports two complementary acquisition patterns: + * + * | Pattern | When to use | + * |---------|-------------| + * | **Row-based** (`addRow` / `addRows`) | Events arrive one at a time during acquisition; the total count is not known in advance. | + * | **Column-based** (write full column vectors) | All event data are available in memory at once and can be written in a single bulk call per column. Add additional complete columns after acquisition is completed. | + * + * Both patterns share the same setup steps (I/O, NWBFile, table creation, and + * `startRecording`) and differ only in how data are written to the table. + * The two patterns are not exclusive and hybrid approaches are possible, + * but require caution. In particular when using column-based append, we must + * ensure that all columns have been appropriately updated such that + * all columns have indeed the same number rows before we use `addRow` or + * `addRows` to avoid inconsistent data. + * + * \par + * \note + * Chunked storage is **required** for when the total number of rows is not known in advance + * and we need to expand the columns at runtime. The `rowChunkSize` parameter of the + * `createDefaultDataSpecs` controls how many rows are stored per HDF5 chunk. In practice, + * the number of rows per chunk should typically be the same for all columns of a table. + * The chunking behavior can also be customized via the individual `DataSpec` of the columns. + * Avoid creating too small chunks (e.g., single rows) as well as too large chunks + * (e.g., millions of rows per chunk). + * + * \section events_setup Setup: I/O, NWBFile, and table creation + * + * \subsection events_io 1. Create the I/O object + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_io_snippet + * + * \subsection events_nwbfile 2. Create the NWBFile + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_nwbfile_snippet + * + * \subsection events_configure_table 3. Configure and create the EventsTable + * + * Before creating the table, build a list of column specifications using + * \ref AQNWB::NWB::EventsTable::createDefaultDataSpecs "EventsTable::createDefaultDataSpecs". + * This factory returns the default set of columns (always including the `id` and + * `timestamp` columns) and allows you to decide to include the optional `duration` and `annotation` + * columns and control the HDF5 chunk size used for each column. + * + * To add a **custom column** to the table, create a + * \ref AQNWB::NWB::VectorData::DataSpec "VectorData::DataSpec" (or a spec for any + * other \ref AQNWB::NWB::VectorData "VectorData" subtype) and push it onto the spec + * vector before passing it to + * \ref AQNWB::NWB::NWBFile::createEventsTable "NWBFile::createEventsTable". + * The spec bundles the column name, dataset configuration (data type, initial size, + * chunk size), and description in one object. Once the spec list is ready, pass it to + * \ref AQNWB::NWB::NWBFile::createEventsTable "NWBFile::createEventsTable" together + * with the table name, description, and an optional source description: + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_create_table_snippet + * + * \note + * If you do not require custom columns (or want to create them later via + * \ref AQNWB::NWB::EventsTable::addColumn "addColumn"), then you can bypass the + * creation of the table configuration and instead configure the table directly + * via \ref AQNWB::NWB::NWBFile::createEventsTable "NWBFile::createEventsTable". + * + * \subsection events_start 4. Start the recording + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_start_snippet + * + * \note + * When using \ref AQNWB::IO::HDF5::HDF5IO "HDF5IO", calling `startRecording` enables + * \ref hdf5io_swmr "SWMR mode" by default. No new datasets or groups can be added + * after this point unless the file is closed and reopened. + * + * \section events_rowbased Pattern 1: Row-based acquisition + * + * Use this pattern when events are detected one at a time during an ongoing recording + * session and the total event count is not known in advance. + * + * \subsection events_rowbased_addrow Add rows + * + * Call \ref AQNWB::NWB::DynamicTable::addRow "addRow" to append a single event, or + * \ref AQNWB::NWB::DynamicTable::addRows "addRows" to append several events at once. + * Each row is expressed as a + * \ref AQNWB::NWB::DynamicTable::RowData "DynamicTable::RowData", which is an + * `std::unordered_map` that maps column names to typed scalar + * values. Row IDs are auto-generated (0, 1, 2, …) when not explicitly supplied. + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_write_snippet + * + * \note + * `addRow` and `addRows` write data immediately to the underlying HDF5 datasets by + * extending the chunked arrays one row (or batch of rows) at a time. Call + * `io->flush()` at any point to ensure data are moved to disk. + * + * \subsection events_rowbased_stop Stop the recording + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_rowbased_stop_snippet + * + * \section events_colbased Pattern 2: Column-based (bulk) acquisition + * + * Use this pattern when all event data are collected in memory first and can be + * written to the file in a single pass. This is common for post-acquisition or + * offline processing pipelines or when events are buffered and flushed at the end of a trial. + * + * \subsection events_colbased_adddata Add data to columns + * + * After `startRecording`, retrieve each column object via the corresponding + * `read*Column()` accessor on the table, then call + * \ref AQNWB::NWB::Data::recordData "recordData" on the column and + * \ref AQNWB::IO::BaseRecordingData::writeDataBlock "writeDataBlock" to write the + * full column vector in one call. Finally, set the row IDs with + * \ref AQNWB::NWB::DynamicTable::setRowIDs "setRowIDs". + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_colbased_create_table_snippet + * + * Custom columns that were not part of the original spec list can be added after + * table creation but **before** `startRecording()` using + * \ref AQNWB::NWB::DynamicTable::addColumn "addColumn". Create a + * \ref AQNWB::NWB::VectorData "VectorData" object, initialize it with an + * \ref AQNWB::IO::ArrayDataSetConfig "ArrayDataSetConfig", and pass it to + * `addColumn`: + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_colbased_addcolumn_snippet + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_colbased_start_snippet + * + * Write all columns — including any custom ones — after `startRecording`: + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_colbased_write_snippet + * + * \subsection events_colbased_stop Stop the recording + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_colbased_stop_snippet + * + * \section events_meanings Annotating columns with a MeaningsTable + * + * When an \ref AQNWB::NWB::EventsTable "EventsTable" column stores **coded + * values** (e.g., integer event-type codes, stimulus IDs, or category labels), + * a \ref AQNWB::NWB::MeaningsTable "MeaningsTable" can be attached to that + * column to document what each code means. The MeaningsTable lists every + * possible value together with a human-readable description, making the data + * self-documenting and easier to interpret. + * + * \subsection events_meanings_setup Setup: create the EventsTable with a coded column + * + * First, create the \ref AQNWB::NWB::EventsTable "EventsTable" as usual and + * include the column whose values you want to annotate. In this example we + * add an integer `event_type` column: + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_create_table_snippet + * + * \subsection events_meanings_create Create the MeaningsTable + * + * Before calling `startRecording()`, call + * \ref AQNWB::NWB::DynamicTable::createMeaningsTable "createMeaningsTable" + * on the \ref AQNWB::NWB::EventsTable "EventsTable" (which is a + * \ref AQNWB::NWB::DynamicTable "DynamicTable") and pass the name of the + * column to annotate. The method automatically: + * + * - creates the \ref AQNWB::NWB::MeaningsTable "MeaningsTable" at the path + * `meanings_tables/_meanings` inside the table group, + * - adds `value` and `meaning` columns with the correct data type, and + * - links the \ref AQNWB::NWB::MeaningsTable "MeaningsTable" back to the + * target \ref AQNWB::NWB::VectorData "VectorData" column. + * + * Since \ref AQNWB::NWB::MeaningsTable "MeaningsTable" is a subclass of + * \ref AQNWB::NWB::DynamicTable "DynamicTable", it supports the same + * approaches to add custom columns via `addColumn` + * as well as the same row- and column-based write patterns described above. + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_create_meanings_table_snippet + * + * \subsection events_meanings_start Start the recording + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_start_snippet + * + * \subsection events_meanings_write_events Write event data + * + * Write the event rows to the \ref AQNWB::NWB::EventsTable "EventsTable" + * using `addRow` / `addRows` as usual: + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_write_events_snippet + * + * \subsection events_meanings_write_meanings Write meanings data + * + * Then populate the \ref AQNWB::NWB::MeaningsTable "MeaningsTable" by writing + * the `value` and `meaning` columns. All possible values of the coded + * column may be listed, even if they do not appear in the recorded data. + * Since meanings only list values, we do not need to record the events first, + * i.e., if the meanings data is known beforehand, then we can also populate the + * MeaningsTable before writing the data to the main table. + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_write_meanings_snippet + * + * \subsection events_meanings_stop Stop the recording + * + * \snippet tests/examples/testEventsWorkflowExamples.cpp example_events_meanings_stop_snippet + * + * \section events_tips Tips and best practices + * + * - **One table per event type.** Keep event types with different metadata, e.g., lick events, + * TTL, or reward deliveries in separate \ref AQNWB::NWB::EventsTable "EventsTable" instances so + * that every row in a table shares the same column schema. + * - **Choose the right chunk size.** For row-based acquisition, a `rowChunkSize` of + * 100–1000 may be a good trade-off between write and read + * performance. Larger chunks reduce overhead when reading but can waste space if the + * chunks are not aligned well with rows of the table. + * - **Avoid many small write operations:** Depending on the number and rate of events, + * it may be useful to collect multiple rows in memory when using row-based acquisition + * in order to use `addRows` to write larger blocks of data to avoid many + * small I/O operations and repeated updates to the same chunks in the file. + * - **Customize the column layout.** Call + * \ref AQNWB::NWB::EventsTable::createDefaultDataSpecs "EventsTable::createDefaultDataSpecs" + * to obtain the default spec vector, modify individual entries (e.g., change the + * chunk size of a specific column), and pass the modified vector to + * \ref AQNWB::NWB::NWBFile::createEventsTable "NWBFile::createEventsTable". + * - **Add custom columns.** After table creation (but before `startRecording`), call + * \ref AQNWB::NWB::DynamicTable::addColumn "DynamicTable::addColumn" to attach + * additional \ref AQNWB::NWB::VectorData "VectorData" columns for any per-event + * metadata not covered by the built-in columns. + * - **Document coded columns with a MeaningsTable.** When a column stores integer + * codes or other enumerated values, call + * \ref AQNWB::NWB::DynamicTable::createMeaningsTable "createMeaningsTable" + * before `startRecording` to attach a + * \ref AQNWB::NWB::MeaningsTable "MeaningsTable" that maps every possible code + * to a human-readable label. You may list **all** possible values in the MeaningsTable, + * even if a particular allowed value does not appear in the recorded data. + */ diff --git a/docs/pages/userdocs/read.dox b/docs/pages/userdocs/read.dox index 045e8917c..3dcfb40e3 100644 --- a/docs/pages/userdocs/read.dox +++ b/docs/pages/userdocs/read.dox @@ -184,6 +184,67 @@ * * \snippet tests/examples/test_ecephys_data_read.cpp example_use_std_variant_to_compute_on_data * + * \subsection read_example_dynamic_table Reading data from DynamicTable objects + * + * First, let's get the table we want to read from. In this example, we'll use the `InvalidTimes` table. + * We can also see the standard approach of reading a single column's data, which is efficient when you + * only need data from one or a few columns. + * + * \snippet tests/examples/test_ecephys_data_read.cpp example_read_dynamic_table_rows_setup_snippet + * + * In addition to reading individual columns, \ref AQNWB::NWB::DynamicTable::readRows "DynamicTable::readRows(start, count, colNames)" + * provides a convenient method for reading a consecutive range of rows from the table. Note, `start` + * as the starting row index is *inclusive*, while `count` is the number of rows to read. + * `colNames` is optional and allows reading of a subset of columns. + * + * This method returns a `std::vector`, where each \ref AQNWB::Types::RowData "RowData" is a map + * of column names to their corresponding \ref AQNWB::Types::CellValue "CellValue" values for that row. + * + * Now, let's use the `readRows` approach to read a slice of rows (e.g., rows 0 to 2). This is convenient + * when you need to process data row-by-row across multiple columns. + * + * \snippet tests/examples/test_ecephys_data_read.cpp example_read_dynamic_table_rows_all_cols_snippet + * + * \note + * The \ref AQNWB::Types::CellValue "CellValue" struct holds an `std::variant`. \ref AQNWB::Types::CellValue "CellValue" + * provides an implicit conversion operator for convenient shorthand syntax when extracting + * values (e.g., `float val = row.at("col");`). However, because it relies on implicit conversion, it + * could lead to ambiguous function calls or unexpected behavior if the compiler infers the wrong type. + * For a safer and more explicit approach, it is recommended to use the `get()` method + * (e.g., `float val = row.at("col").get();`). However, in either approach, a mismatch + * between the underlying type and the target type will result in a `std::bad_variant_access` + * runtime error. For additional safety: + * - Check before you get via \ref AQNWB::Types::CellValue.holds_alternative "CellValue.holds_alternative": + * You can check if the variant holds a specific type before trying to extract it. If it returns true, + * `std::get` is guaranteed to succeed. + * - **Get a pointer via `std::get_if`:** Instead of `std::get` (which throws on failure), you can use + * `std::get_if`. It returns a pointer to the value if the type matches, or a `nullptr` if it doesn't + * so you then check if the pointer is null. + * - **Handle all types via `std::visit`:** Using `std::visit` you provide a function + * (or a generic lambda with `auto`) that can accept any of the possible types, and `std::visit` + * will automatically call your function with the correct type that is currently stored. This + * eliminates the need to guess the type and prevents the exception entirely. + * + * You can also specify a subset of columns to read by passing a vector of column names as the third + * argument to `readRows`. This can improve performance if you only need data from specific columns. + * + * \snippet tests/examples/test_ecephys_data_read.cpp example_read_dynamic_table_rows_specific_cols_snippet + * + * Finally, you can easily print the contents of a `DynamicTable` using the + * \ref AQNWB::NWB::DynamicTable::toString "DynamicTable::toString" method. This is useful for debugging + * and inspecting the table's contents. + * + * \snippet tests/examples/test_ecephys_data_read.cpp example_read_dynamic_table_to_string_snippet + * + * This will output the table contents in a readable CSV format: + * + * \code + * InvalidTimes table contents: + * id,start_time,stop_time,tags + * 0,1.000000,2.000000,"[device_error, user_error]" + * 1,3.000000,4.000000,"[external_interference]" + * 2,5.000000,6.000000,"[lost_connection, external_interference, user_error]" + * \endcode * * \section read_further_reading Further reading * - \ref read_design_page discusses the underlying software design of AqNWB for data read diff --git a/docs/pages/userdocs/time_intervals.dox b/docs/pages/userdocs/time_intervals.dox new file mode 100644 index 000000000..5ba42e5a5 --- /dev/null +++ b/docs/pages/userdocs/time_intervals.dox @@ -0,0 +1,166 @@ +/** + * \page time_intervals Annotating Time Intervals ⏱️ + * + * \tableofcontents + * + * \section time_intervals_overview Overview + * + * AqNWB provides the \ref AQNWB::NWB::TimeIntervals "TimeIntervals" type for recording + * time intervals (such as experimental epochs, trials, or invalid times), where each interval + * is defined by a `start_time` and a `stop_time`. Additional user-defined columns can be + * appended to store any per-interval metadata. + * + * NWB provides several standard `TimeIntervals` tables that can be created via convenience methods on `NWBFile`: + * - **epochs**: \ref AQNWB::NWB::NWBFile::createEpochs "NWBFile::createEpochs" for storing time intervals marking + * coarse-grained experimental phases or subdivisions of a recording session, such as baseline, task, rest, or + * sleep stages + * - **trials**: \ref AQNWB::NWB::NWBFile::createTrials "NWBFile::createTrials" for storing time intervals + * corresponding to repeated experimental units with consistent structure, such as individual + * stimulus-response-reward cycles. + * - **invalid_times**: \ref AQNWB::NWB::NWBFile::createInvalidTimes "NWBFile::createInvalidTimes" for indicating + * time intervals that should be removed from analysis. + * + * AqNWB supports two complementary acquisition patterns: + * + * | Pattern | When to use | + * |---------|-------------| + * | **Row-based** (`addRow` / `addRows`) | Intervals arrive one at a time during acquisition; the total count is not known in advance. | + * | **Column-based** (write full column vectors) | All interval data are available in memory at once and can be written in a single bulk call per column. Add additional complete columns after acquisition is completed. | + * + * Both patterns share the same setup steps (I/O, NWBFile, table creation, and + * `startRecording`) and differ only in how data are written to the table. + * The two patterns are not exclusive and hybrid approaches are possible, + * but require caution. In particular when using column-based append, we must + * ensure that all columns have been appropriately updated such that + * all columns have indeed the same number of rows before we use `addRow` or + * `addRows` to avoid inconsistent data. + * + * \par + * \note + * Chunked storage is **required** for when the total number of rows is not known in advance + * and we need to expand the columns at runtime. The `rowChunkSize` parameter of the + * `createDefaultDataSpecs` controls how many rows are stored per HDF5 chunk. In practice, + * the number of rows per chunk should typically be the same for all columns of a table. + * The chunking behavior can also be customized via the individual `DataSpec` of the columns. + * Avoid creating too small chunks (e.g,. single rows) as well as too large chunks + * (e.g., millions of rows per chunk). + * + * \par + * \note + * `TimeIntervals` is intended for storing general annotations of time ranges. Depending on the application + * (e.g., when intervals are generated by data acquisition or automatic data processing), it can be useful + * to describe intervals (or instantaneous events) in time as an \ref events "EventsTable". + * + * \section time_intervals_setup Setup: I/O, NWBFile, and table creation + * + * \subsection time_intervals_io 1. Create the I/O object + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_io_snippet + * + * \subsection time_intervals_nwbfile 2. Create the NWBFile + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_nwbfile_snippet + * + * \subsection time_intervals_configure_table 3. Configure and create the TimeIntervals table + * + * Before creating the table, build a list of column specifications using + * \ref AQNWB::NWB::TimeIntervals::createDefaultDataSpecs "TimeIntervals::createDefaultDataSpecs". + * This factory returns the default set of columns (always including the `id`, `start_time`, + * and `stop_time` columns) and allows you to decide to include the optional `tags` + * column and control the HDF5 chunk size used for each column. + * + * To add a **custom column** to the table, create a + * \ref AQNWB::NWB::VectorData::DataSpec "VectorData::DataSpec" (or a spec for any + * other \ref AQNWB::NWB::VectorData "VectorData" subtype) and push it onto the spec + * vector before passing it to + * \ref AQNWB::NWB::NWBFile::createTimeIntervals "NWBFile::createTimeIntervals". + * The spec bundles the column name, dataset configuration (data type, initial size, + * chunk size), and description in one object. Once the spec list is ready, pass it to + * \ref AQNWB::NWB::NWBFile::createTimeIntervals "NWBFile::createTimeIntervals" together + * with the table name and description: + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_create_table_snippet + * + * \note + * If you do not require custom columns (or want to create them later via + * \ref AQNWB::NWB::TimeIntervals::addColumn "addColumn)", then you can bypass the + * creation of the table configuration and instead configure the table directly + * via \ref AQNWB::NWB::NWBFile::createTimeIntervals "NWBFile::createTimeIntervals" + * or the convenience methods \ref AQNWB::NWB::NWBFile::createEpochs "NWBFile::createEpochs", + * \ref AQNWB::NWB::NWBFile::createTrials "NWBFile::createTrials", and + * \ref AQNWB::NWB::NWBFile::createInvalidTimes "NWBFile::createInvalidTimes". + * + * \subsection time_intervals_start 4. Start the recording + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_start_snippet + * + * \note + * When using \ref AQNWB::IO::HDF5::HDF5IO "HDF5IO", calling `startRecording` enables + * \ref hdf5io_swmr "SWMR mode" by default. No new datasets or groups can be added + * after this point unless the file is closed and reopened. + * + * \section time_intervals_rowbased Pattern 1: Row-based acquisition + * + * Use this pattern when intervals are detected one at a time during an ongoing recording + * session and the total interval count is not known in advance. + * + * \subsection time_intervals_rowbased_addrow Add rows + * + * Call \ref AQNWB::NWB::DynamicTable::addRow "addRow" to append a single interval, or + * \ref AQNWB::NWB::DynamicTable::addRows "addRows" to append several intervals at once. + * Each row is expressed as a + * \ref AQNWB::NWB::DynamicTable::RowData "DynamicTable::RowData", which is an + * `std::unordered_map` that maps column names to typed scalar + * or vector values. Row IDs are auto-generated (0, 1, 2, …) when not explicitly supplied. + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_write_snippet + * + * \note + * `addRow` and `addRows` write data immediately to the underlying HDF5 datasets by + * extending the chunked arrays one row (or batch of rows) at a time. Call + * `io->flush()` at any point to ensure data are moved to disk. + * + * \subsection time_intervals_rowbased_stop Stop the recording + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_rowbased_stop_snippet + * + * \section time_intervals_colbased Pattern 2: Column-based (bulk) acquisition + * + * Use this pattern when all interval data are collected in memory first and can be + * written to the file in a single pass. This is common for post-acquistion or + * offline processing pipelines. + * + * \subsection time_intervals_colbased_adddata Add data to columns + * + * After `startRecording`, retrieve each column object via the corresponding + * `record*()` accessor on the table, then call + * \ref AQNWB::IO::BaseRecordingData::writeDataBlock "writeDataBlock" to write the + * full column vector in one call. Finally, set the row IDs with + * \ref AQNWB::NWB::DynamicTable::setRowIDs "setRowIDs". + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_colbased_create_table_snippet + * + * Custom columns that were not part of the original spec list can be added after + * table creation but **before** `startRecording()` using + * \ref AQNWB::NWB::DynamicTable::addColumn "addColumn". Create a + * \ref AQNWB::NWB::VectorData "VectorData" object, initialize it with an + * \ref AQNWB::IO::ArrayDataSetConfig "ArrayDataSetConfig", and pass it to + * `addColumn`: + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_colbased_addcolumn_snippet + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_colbased_start_snippet + * + * Write all columns — including any custom ones — after `startRecording`: + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_colbased_write_snippet + * + * \subsection time_intervals_colbased_stop Stop the recording + * + * \snippet tests/examples/testTimeIntervalsWorkflowExamples.cpp example_timeintervals_colbased_stop_snippet + * + * \section time_intervals_further_reading Further reading + * + * - **Reading DynamicTable.** See the \ref read_example_dynamic_table "Reading DynamicTable" section in the reading tutorial for details on how to read data from a `TimeIntervals` table. + * - **Tips and best practices.** See the \ref events_tips "Tips and best practices" section in the Events tutorial for general advice on chunking, writing, and customizing tables, which apply equally to `TimeIntervals`. + */ diff --git a/docs/pages/userdocs/workflow.dox b/docs/pages/userdocs/workflow.dox index 7fbd7d0bd..85ce42717 100644 --- a/docs/pages/userdocs/workflow.dox +++ b/docs/pages/userdocs/workflow.dox @@ -140,13 +140,13 @@ * \snippet tests/examples/testWorkflowExamples.cpp example_workflow_write_snippet * * \note - * We should always reuse the existing objects that are cached in + * The \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create" factory methods + * automatically cache and reuse existing objects that are registered in * \ref AQNWB::IO::RecordingObjects "RecordingObjects". Calling for example * ``ElectricalSeries::create("/acquisition/esdata0", io)`` multiple times - * for the same I/O and path will result in the creation and registration of - * effectively duplicate objects that attempt to access (and potentially modify) - * the same objects in the NWB file, hence, leading to potentially unintended - * interactions, e.g., one object overwriting changes of the other. + * for the same I/O and path will return the exact same object instance. This ensures + * that the state of the object (e.g., the recording position) is maintained and prevents + * unintended interactions from duplicate objects attempting to access the same NWB file contents. * * \subsubsection write_data_allchannels Writing all channels at once (interleaved data) * @@ -198,7 +198,7 @@ * retrieve the \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object for * writing data to the datasets. The \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" * objects for all datasets that belong to a recording object are cached by the - * \ref AQNWB::NWB::RegisteredType "RegisteredType" base class to ensure that the current + * \ref AQNWB::IO::BaseIO "BaseIO" object to ensure that the current * state of the recording process (e.g., the \ref AQNWB::IO::BaseRecordingData::getPosition "position") * are stored. With this we can manually write individual data blocks ourselves while * ensuring continuation of the recording process: @@ -208,12 +208,12 @@ * \subsection advanced_recording_workflow_reset Resetting the recording position * * If we want to clear the recording cache, we can do so by calling - * \ref AQNWB::NWB::RegisteredType::clearRecordingDataCache "electricalSeries0->clearRecordingDataCache()". - * In this case the cached \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object will be + * \ref AQNWB::IO::BaseIO::clearRecordingDataCache "io->clearRecordingDataCache()". + * In this case the cached \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" objects will be * deleted, such that the next time we request it, a new \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" * object will be constructed that will then start again writing values from the beginning of the dataset. * If we want to clear the \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object for * a single datasets, we can set `reset=true` when retrieving the * \ref AQNWB::IO::BaseRecordingData "BaseRecordingData" object (e.g., via * \ref AQNWB::NWB::TimeSeries::recordData "auto dataRecorder = electricalSeries0->recordData(true)") - */ \ No newline at end of file + */ diff --git a/resources/utils/aqnwb_utils.py b/resources/utils/aqnwb_utils.py index e1f026c3d..742a01140 100644 --- a/resources/utils/aqnwb_utils.py +++ b/resources/utils/aqnwb_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S uv run --script # /// script -# requires-python = ">=3.8" +# requires-python = ">=3.10" # dependencies = ["hdmf", "pynwb", "ruamel.yaml"] # /// diff --git a/resources/utils/generate_spec_files.py b/resources/utils/generate_spec_files.py index 9f08aa1aa..f05d76a6d 100644 --- a/resources/utils/generate_spec_files.py +++ b/resources/utils/generate_spec_files.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S uv run --script # /// script -# requires-python = ">=3.8" +# requires-python = ">=3.10" # dependencies = ["ruamel.yaml"] # /// diff --git a/resources/utils/pyproject.toml b/resources/utils/pyproject.toml index fb1db67f1..bdb1b74b5 100644 --- a/resources/utils/pyproject.toml +++ b/resources/utils/pyproject.toml @@ -4,10 +4,10 @@ build-backend = "hatchling.build" [project] name = "aqnwb-utils" -version = "0.1.0" +version = "0.2.0" description = "Command-line utilities for AqNWB development" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" license = {text = "BSD-3-Clause"} authors = [ {name = "AqNWB Development Team"} diff --git a/resources/utils/schematype_to_aqnwb.py b/resources/utils/schematype_to_aqnwb.py index a7a80eda5..b130b6fdc 100755 --- a/resources/utils/schematype_to_aqnwb.py +++ b/resources/utils/schematype_to_aqnwb.py @@ -1,6 +1,6 @@ #!/usr/bin/env -S uv run --script # /// script -# requires-python = ">=3.8" +# requires-python = ">=3.10" # dependencies = ["hdmf", "pynwb", "ruamel.yaml"] # /// """ @@ -93,8 +93,15 @@ def render_define_registered_field( {{ std::string parentPath = AQNWB::mergePaths(m_path, "{field_path}"); std::string objectPath = AQNWB::mergePaths(parentPath, objectName); - if (m_io->objectExists(objectPath)) {{ - return std::make_shared<{neurodata_type}>(objectPath, m_io); + auto ioPtr = getIO(); + if (ioPtr == nullptr) {{ + std::cerr << "IO object has been deleted. Can't read {neurodata_type}: " + << m_path << std::endl; + return nullptr; + }} + + if (ioPtr->objectExists(objectPath)) {{ + return std::make_shared<{neurodata_type}>(objectPath, ioPtr); }} return nullptr; }} @@ -629,7 +636,7 @@ def attr_init( cpp_param_var_name = param['variable_name'] re = f" // TODO: Initialize {param['path']} attribute" if param['value_fixed']: - re += f"with fixed value {param['cpp_type']} {param['cpp_default_value']}" + re += f" with fixed value {param['cpp_type']} {param['cpp_default_value']}" if is_inherited or is_overridden: re += f". This attribute is_inheritted={is_inherited}, is_overridden={is_overridden}" re += "\n" @@ -647,9 +654,9 @@ def attr_init( parent_path_str = 'm_path' # Define the create call if attr_cpp_type == "std::string": - re += f' // m_io->createAttribute({cpp_param_var_name}, {parent_path_str}, "{attr_name}");\n\n' + re += f' // ioPtr->createAttribute({cpp_param_var_name}, {parent_path_str}, "{attr_name}");\n\n' elif attr.shape is None: - re += f' // m_io->createAttribute({attr_base_type}, &{cpp_param_var_name}, {parent_path_str}, "{attr_name}");\n\n' + re += f' // ioPtr->createAttribute({attr_base_type}, &{cpp_param_var_name}, {parent_path_str}, "{attr_name}");\n\n' return re def dataset_init(dataset: Spec, param: dict, is_inherited: bool, is_overridden: bool) -> str: @@ -680,10 +687,10 @@ def dataset_init(dataset: Spec, param: dict, is_inherited: bool, is_overridden: # add example initializtion hints dataset_cpp_type = param['cpp_type'] if dataset_cpp_type == "const std::string&": - re += f" // m_io->createStringDataSet({pathVarName}, {cpp_param_var_name})\n" + re += f" // ioPtr->createStringDataSet({pathVarName}, {cpp_param_var_name})\n" else: if "ArrayDataSetConfig" in dataset_cpp_type: - re += f" // std::unique_ptr {cpp_param_var_name}Data = m_io->createArrayDataSet({cpp_param_var_name}, {pathVarName});\n" + re += f" // std::unique_ptr {cpp_param_var_name}Data = ioPtr->createArrayDataSet({cpp_param_var_name}, {pathVarName});\n" else: re += f" // create scalar dataset at {pathVarName} with {'default' if not param['value_fixed'] else 'fixed'} value {param['cpp_type']} {param['cpp_default_value']}\n" re += "\n" @@ -745,6 +752,14 @@ def dataset_init(dataset: Spec, param: dict, is_inherited: bool, is_overridden: parent_initialize_call += f" // Status parentInitStatus {parent_class_name}::initialize()" parent_initialize_call += f" // initStatus = initStatus && parentInitStatus;" + #### Render retrieval of the IO object + retrieve_io_src = """auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr << "IO object has been deleted. Can't initialize %s: " + << m_path << std::endl; + return Status::Failure; + }""" % class_name + #### Render initialization for attributes, datasets and groups initialize_fields_src = "" for param in all_initialize_params: @@ -757,7 +772,7 @@ def dataset_init(dataset: Spec, param: dict, is_inherited: bool, is_overridden: path_var_name = f"{snake_to_camel(param['path'].replace('/', '_'))}Path" initialize_fields_src += f" // TODO: Initialize {param['path']} group\n" initialize_fields_src += f' // auto {path_var_name} = AQNWB::mergePaths(m_path, "{param["path"]}");\n' - initialize_fields_src += f" // m_io->createGroup({path_var_name});\n\n" + initialize_fields_src += f" // ioPtr->createGroup({path_var_name});\n\n" else: # Typed group passed as parameter if param['optional_registered_type']: initialize_fields_src += f" // TODO: Optional RegisteredType {param['cpp_type']} passed as parameter {param['variable_name']}. Usually created after initialize.\n\n" @@ -786,6 +801,9 @@ def dataset_init(dataset: Spec, param: dict, is_inherited: bool, is_overridden: {{ Status initStatus = Status::Success; + // Retrieve the IO object + {retrieve_io_src} + {fixed_value_inits} // Call parent initialize method. {parent_initialize_call}; @@ -992,6 +1010,51 @@ def get_referenced_types(neurodata_type: Spec, type_to_namespace_map: Dict[str, return set(referenced_types) # Return unique types only +def render_forward_declarations( + referenced_types: set[str], + type_to_namespace_map: Dict[str, str], + current_type_name: str, +) -> str: + """ + Render forward declarations for referenced neurodata types. + + Forward declarations avoid circular includes when a generated parent type + references one of its generated subtypes while that subtype inherits from + the parent (e.g., DynamicTable <-> MeaningsTable in newer HDMF Common + schemas). + + Parameters: + referenced_types (set[str]): Referenced neurodata type names. + type_to_namespace_map (Dict[str, str]): Mapping of types to their namespaces. + current_type_name (str): Name of the type currently being generated. + + Returns: + str: Forward declaration blocks grouped by C++ namespace. + """ + declarations_by_namespace = {} + for referenced_type in sorted(referenced_types): + if referenced_type == current_type_name: + continue + referenced_namespace = to_cpp_namespace_name( + type_to_namespace_map.get(referenced_type, "") + ) + declarations_by_namespace.setdefault(referenced_namespace, []).append( + referenced_type + ) + + if not declarations_by_namespace: + return "" + + forward_declarations = "// Forward declarations for referenced types\n" + for referenced_namespace in sorted(declarations_by_namespace): + forward_declarations += f"namespace {referenced_namespace} {{\n" + for referenced_type in declarations_by_namespace[referenced_namespace]: + forward_declarations += f"class {referenced_type};\n" + forward_declarations += f"}} // namespace {referenced_namespace}\n" + + return forward_declarations + + def parse_schema_file(file_path: Path) -> Tuple[SpecNamespace, Dict[str, Spec], Dict[str, Path], Dict[str, str]]: """ Parse a schema file and return the namespace and data types using PyNWB. @@ -1187,17 +1250,17 @@ def generate_header_file( else: header += '#include "nwb/hdmf/base/Container.hpp"\n' - # Determine additional includes required for DEFINE_REGISTERED_FIELD macro definitions - # for groups and datasets, and links to other neurodata_types referenced via attributes + # Add forward declarations for referenced types used in generated shared_ptr + # accessors and initialize signatures. This avoids circular includes for + # schema patterns where a parent type references a child type that inherits + # from the same parent. referenced_types = get_referenced_types(neurodata_type, type_to_namespace_map) if len(referenced_types) > 0: - header += "// Includes for types that are referenced and used\n" - # Add includes for all referenced types - for ref_type in referenced_types: - ref_namespace = type_to_namespace_map.get(ref_type, namespace.name) - ref_subfolder = get_schema_subfolder_name(type_to_file_map.get(ref_type, None)) - ref_include_path = f"{ref_namespace}/{ref_subfolder}/{ref_type}.hpp" - header += f'#include "{ref_include_path}"\n' + header += render_forward_declarations( + referenced_types=referenced_types, + type_to_namespace_map=type_to_namespace_map, + current_type_name=type_name, + ) # Include the namespace header header += "// Include for the namespace schema header\n" @@ -1343,19 +1406,25 @@ def is_commented_field_def(input_field: str) -> bool: parent_class_name = parent_class.split("::")[-1] if "::" in parent_class else parent_class if is_included_type: + use_namespace = f""""{actual_cpp_namespace_name}") // TODO: Use namespace from schema header""" + # Use the known NWB and HDMF_COMMON namespace header name definitons when used as includes + if actual_cpp_namespace_name == "HDMF_COMMON": + use_namespace = "AQNWB::SPEC::HDMF_COMMON::namespaceName)" + elif actual_cpp_namespace_name == "CORE": + use_namespace = "AQNWB::SPEC::CORE::namespaceName)" header += f""" - REGISTER_SUBCLASS( - {class_name}, - {parent_class_name}, - "{actual_cpp_namespace_name}") // TODO: Use namespace from schema header - """ + REGISTER_SUBCLASS( + {class_name}, + {parent_class_name}, + {use_namespace} + """ else: header += f""" - REGISTER_SUBCLASS( - {class_name}, - {parent_class_name}, - AQNWB::SPEC::{actual_cpp_namespace_name}::namespaceName) - """ + REGISTER_SUBCLASS( + {class_name}, + {parent_class_name}, + AQNWB::SPEC::{actual_cpp_namespace_name}::namespaceName) + """ header += f""" }}; @@ -1367,7 +1436,11 @@ def is_commented_field_def(input_field: str) -> bool: def generate_implementation_file( - namespace: SpecNamespace, neurodata_type: Spec, all_types: Dict[str, Spec], type_to_namespace_map: Dict[str, str] + namespace: SpecNamespace, + neurodata_type: Spec, + all_types: Dict[str, Spec], + type_to_file_map: Dict[str, Path], + type_to_namespace_map: Dict[str, str], ) -> str: """ Generate C++ implementation file for a neurodata type. @@ -1376,6 +1449,7 @@ def generate_implementation_file( namespace (SpecNamespace): The namespace object. neurodata_type (Spec): The neurodata type spec. all_types (Dict[str, Spec]): A dictionary of all neurodata types. + type_to_file_map (Dict[str, Path]): Mapping of types to their source schema files. type_to_namespace_map (Dict[str, str]): Mapping of types to their namespaces. Returns: @@ -1415,9 +1489,22 @@ def generate_implementation_file( parent_neurodata_type=parent_neurodata_type_spec ) + referenced_type_includes = "" + referenced_types = get_referenced_types(neurodata_type, type_to_namespace_map) + if referenced_types: + referenced_type_includes += "// Includes for referenced types\n" + for ref_type in sorted(referenced_types): + if ref_type == class_name: + continue + ref_namespace = type_to_namespace_map.get(ref_type, namespace.name) + ref_subfolder = get_schema_subfolder_name(type_to_file_map.get(ref_type, None)) + ref_include_path = f"{ref_namespace}/{ref_subfolder}/{ref_type}.hpp" + referenced_type_includes += f'#include "{ref_include_path}"\n' + # Start building the implementation file impl = f"""#include "{class_name}.hpp" #include "Utils.hpp" +{referenced_type_includes} using namespace {cpp_namespace_name}; using namespace AQNWB::IO; @@ -1816,7 +1903,7 @@ def main(args) -> None: cpp_file_name = f"{class_name}.cpp" logger.info(f" Generating implementation file: {cpp_file_name}") impl_file = generate_implementation_file( - namespace, neurodata_type, neurodata_types, type_to_namespace_map + namespace, neurodata_type, neurodata_types, type_to_file_map, type_to_namespace_map ) impl_path = type_output_dir / cpp_file_name with open(impl_path, "w") as f: diff --git a/src/Types.hpp b/src/Types.hpp index c4b59859d..b12be455b 100644 --- a/src/Types.hpp +++ b/src/Types.hpp @@ -1,9 +1,14 @@ #pragma once #include +#include #include #include +#include #include +#include +#include +#include #include namespace AQNWB @@ -15,122 +20,356 @@ class Channel; /** * @brief Provides definitions for various types used in the project. */ -class Types +namespace Types { -public: - /** - * @brief Represents the status of an operation. - */ - enum Status - { - Success = 1, - Failure = -1 - }; +/** + * @brief Represents the status of an operation. + */ +enum Status +{ + Success = 1, + Failure = -1 +}; + +/** + * @brief Overloaded && operator for Status enum + * @param lhs Left-hand side Status + * @param rhs Right-hand side Status + * @return Success if both statuses are Success, Failure otherwise + */ +inline Status operator&&(Status lhs, Status rhs) +{ + return (lhs == Success && rhs == Success) ? Success : Failure; +} + +/** + * @brief Overloaded || operator for Status enum + * @param lhs Left-hand side Status + * @param rhs Right-hand side Status + * @return Success if either status is Success, Failure otherwise + */ +inline Status operator||(Status lhs, Status rhs) +{ + return (lhs == Success || rhs == Success) ? Success : Failure; +} + +/** + * @brief Types of object used in the NWB schema + */ +enum StorageObjectType +{ + Group = 0, + Dataset = 1, + Attribute = 2, + Undefined = -1 +}; + +/** + * @brief Convert StorageObjectType enum value to string + * @param type The StorageObjectType value to convert + * @return String representation of the StorageObjectType + */ +inline std::string storageObjectTypeToString(StorageObjectType type) +{ + switch (type) { + case Group: + return "Group"; + case Dataset: + return "Dataset"; + case Attribute: + return "Attribute"; + case Undefined: + return "Undefined"; + default: + return "Unknown"; + } +} + +/** + * \brief Helper struct to check if a value is a data field, i.e., + * Dataset or Attribute + * + * This function is used to enforce constraints on templated functions that + * should only be callable for valid StorageObjectType values + */ +template +struct IsDataStorageObjectType + : std::integral_constant +{ +}; + +/** + * @brief Alias for the size type used in the project. + */ +using SizeType = size_t; + +/** + * @brief Value to use to indicate that a SizeType index is not set. + */ +inline constexpr SizeType SizeTypeNotSet = + (std::numeric_limits::max)(); + +/** + * @brief Alias for an array of size types used in the project. + */ +using SizeArray = std::vector; + +/** + * @brief Alias for a vector of channels. + */ +using ChannelVector = std::vector; + +/** + * @brief Variant data type for representing a single scalar value. + */ +using ScalarDataVariant = std::variant; + +/** + * @brief Variant data type for representing any 1D vector of scalar values. + */ +using VectorDataVariant = std::variant, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector>; + +/** + * @brief Struct to hold namespace information. + */ +struct NamespaceInfo +{ + std::string name; ///< The name of the namespace. + std::string version; ///< The version of the namespace. + + /** @brief The specVariables of the namespace. + * + * This is a vector of pairs, where each pair consists of 1) the + * name of the specification filed (e.g., "nwb.base") and 2) the + * string with the JSON specification of the format schema. + **/ + std::vector> + specVariables; ///< The specVariables of the namespace. +}; + +/** + * @brief Represents a single cell value in a DynamicTable row. + * + * A cell can hold either a scalar value (for regular columns) or a vector of + * values (for ragged array columns). This struct wraps a std::variant of both + * types and provides templated constructors to allow implicit conversion from + * both scalar and vector types. This enables clean syntax like `RowData row = + * {{"col1", 5}, {"col2", std::vector{1, 2}}}`. + * + * Note: The constructors for this struct are intentionally implicit (not marked + * `explicit`) to allow for this clean initializer list syntax. The cppcheck + * warnings for `noExplicitConstructor` are suppressed because this implicit + * conversion is a deliberate design choice for the API. Otherwise, users would + * have to explicitly wrap values in `CellValue` when constructing rows, via + * `RowData row = {{"col1", CellValue(5)}, {"col2", + * CellValue(std::vector{1, 2}))}`, which would be cumbersome. + */ +struct CellValue +{ + std::variant value; + + CellValue() = default; /** - * @brief Overloaded && operator for Status enum - * @param lhs Left-hand side Status - * @param rhs Right-hand side Status - * @return Success if both statuses are Success, Failure otherwise + * @brief Implicit constructor for scalar values. + * + * This constructor is enabled only if the input type T is constructible + * into ScalarDataVariant and is NOT a CellValue itself (to prevent hiding + * the copy/move constructors). It forwards the value to the + * ScalarDataVariant. */ - friend Status operator&&(Status lhs, Status rhs) + template + && !std::is_same_v, CellValue>>> + // cppcheck-suppress noExplicitConstructor + CellValue(T&& val) + : value(ScalarDataVariant(std::forward(val))) { - return (lhs == Success && rhs == Success) ? Success : Failure; } /** - * @brief Overloaded || operator for Status enum - * @param lhs Left-hand side Status - * @param rhs Right-hand side Status - * @return Success if either status is Success, Failure otherwise + * @brief Implicit constructor for vector values. + * + * This constructor is enabled only if the input type T is constructible + * into VectorDataVariant. It forwards the vector to the VectorDataVariant. */ - friend Status operator||(Status lhs, Status rhs) + template>, + typename = void> + // cppcheck-suppress noExplicitConstructor + CellValue(T&& val) + : value(VectorDataVariant(std::forward(val))) { - return (lhs == Success || rhs == Success) ? Success : Failure; } /** - * @brief Types of object used in the NWB schema + * @brief Implicit constructor for string literals. + * + * Required because string literals (const char*) would otherwise match the + * scalar template but fail to implicitly convert to std::string inside the + * variant. */ - enum StorageObjectType + // cppcheck-suppress noExplicitConstructor + CellValue(const char* val) + : value(ScalarDataVariant(std::string(val))) { - Group = 0, - Dataset = 1, - Attribute = 2, - Undefined = -1 - }; + } /** - * @brief Convert StorageObjectType enum value to string - * @param type The StorageObjectType value to convert - * @return String representation of the StorageObjectType + * @brief Checks if the CellValue holds a specific type. + * + * @tparam T The type to check for. + * @return true if the CellValue holds the specified type, false otherwise. */ - static std::string storageObjectTypeToString(StorageObjectType type) + template + bool holds_alternative() const { - switch (type) { - case Group: - return "Group"; - case Dataset: - return "Dataset"; - case Attribute: - return "Attribute"; - case Undefined: - return "Undefined"; - default: - return "Unknown"; + bool result = false; + if constexpr (std::is_constructible_v) { + if (std::holds_alternative(value)) { + result = result + || std::holds_alternative(std::get(value)); + } + } + if constexpr (std::is_constructible_v) { + if (std::holds_alternative(value)) { + result = result + || std::holds_alternative(std::get(value)); + } } + return result; } /** - * \brief Helper struct to check if a value is a data field, i.e., - * Dataset or Attribute + * @brief Helper method to extract the underlying value. * - * This function is used to enforce constraints on templated functions that - * should only be callable for valid StorageObjectType values + * @tparam T The expected type of the value. + * @return The value of type T. + * @throws std::bad_variant_access if the requested type does not match the + * stored type. */ - template - struct IsDataStorageObjectType - : std::integral_constant + template + const T& get() const { - }; - - /** - * @brief Alias for the size type used in the project. - */ - using SizeType = size_t; - - /** - * @brief Value to use to indicate that a SizeType index is not set. - */ - static constexpr SizeType SizeTypeNotSet = - (std::numeric_limits::max)(); - - /** - * @brief Alias for an array of size types used in the project. - */ - using SizeArray = std::vector; + if constexpr (std::is_constructible_v) { + return std::get(std::get(value)); + } else { + return std::get(std::get(value)); + } + } /** - * @brief Alias for a vector of channels. + * @brief Implicit conversion operator to extract the underlying value. + * + * This operator allows for convenient shorthand syntax when extracting values + * (e.g., `float val = row.at("col");`). However, because it relies on + * implicit conversion, it can sometimes lead to ambiguous function calls or + * unexpected behavior if the compiler infers the wrong type. + * + * For a safer and more explicit approach, it is recommended to use the + * `get()` method (e.g., `float val = row.at("col").get();`). + * + * @tparam T The expected type of the value. + * @return The value of type T. + * @throws std::bad_variant_access if the requested type does not match the + * stored type. */ - using ChannelVector = std::vector; + template + || std::is_constructible_v>> + operator T() const + { + return get(); + } /** - * @brief Struct to hold namespace information. + * @brief Convert the cell value to a string representation. + * @return A string representation of the cell value. */ - struct NamespaceInfo + std::string toString() const { - std::string name; ///< The name of the namespace. - std::string version; ///< The version of the namespace. - - /** @brief The specVariables of the namespace. - * - * This is a vector of pairs, where each pair consists of 1) the - * name of the specification filed (e.g., "nwb.base") and 2) the - * string with the JSON specification of the format schema. - **/ - std::vector> - specVariables; ///< The specVariables of the namespace. - }; + return std::visit( + [](auto&& arg) -> std::string + { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return std::visit( + [](auto&& scalarArg) -> std::string + { + using ScalarT = std::decay_t; + if constexpr (std::is_same_v) { + return "null"; + } else if constexpr (std::is_same_v) { + return scalarArg; + } else { + return std::to_string(scalarArg); + } + }, + arg); + } else if constexpr (std::is_same_v) { + return std::visit( + [](auto&& vectorArg) -> std::string + { + using VectorT = std::decay_t; + if constexpr (std::is_same_v) { + return "[]"; + } else { + std::string res = "["; + for (size_t i = 0; i < vectorArg.size(); ++i) { + if constexpr (std::is_same_v>) { + res += vectorArg[i]; + } else { + res += std::to_string(vectorArg[i]); + } + if (i < vectorArg.size() - 1) { + res += ", "; + } + } + res += "]"; + return res; + } + }, + arg); + } else { + return "Unknown"; + } + }, + value); + } }; + +/** + * @brief Represents a row of data in a DynamicTable. + * + * A row is a map from column names to cell values. + */ +using RowData = std::unordered_map; + +} // namespace Types } // namespace AQNWB diff --git a/src/Utils.hpp b/src/Utils.hpp index 3ae2ce036..13afb1729 100644 --- a/src/Utils.hpp +++ b/src/Utils.hpp @@ -12,6 +12,7 @@ #include #include +#include "Types.hpp" #include "io/BaseIO.hpp" #include "io/hdf5/HDF5IO.hpp" @@ -226,6 +227,26 @@ static inline std::shared_ptr createIO(const std::string& type, } } +/** + * @brief Check whether a path is equal to or a descendant of a parent path. + * + * A matching prefix must end at a path separator so that, for example, + * `/events2` is not considered a descendant of `/events`. + * + * @param path The path to check. + * @param parentPath The parent path. + * @return True if the path equals the parent path or is a descendant of it. + */ +static inline bool isPathOrDescendant(const std::string& path, + const std::string& parentPath) +{ + if (parentPath == "/") { + return !path.empty() && path.front() == '/'; + } + return path.compare(0, parentPath.size(), parentPath) == 0 + && (path.size() == parentPath.size() || path[parentPath.size()] == '/'); +} + /** * @brief Merge two paths into a single path, handling extra trailing and * starting "/". @@ -365,4 +386,36 @@ static inline void checkStatus(Status status, const std::string& operation) } } +/** + * @brief Appends a CellValue to a BaseDataVectorVariant buffer. + * @param buffer The buffer to append to. + * @param value The CellValue to append. + * @return Status::Success if successful, otherwise Status::Failure. + */ +static inline Status appendCellValueToBuffer( + IO::BaseDataType::BaseDataVectorVariant& buffer, + const Types::CellValue& value) +{ + return std::visit( + [&value](auto& vec) -> Status + { + using VecType = std::decay_t; + if constexpr (std::is_same_v) { + return Status::Failure; + } else { + using ElementType = typename VecType::value_type; + if (value.holds_alternative()) { + vec.push_back(value.get()); + return Status::Success; + } else if (value.holds_alternative>()) { + const auto& vals = value.get>(); + vec.insert(vec.end(), vals.begin(), vals.end()); + return Status::Success; + } + return Status::Failure; + } + }, + buffer); +} + } // namespace AQNWB diff --git a/src/io/BaseIO.cpp b/src/io/BaseIO.cpp index 61ac6f8f3..45f049c1d 100644 --- a/src/io/BaseIO.cpp +++ b/src/io/BaseIO.cpp @@ -277,8 +277,7 @@ std::unordered_map BaseIO::findTypes( } } // If the object is not a neurodata type then try to continue the search - else - { + else { // Get the list of objects inside the current group std::vector> objects = getStorageObjects(current_path, StorageObjectType::Undefined); @@ -299,6 +298,33 @@ std::unordered_map BaseIO::findTypes( return found_types; } +std::shared_ptr BaseIO::getDataSet(const std::string& path, + bool reset) +{ + if (!reset) { + auto it = m_recordingDataCache.find(path); + if (it != m_recordingDataCache.end()) { + return it->second; + } + } + auto dataset = getDataSetImpl(path); + if (dataset) { + m_recordingDataCache[path] = dataset; + } + return dataset; +} + +void BaseIO::clearRecordingDataCache() +{ + m_recordingDataCache.clear(); +} + +const std::unordered_map>& +BaseIO::getRecordingDataCache() const +{ + return m_recordingDataCache; +} + std::string BaseIO::findObject(const std::string& name, const std::string& starting_path) const { @@ -389,14 +415,7 @@ Status BaseIO::stopRecording() std::cerr << "Warning: Failed to finalize some recording objects" << std::endl; } - Status clearStatus = recording_objects->clearRecordingDataCache(); - if (clearStatus != Status::Success) { - // Log the error but continue with stopping recording - std::cerr << "Warning: Failed to clear recording data cache for some " - "recording objects" - << std::endl; - } - status = status && finalizeStatus && clearStatus; + status = status && finalizeStatus; } return status; } @@ -407,6 +426,7 @@ Status BaseIO::close() if (recording_objects) { m_recording_objects->clear(); } + this->clearRecordingDataCache(); return Status::Success; } diff --git a/src/io/BaseIO.hpp b/src/io/BaseIO.hpp index 522c80546..cc3baf976 100644 --- a/src/io/BaseIO.hpp +++ b/src/io/BaseIO.hpp @@ -97,19 +97,91 @@ class BaseDataType return type == other.type && typeSize == other.typeSize; } + // Variant data type for representing a single scalar with BaseDataType values + using BaseDataVariant = AQNWB::Types::ScalarDataVariant; + // Variant data type for representing any 1D vector with BaseDataType values - using BaseDataVectorVariant = std::variant, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector>; + using BaseDataVectorVariant = AQNWB::Types::VectorDataVariant; + + /** + * @brief Create an empty BaseDataVectorVariant matching the given type. + * + * This helper is useful for code that needs to accumulate values for an + * arbitrary runtime BaseDataType before writing them in batch. + * + * @param dataType The runtime data type to create an empty buffer for. + * @return An empty vector variant matching the given type, or std::monostate + * if the type is unsupported. + */ + static BaseDataVectorVariant createEmptyVectorVariant( + const BaseDataType& dataType) + { + switch (dataType.type) { + case T_U8: + return std::vector {}; + case T_U16: + return std::vector {}; + case T_U32: + return std::vector {}; + case T_U64: + return std::vector {}; + case T_I8: + return std::vector {}; + case T_I16: + return std::vector {}; + case T_I32: + return std::vector {}; + case T_I64: + return std::vector {}; + case T_F32: + return std::vector {}; + case T_F64: + return std::vector {}; + case T_STR: + case V_STR: + return std::vector {}; + } + return std::monostate {}; + } + + /** + * @brief Check whether a CellValue has a type compatible with this data type. + * + * Fixed- and variable-length string datasets both accept string values. + * + * @param value The cell value to check. + * @return True if the value's scalar or vector type matches this data type. + */ + bool isCompatibleCellValue(const Types::CellValue& value) const + { + const BaseDataVectorVariant expectedValues = + createEmptyVectorVariant(*this); + if (std::holds_alternative(expectedValues)) { + return false; + } + + const SizeType expectedTypeIndex = expectedValues.index(); + return std::visit([expectedTypeIndex](const auto& values) + { return values.index() == expectedTypeIndex; }, + value.value); + } + + /** + * @brief Check whether a vector buffer has a type compatible with this data + * type. + * + * Fixed- and variable-length string datasets both accept string buffers. + * + * @param buffer The vector buffer to check. + * @return True if the buffer's element type matches this data type. + */ + bool isCompatibleVector(const BaseDataVectorVariant& buffer) const + { + const BaseDataVectorVariant expectedBuffer = + createEmptyVectorVariant(*this); + return !std::holds_alternative(expectedBuffer) + && buffer.index() == expectedBuffer.index(); + } /** * @brief Get the BaseDataType from a std::type_index @@ -832,12 +904,28 @@ class BaseIO const BaseArrayDataSetConfig& config, const std::string& path) = 0; /** - * @brief Returns a pointer to a dataset at a given path. + * @brief Returns a cached pointer to a BaseRecordingData dataset at a given + * path. * @param path The location in the file of the dataset. + * @param reset If true, bypasses the cache and fetches a new dataset via + * getDataSetImpl and updates the cache. If false, returns the cached dataset + * if it exists. * @return A shared pointer to the dataset. */ - virtual std::shared_ptr getDataSet( - const std::string& path) = 0; + std::shared_ptr getDataSet(const std::string& path, + bool reset = false); + + /** + * @brief Clears the recording data cache. + */ + void clearRecordingDataCache(); + + /** + * @brief Gets the recording data cache. + * @return A const reference to the recording data cache. + */ + const std::unordered_map>& + getRecordingDataCache() const; /** * @brief Returns the size of the dataset or attribute for each dimension. @@ -923,10 +1011,24 @@ class BaseIO bool m_opened; /** - * @brief The recording objects for tracking all RegisteredType objects used + * @brief Implementation of getDataSet to be provided by derived classes. + * @param path The location in the file of the dataset. + * @return A shared pointer to the dataset. + */ + virtual std::shared_ptr getDataSetImpl( + const std::string& path) = 0; + + /** + * @brief The recording objects collection to manage the recording state * for recording associated with this IO object. */ std::shared_ptr m_recording_objects; + + /** + * @brief Cache of BaseRecordingData objects to retain recording state. + */ + std::unordered_map> + m_recordingDataCache; }; /** diff --git a/src/io/RecordingObjects.cpp b/src/io/RecordingObjects.cpp index de4241757..80c1cfb24 100644 --- a/src/io/RecordingObjects.cpp +++ b/src/io/RecordingObjects.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include "io/RecordingObjects.hpp" @@ -21,28 +22,39 @@ RecordingObjects::~RecordingObjects() {} SizeType RecordingObjects::getRecordingIndex( const std::shared_ptr& object) const { - // Check if object already exists in the vector - for (SizeType i = 0; i < m_recording_objects.size(); ++i) { - // compares identity of the raw pointer for the RegisteredType object - if (m_recording_objects[i].get() == object.get()) { - return i; // Return existing index - } + if (!object) { + return AQNWB::Types::SizeTypeNotSet; + } + auto it = m_path_to_index.find(object->getPath()); + if (it != m_path_to_index.end()) { + return it->second; } - // Return sentinel value for failure - return std::numeric_limits::max(); + return AQNWB::Types::SizeTypeNotSet; } SizeType RecordingObjects::addRecordingObject( const std::shared_ptr& object) { - // Check if object already exists in the vector - SizeType objectIndex = getRecordingIndex(object); - if (isValidIndex(objectIndex)) { - return objectIndex; // Return existing index if found + if (!object) { + return AQNWB::Types::SizeTypeNotSet; + } + + // Check if object already exists in the map + auto it = m_path_to_index.find(object->getPath()); + if (it != m_path_to_index.end()) { + // Update the object in the vector if it's a different instance + if (m_recording_objects[it->second].get() != object.get()) { + throw std::runtime_error( + "Attempting to add a different instance of a RegisteredType with the " + "same path."); + } + return it->second; } else { // If not found, add it and return the new index m_recording_objects.push_back(object); - return m_recording_objects.size() - 1; + SizeType newIndex = m_recording_objects.size() - 1; + m_path_to_index[object->getPath()] = newIndex; + return newIndex; } } @@ -59,11 +71,11 @@ RecordingObjects::getRecordingObject(const SizeType& objectInd) std::shared_ptr RecordingObjects::getRecordingObject(const std::string& path) const { - auto it = std::find_if(m_recording_objects.begin(), - m_recording_objects.end(), - [&path](const auto& obj) - { return obj && obj->getPath() == path; }); - return (it != m_recording_objects.end()) ? *it : nullptr; + auto it = m_path_to_index.find(path); + if (it != m_path_to_index.end()) { + return m_recording_objects[it->second]; + } + return nullptr; } Status RecordingObjects::finalize() @@ -80,26 +92,6 @@ Status RecordingObjects::finalize() return overallStatus; } -Status RecordingObjects::clearRecordingDataCache() -{ - Status overallStatus = Status::Success; - - // Call clearRecordingDataCache on all RegisteredType objects in the - // collection - for (auto& object : m_recording_objects) { - if (object) { - try { - object->clearRecordingDataCache(); - } catch (const std::exception& e) { - std::cerr << "Error clearing recording data cache for object at path: " - << object->getPath() << ". Error: " << e.what() << std::endl; - overallStatus = Status::Failure; - } - } - } - return overallStatus; -} - std::string RecordingObjects::toString() const { std::stringstream result; diff --git a/src/io/RecordingObjects.hpp b/src/io/RecordingObjects.hpp index cccad9985..167d858a5 100644 --- a/src/io/RecordingObjects.hpp +++ b/src/io/RecordingObjects.hpp @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include "Channel.hpp" #include "Types.hpp" #include "nwb/base/TimeSeries.hpp" @@ -64,8 +67,6 @@ class RecordingObjects /** * @brief Gets the RegisteredType object from the recording objects collection * based on the path of the object. - * NOTE: This function performs a linear search. If for some reason multiple - * objects with the same path exist, the first one found will be returned. * @param path The path of the object to retrieve. * @return Shared pointer to the RegisteredType object if found, nullptr * otherwise. @@ -86,7 +87,11 @@ class RecordingObjects /** * @brief Clear the recording objects collection. */ - void clear() { m_recording_objects.clear(); } + void clear() + { + m_recording_objects.clear(); + m_path_to_index.clear(); + } /** * @brief Finalize all RegisteredType objects managed by this RecordingObjects @@ -95,14 +100,6 @@ class RecordingObjects */ Status finalize(); - /** - * @brief Clear recording data cache for all RegisteredType objects - * managed by this RecordingObjects instance. This method calls - * clearRecordingDataCache() on all objects in the collection. - * @return The status of the clear operation. - */ - Status clearRecordingDataCache(); - /** * @brief Get the number of recording objects */ @@ -121,6 +118,11 @@ class RecordingObjects */ std::vector> m_recording_objects; + /** + * @brief Map from object path to its index in m_recording_objects + */ + std::map m_path_to_index; + /** * @brief The name of the collection of recording objects */ diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index bc2930baf..a320fbc73 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -1534,7 +1534,7 @@ AQNWB::IO::BaseDataType HDF5IO::getStorageObjectDataType( + ". Groups do not have data types."); } -std::shared_ptr HDF5IO::getDataSet( +std::shared_ptr HDF5IO::getDataSetImpl( const std::string& path) { std::unique_ptr data; diff --git a/src/io/hdf5/HDF5IO.hpp b/src/io/hdf5/HDF5IO.hpp index 7aaaeaf20..2b2f1b2e2 100644 --- a/src/io/hdf5/HDF5IO.hpp +++ b/src/io/hdf5/HDF5IO.hpp @@ -342,14 +342,6 @@ class HDF5IO : public BaseIO const IO::BaseArrayDataSetConfig& config, const std::string& path) override; - /** - * @brief Returns a pointer to a dataset at a given path. - * @param path The location in the file of the dataset. - * @return A shared pointer to the dataset. - */ - std::shared_ptr getDataSet( - const std::string& path) override; - /** * @brief Returns the size of the dataset or attribute for each dimension. * @param path The location of the dataset or attribute in the file @@ -458,6 +450,14 @@ class HDF5IO : public BaseIO static H5::DataType getH5Type(IO::BaseDataType type); protected: + /** + * @brief Implementation of getDataSet to be provided by derived classes. + * @param path The location in the file of the dataset. + * @return A shared pointer to the dataset. + */ + std::shared_ptr getDataSetImpl( + const std::string& path) override; + /** * @brief Creates a new group if it does not exist. * @param path The location in the file of the group. diff --git a/src/nwb/NWBFile.cpp b/src/nwb/NWBFile.cpp index 226a63982..c0f315fdc 100644 --- a/src/nwb/NWBFile.cpp +++ b/src/nwb/NWBFile.cpp @@ -15,7 +15,9 @@ #include "nwb/device/Device.hpp" #include "nwb/ecephys/ElectricalSeries.hpp" #include "nwb/ecephys/SpikeEventSeries.hpp" +#include "nwb/epoch/TimeIntervals.hpp" #include "nwb/file/ElectrodeGroup.hpp" +#include "nwb/file/Subject.hpp" #include "nwb/misc/AnnotationSeries.hpp" #include "spec/NamespaceRegistry.hpp" #include "spec/core.hpp" @@ -49,11 +51,13 @@ NWBFile::NWBFile(const std::string& path, std::shared_ptr io) NWBFile::~NWBFile() {} -Status NWBFile::initialize(const std::string& identifierText, - const std::string& description, - const std::string& dataCollection, - const std::string& sessionStartTime, - const std::string& timestampsReferenceTime) +Status NWBFile::initialize( + const std::string& identifierText, + const std::string& description, + const std::string& dataCollection, + const std::string& sessionStartTime, + const std::string& timestampsReferenceTime, + const std::optional& subjectSpec) { auto ioPtr = getIO(); if (!ioPtr) { @@ -88,16 +92,36 @@ Status NWBFile::initialize(const std::string& identifierText, // Check that the file is empty and initialize if it is bool fileInitialized = isInitialized(); + Status initStatus = Status::Success; if (!fileInitialized) { Status createStatus = createFileStructure(identifierText, description, dataCollection, useSessionStartTime, useTimestampsReferenceTime); - return createStatus; - } else { - return Status::Success; + initStatus = initStatus && createStatus; + } + + // Create subject group and its contents if subject metadata is provided + if (subjectSpec.has_value()) { + const std::string subjectPath = + mergePaths(NWBFile::GENERAL_PATH, "subject"); + if (!ioPtr->objectExists(subjectPath)) { + auto subject = AQNWB::NWB::Subject::create(subjectPath, ioPtr); + if (!subject) { + std::cerr << "NWBFile::initialize failed to create Subject at " + << subjectPath << "." << std::endl; + return Status::Failure; + } + Status subjectInitStatus = subject->initialize(subjectSpec.value()); + initStatus = initStatus && subjectInitStatus; + } else { + std::cerr << "Subject group already exists in the file. Skipping " + "subject initialization." + << std::endl; + } } + return initStatus; } bool NWBFile::isInitialized() const @@ -121,6 +145,9 @@ bool NWBFile::isInitialized() const "stimulus", "general", "specifications"}; + // Note, "events" and "intervals" are optional and will be + // created when the corresponding tables are created, so we don't include them + // in the required objects set // Set to keep track of found objects std::unordered_set foundObjects; @@ -169,6 +196,9 @@ Status NWBFile::createFileStructure(const std::string& identifierText, ioPtr->createGroup(NWBFile::GENERAL_PATH); ioPtr->createGroup(mergePaths(NWBFile::GENERAL_PATH, "/devices")); ioPtr->createGroup(mergePaths(NWBFile::GENERAL_PATH, "/extracellular_ephys")); + // NWBFile::EVENTS_PATH and NWBFile::INTERVALS_PATH are optional and will + // be created when the corresponding tables are created + if (dataCollection != "") { ioPtr->createStringDataSet( mergePaths(NWBFile::GENERAL_PATH, "/data_collection"), dataCollection); @@ -193,11 +223,14 @@ Status NWBFile::createFileStructure(const std::string& identifierText, ioPtr->createStringDataSet("/timestamps_reference_time", timestampsReferenceTime); ioPtr->createStringDataSet("/identifier", identifierText); + return Status::Success; } std::shared_ptr NWBFile::createElectrodesTable( - std::vector recordingArrays, bool finalizeTable) + std::vector recordingArrays, + bool finalizeTable, + const SizeType rowChunkSize) { auto ioPtr = getIO(); if (!ioPtr) { @@ -206,8 +239,31 @@ std::shared_ptr NWBFile::createElectrodesTable( return nullptr; } + if (!ioPtr->canModifyObjects()) { + std::cerr + << "NWBFile::createElectrodesTable IO object cannot modify objects." + << std::endl; + return nullptr; + } + auto electrodeTable = NWB::ElectrodesTable::create(ioPtr); - electrodeTable->initialize(); + if (!electrodeTable) { + std::cerr << "NWBFile::createElectrodesTable failed to create " + "ElectrodesTable object." + << std::endl; + return nullptr; + } + + auto specs = ElectrodesTable::createDefaultDataSpecs(rowChunkSize); + Status initStatus = electrodeTable->initialize( + "metadata about extracellular electrodes", specs); + if (initStatus != Status::Success) { + std::cerr << "NWBFile::createElectrodesTable failed to initialize " + "ElectrodesTable." + << std::endl; + return nullptr; + } + for (const auto& channelVector : recordingArrays) { electrodeTable->addElectrodes(channelVector); } @@ -232,12 +288,247 @@ std::shared_ptr NWBFile::createElectrodesTable( } } if (finalizeTable) { - electrodeTable->finalize(); + Status finalizeStatus = electrodeTable->finalize(); + if (finalizeStatus != Status::Success) { + std::cerr << "NWBFile::createElectrodesTable failed to finalize " + "ElectrodesTable." + << std::endl; + return nullptr; + } } return electrodeTable; } +Status NWBFile::requireEventsGroup(std::shared_ptr io) +{ + if (!io) { + return Status::Failure; + } + if (!io->objectExists(NWBFile::EVENTS_PATH)) { + if (!io->canModifyObjects()) { + return Status::Failure; + } + return io->createGroup(NWBFile::EVENTS_PATH); + } + return Status::Success; +} + +Status NWBFile::requireIntervalsGroup(std::shared_ptr io) +{ + if (!io) { + return Status::Failure; + } + if (!io->objectExists(NWBFile::INTERVALS_PATH)) { + if (!io->canModifyObjects()) { + return Status::Failure; + } + return io->createGroup(NWBFile::INTERVALS_PATH); + } + return Status::Success; +} + +std::shared_ptr NWBFile::createEventsTable( + const std::string& name, + const std::string& description, + const std::optional& sourceDescription, + std::optional timestampResolution, + bool createDurationColumn, + std::optional durationResolution, + const bool createAnnotationColumn, + const SizeType rowChunkSize) +{ + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "NWBFile::createEventsTable IO object has been deleted." + << std::endl; + return nullptr; + } + + if (!ioPtr->canModifyObjects()) { + std::cerr << "NWBFile::createEventsTable IO object cannot modify objects." + << std::endl; + return nullptr; + } + + std::string tablePath = AQNWB::mergePaths(NWBFile::EVENTS_PATH, name); + auto eventsTable = NWB::EventsTable::create(tablePath, ioPtr); + if (!eventsTable) { + std::cerr + << "NWBFile::createEventsTable failed to create EventsTable object." + << std::endl; + return nullptr; + } + + auto specs = EventsTable::createDefaultDataSpecs(timestampResolution, + createDurationColumn, + durationResolution, + createAnnotationColumn, + rowChunkSize); + Status initStatus = + eventsTable->initialize(description, sourceDescription, specs); + + if (initStatus != Status::Success) { + std::cerr << "NWBFile::createEventsTable failed to initialize EventsTable." + << std::endl; + return nullptr; + } + + return eventsTable; +} + +std::shared_ptr NWBFile::createEventsTable( + const std::string& name, + const std::string& description, + const std::string& sourceDescription, + const std::vector& columnSpecs) +{ + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "NWBFile::createEventsTable IO object has been deleted." + << std::endl; + return nullptr; + } + + if (!ioPtr->canModifyObjects()) { + std::cerr << "NWBFile::createEventsTable IO object cannot modify objects." + << std::endl; + return nullptr; + } + + std::string tablePath = AQNWB::mergePaths(NWBFile::EVENTS_PATH, name); + auto eventsTable = NWB::EventsTable::create(tablePath, ioPtr); + if (!eventsTable) { + std::cerr + << "NWBFile::createEventsTable failed to create EventsTable object." + << std::endl; + return nullptr; + } + + Status initStatus = + eventsTable->initialize(description, sourceDescription, columnSpecs); + + if (initStatus != Status::Success) { + std::cerr << "NWBFile::createEventsTable failed to initialize EventsTable." + << std::endl; + return nullptr; + } + + return eventsTable; +} + +std::shared_ptr NWBFile::createEpochs( + const bool createTagsColumn, const SizeType rowChunkSize) +{ + std::string description = + "Time intervals marking coarse-grained experimental phases or " + "subdivisions of a recording session, such as baseline, task, rest, or " + "sleep stage"; + return createTimeIntervals( + "epochs", description, createTagsColumn, rowChunkSize); +} + +std::shared_ptr NWBFile::createTrials( + const bool createTagsColumn, const SizeType rowChunkSize) +{ + std::string description = + "Time intervals corresponding to repeated experimental units with " + "consistent structure, such as individual stimulus-response-reward " + "cycles."; + return createTimeIntervals( + "trials", description, createTagsColumn, rowChunkSize); +} + +std::shared_ptr NWBFile::createInvalidTimes( + const bool createTagsColumn, const SizeType rowChunkSize) +{ + std::string description = + "Time intervals that should be removed from analysis"; + return createTimeIntervals( + "invalid_times", description, createTagsColumn, rowChunkSize); +} + +std::shared_ptr NWBFile::createTimeIntervals( + const std::string& name, + const std::string& description, + const bool createTagsColumn, + const SizeType rowChunkSize) +{ + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "NWBFile::createTimeIntervals IO object has been deleted." + << std::endl; + return nullptr; + } + + if (!ioPtr->canModifyObjects()) { + std::cerr << "NWBFile::createTimeIntervals IO object cannot modify objects." + << std::endl; + return nullptr; + } + + std::string tablePath = AQNWB::mergePaths(NWBFile::INTERVALS_PATH, name); + auto timeIntervals = NWB::TimeIntervals::create(tablePath, ioPtr); + if (!timeIntervals) { + std::cerr + << "NWBFile::createTimeIntervals failed to create TimeIntervals object." + << std::endl; + return nullptr; + } + + auto specs = TimeIntervals::createDefaultDataSpecs( + tablePath, rowChunkSize, createTagsColumn); + Status initStatus = timeIntervals->initialize(description, specs); + + if (initStatus != Status::Success) { + std::cerr + << "NWBFile::createTimeIntervals failed to initialize TimeIntervals." + << std::endl; + return nullptr; + } + + return timeIntervals; +} + +std::shared_ptr NWBFile::createTimeIntervals( + const std::string& name, + const std::string& description, + const std::vector& columnSpecs) +{ + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "NWBFile::createTimeIntervals IO object has been deleted." + << std::endl; + return nullptr; + } + + if (!ioPtr->canModifyObjects()) { + std::cerr << "NWBFile::createTimeIntervals IO object cannot modify objects." + << std::endl; + return nullptr; + } + + std::string tablePath = AQNWB::mergePaths(NWBFile::INTERVALS_PATH, name); + auto timeIntervals = NWB::TimeIntervals::create(tablePath, ioPtr); + if (!timeIntervals) { + std::cerr + << "NWBFile::createTimeIntervals failed to create TimeIntervals object." + << std::endl; + return nullptr; + } + + Status initStatus = timeIntervals->initialize(description, columnSpecs); + + if (initStatus != Status::Success) { + std::cerr + << "NWBFile::createTimeIntervals failed to initialize TimeIntervals." + << std::endl; + return nullptr; + } + + return timeIntervals; +} + Status NWBFile::createElectricalSeries( std::vector recordingArrays, std::vector recordingNames, diff --git a/src/nwb/NWBFile.hpp b/src/nwb/NWBFile.hpp index 9029b2aaa..1319e607a 100644 --- a/src/nwb/NWBFile.hpp +++ b/src/nwb/NWBFile.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -14,7 +15,10 @@ #include "nwb/base/NWBContainer.hpp" #include "nwb/base/ProcessingModule.hpp" #include "nwb/base/TimeSeries.hpp" +#include "nwb/epoch/TimeIntervals.hpp" +#include "nwb/event/EventsTable.hpp" #include "nwb/file/ElectrodesTable.hpp" +#include "nwb/file/Subject.hpp" #include "spec/core.hpp" /*! @@ -27,6 +31,15 @@ namespace AQNWB::NWB /** * @brief The NWBFile class provides an interface for setting up and managing * the NWB file. + * + * \note + * **Handling of Optional Paths:** + * Some groups in the NWB file structure (e.g., `/events`, `/intervals`) are + * optional and are not created by default during `initialize()`. These groups + * are created automatically on-demand when objects are added to them (e.g., via + * `createEventsTable` or `createTimeIntervals`). If you are creating objects in + * these paths manually, you can use helper methods like `requireEventsGroup` or + * `requireIntervalsGroup` to ensure the parent groups exist. */ class NWBFile : public NWBContainer { @@ -47,6 +60,10 @@ class NWBFile : public NWBContainer inline const static std::string GENERAL_PATH = "/general"; /// @brief The path to the analysis group in the NWB file inline const static std::string ANALYSIS_PATH = "/analysis"; + /// @brief The path to the root events group in the NWB file + inline const static std::string EVENTS_PATH = "/events"; + /// @brief The path to the root intervals group in the NWB file + inline const static std::string INTERVALS_PATH = "/intervals"; /** \brief Convenience factor method since the path is fixed to '/' * @param io A shared pointer to the IO object. @@ -98,12 +115,18 @@ class NWBFile : public NWBContainer * time. If empty (default), then the getCurrentTime() will be used. * @param timestampsReferenceTime ISO formatted time string with the timestamp * reference time. If empty (default), then the getCurrentTime() will be used. + * @param subjectSpec Optional subject metadata. Defaults to @c std::nullopt, + * so no Subject group is created unless a SubjectSpec is supplied. For DANDI + * validation, the supplied SubjectSpec must include subjectId, species, sex, + * and either age or dateOfBirth. */ Status initialize(const std::string& identifierText, const std::string& description = "a recording session", const std::string& dataCollection = "", const std::string& sessionStartTime = "", - const std::string& timestampsReferenceTime = ""); + const std::string& timestampsReferenceTime = "", + const std::optional& + subjectSpec = std::nullopt); /** * @brief Check if the NWB file is initialized. @@ -127,11 +150,164 @@ class NWBFile : public NWBContainer * after creation to write it to the file. If false, the * caller must call finalize() on the returned table * object to write it to the file. + * @param rowChunkSize The chunk size to use for the rows of the table. * @return The generated ElectrodesTable or nullptr if failed. */ std::shared_ptr createElectrodesTable( std::vector recordingArrays, - bool finalizeTable = true); + bool finalizeTable = true, + const SizeType rowChunkSize = 100); + + /** + * @brief Ensure that the events group exists in the NWB file. + * + * This function checks if the events group exists and creates it if it does + * not. + * + * @param io The shared pointer to the IO object. + * @return Status::Success if the group exists or was successfully created, + * Status::Failure otherwise. + */ + static Status requireEventsGroup(std::shared_ptr io); + + /** + * @brief Ensure that the intervals group exists in the NWB file. + * + * This function checks if the intervals group exists and creates it if it + * does not. + * + * @param io The shared pointer to the IO object. + * @return Status::Success if the group exists or was successfully created, + * Status::Failure otherwise. + */ + static Status requireIntervalsGroup(std::shared_ptr io); + + /** + * @brief Create an EventsTable in the EVENTS_PATH group. + * Note, this function will fail if the file is in a mode where + * new objects cannot be added, which can be checked via + * nwbfile.io->canModifyObjects() + * @param name The name of the EventsTable to create. + * @param description Description of the table. + * @param sourceDescription Optional short text description of where the + * events came from. + * @param timestampResolution The temporal resolution of the timestamps in + * seconds. + * @param createDurationColumn Whether to create the optional duration column + * (default: false). If true, the duration column will be created with the + * specified `durationResolution`. If false, the duration column will not be + * created. + * @param durationResolution The temporal resolution of the optional duration + * column in seconds. See `DurationVectorData::initialize()` for more details. + * If a std::nullopt is provided, the duration.resolution attribute of the + * duration column will not be created. Note, if `createDurationColumn` is + * false, this parameter is ignored. + * @param createAnnotationColumn Whether to create the annotation column. + * @param rowChunkSize The chunk size for the rows of the table. + * @return The generated EventsTable or nullptr if failed. + */ + std::shared_ptr createEventsTable( + const std::string& name, + const std::string& description, + const std::optional& sourceDescription = std::nullopt, + std::optional timestampResolution = std::nullopt, + bool createDurationColumn = false, + std::optional durationResolution = std::nullopt, + const bool createAnnotationColumn = false, + const SizeType rowChunkSize = 100); + + /** + * @brief Create an EventsTable in the EVENTS_PATH group using a pre-built + * column spec list. + * + * This overload is useful when the caller has already constructed a column + * spec vector (e.g. via EventsTable::createDefaultDataSpecs() followed by + * push_back() calls to add custom columns) and wants to pass it directly + * instead of using the individual resolution/flag parameters. + * + * Note, this function will fail if the file is in a mode where + * new objects cannot be added, which can be checked via + * nwbfile.io->canModifyObjects() + * @param name The name of the EventsTable to create. + * @param description Description of the table. + * @param sourceDescription Optional short text description of where the + * events came from. + * @param columnSpecs Pre-built vector of column specs (e.g. from + * EventsTable::createDefaultDataSpecs() with additional custom specs + * appended). + * @return The generated EventsTable or nullptr if failed. + */ + std::shared_ptr createEventsTable( + const std::string& name, + const std::string& description, + const std::string& sourceDescription, + const std::vector& columnSpecs); + + /** + * @brief Create the Epochs table in the INTERVALS_PATH group. + * @param createTagsColumn Whether to create the tags column. + * @param rowChunkSize The chunk size for the rows of the table. + * @return The generated TimeIntervals or nullptr if failed. + */ + std::shared_ptr createEpochs( + const bool createTagsColumn = false, const SizeType rowChunkSize = 100); + + /** + * @brief Create the Trials table in the INTERVALS_PATH group. + * @param createTagsColumn Whether to create the tags column. + * @param rowChunkSize The chunk size for the rows of the table. + * @return The generated TimeIntervals or nullptr if failed. + */ + std::shared_ptr createTrials( + const bool createTagsColumn = false, const SizeType rowChunkSize = 100); + + /** + * @brief Create the Invalid Times table in the INTERVALS_PATH group. + * @param createTagsColumn Whether to create the tags column. + * @param rowChunkSize The chunk size for the rows of the table. + * @return The generated TimeIntervals or nullptr if failed. + */ + std::shared_ptr createInvalidTimes( + const bool createTagsColumn = false, const SizeType rowChunkSize = 100); + + /** + * @brief Create a TimeIntervals table in the INTERVALS_PATH group. + * Note, this function will fail if the file is in a mode where + * new objects cannot be added, which can be checked via + * nwbfile.io->canModifyObjects() + * @param name The name of the TimeIntervals table to create (e.g., "epochs", + * "trials", "invalid_times"). + * @param description Description of the table. + * @param createTagsColumn Whether to create the tags column. + * @param rowChunkSize The chunk size for the rows of the table. + * @return The generated TimeIntervals or nullptr if failed. + */ + std::shared_ptr createTimeIntervals( + const std::string& name, + const std::string& description, + const bool createTagsColumn = false, + const SizeType rowChunkSize = 100); + + /** + * @brief Create a TimeIntervals table in the INTERVALS_PATH group using a + * pre-built column spec list. + * + * This overload is useful when the caller has already constructed a column + * spec vector (e.g. via TimeIntervals::createDefaultDataSpecs() followed by + * push_back() calls to add custom columns) and wants to pass it directly. + * + * Note, this function will fail if the file is in a mode where + * new objects cannot be added, which can be checked via + * nwbfile.io->canModifyObjects() + * @param name The name of the TimeIntervals table to create. + * @param description Description of the table. + * @param columnSpecs Pre-built vector of column specs. + * @return The generated TimeIntervals or nullptr if failed. + */ + std::shared_ptr createTimeIntervals( + const std::string& name, + const std::string& description, + const std::vector& columnSpecs); /** * @brief Create ElectricalSeries objects to record data into. @@ -185,6 +361,21 @@ class NWBFile : public NWBContainer Status createAnnotationSeries(const std::vector& recordingNames, std::vector& containerIndexes); + DEFINE_REGISTERED_FIELD(readEpochs, + TimeIntervals, + "intervals/epochs", + "Table of experimental epochs.") + + DEFINE_REGISTERED_FIELD(readTrials, + TimeIntervals, + "intervals/trials", + "Table of experimental trials.") + + DEFINE_REGISTERED_FIELD(readInvalidTimes, + TimeIntervals, + "intervals/invalid_times", + "Table of invalid times.") + DEFINE_REGISTERED_FIELD(readElectrodesTable, ElectrodesTable, ElectrodesTable::electrodesTablePath, @@ -242,6 +433,19 @@ class NWBFile : public NWBContainer Get a ProcessingModule stored in the processing group) + DEFINE_UNNAMED_REGISTERED_FIELD(readTimeIntervals, + createTimeIntervalsInstance, + TimeIntervals, + "intervals", + Get a TimeIntervals object stored in the + intervals group) + + DEFINE_UNNAMED_REGISTERED_FIELD(readEventsTable, + createEventsTableInstance, + EventsTable, + "events", + Get an EventsTable stored in the events group) + protected: /** * @brief Creates the default file structure. diff --git a/src/nwb/RegisteredType.cpp b/src/nwb/RegisteredType.cpp index 45f7438a1..45aa29d76 100644 --- a/src/nwb/RegisteredType.cpp +++ b/src/nwb/RegisteredType.cpp @@ -72,17 +72,43 @@ std::string RegisteredType::getNamespace() const return ""; } +std::shared_ptr RegisteredType::getExistingRecordingObject( + const std::string& path, std::shared_ptr io) +{ + if (io) { + auto recordingObjects = io->getRecordingObjects(); + if (recordingObjects) { + return recordingObjects->getRecordingObject(path); + } + } + return nullptr; +} + std::shared_ptr RegisteredType::create( const std::string& fullClassName, const std::string& path, std::shared_ptr io, bool fallbackToBase) { - // If no object exists for the path, or if the existing object is of a - // different type, create a new instance - // Look up the factory RegisteredType for the fullClassName the registry + // Check if the object already exists in the cache + auto existing = getExistingRecordingObject(path, io); + if (existing) { + if (existing->getFullTypeName() == fullClassName) { + return existing; + } + std::cerr << "RegisteredType::create: cached object at path " << path + << " has type " << existing->getFullTypeName() + << ", which does not match the requested type " << fullClassName + << "." << std::endl; + return nullptr; + } + + // Look up the factory RegisteredType for the fullClassName the registry auto it = getFactoryMap().find(fullClassName); if (it != getFactoryMap().end()) { + // The `first` element in the factory map is a lambda function generated by + // the `REGISTER_SUBCLASS` macro in `RegisteredType.hpp` which calls + // className::create return it->second.first(path, io); } // If the class is not found, return a base class instance by calling this @@ -112,8 +138,15 @@ std::shared_ptr RegisteredType::create( std::shared_ptr io, bool fallbackToBase) { + if (!io) { + std::cerr << "RegisteredType::create: IO object is not available." + << std::endl; + return nullptr; + } + try { - // Read the "neurodata_type" attribute + // Resolve the on-disk type before consulting the cache so a generic + // wrapper cannot be returned for a derived registered type. std::string fullClassName = io->getFullTypeName(path); // Create an instance of the corresponding RegisteredType subclass return AQNWB::NWB::RegisteredType::create( diff --git a/src/nwb/RegisteredType.hpp b/src/nwb/RegisteredType.hpp index 23b06853d..751349e4b 100644 --- a/src/nwb/RegisteredType.hpp +++ b/src/nwb/RegisteredType.hpp @@ -134,21 +134,38 @@ class RegisteredType : public std::enable_shared_from_this /** * @brief Get the cache of BaseRecordingData objects * @return A reference to the cache of BaseRecordingData objects + * @deprecated The cache is owned by the I/O so there should be no need to + * access it via RegisteredType. Use getIO()->getRecordingDataCache() instead. */ + [[deprecated( + "The cache is owned by the I/O so there should be no need to access it " + "via RegisteredType. Use getIO()->getRecordingDataCache() instead.")]] inline const std::unordered_map>& getCacheRecordingData() const { - return this->m_recordingDataCache; + if (auto io = getIO()) { + return io->getRecordingDataCache(); + } + throw std::runtime_error( + "RegisteredType::getCacheRecordingData: IO object is not available"); } /** * @brief Clear the BaseRecordingData object cache to reset the recording * state + * @deprecated The cache is owned by the I/O so there should be no need to + * access it via RegisteredType. Use getIO()->clearRecordingDataCache() + * instead. */ + [[deprecated( + "The cache is owned by the I/O so there should be no need to access it " + "via RegisteredType. Use getIO()->clearRecordingDataCache() instead.")]] inline virtual void clearRecordingDataCache() { - this->m_recordingDataCache.clear(); + if (auto io = getIO()) { + io->clearRecordingDataCache(); + } } /** @@ -183,7 +200,11 @@ class RegisteredType : public std::enable_shared_from_this getFactoryMap(); /** - * @brief Create an instance of a registered subclass by name. + * @brief Create or retrieve the canonical instance of a registered subclass. + * + * Each I/O object caches at most one canonical RegisteredType per path. If a + * cached instance exists, its full registered type must match fullClassName; + * otherwise this function logs the mismatch and returns nullptr. * * @param fullClassName The combined namespace and class name to instantiate, * i.e., namespace::class @@ -194,8 +215,8 @@ class RegisteredType : public std::enable_shared_from_this * m_defaultUnregisteredGroupTypeClass and * m_defaultUnregisteredDatasetTypeClass depending on whether the type is a * group or dataset. - * @return A unique_ptr to the created instance of the subclass, or nullptr if - * the subclass is not found. + * @return The canonical instance of the subclass, or nullptr if the subclass + * is not found or a cached instance has a different registered type. */ static std::shared_ptr create( const std::string& fullClassName, @@ -204,13 +225,14 @@ class RegisteredType : public std::enable_shared_from_this bool fallbackToBase = false); /** - * @brief Factory method to create an instance of a subclass of RegisteredType - * from file + * @brief Create or retrieve the canonical RegisteredType instance from file. * * The function: 1) reads the "namespace" and "neurodata_type" attributes at * the given path, 2) looks up the corresponding subclass of RegisteredType * for that type in the type registry 3) instantiates the subclass to - * represent the object at the path. + * represent the object at the path. The on-disk type is resolved before a + * cached object is used, ensuring that a cached instance has the same + * registered type as the file object. * * @param path The path of the registered type. * @param io A shared pointer to the IO object. @@ -220,8 +242,8 @@ class RegisteredType : public std::enable_shared_from_this * m_defaultUnregisteredDatasetTypeClass depending on whether the type is a * group or dataset. * - * @return A unique pointer to the created RegisteredType instance, or nullptr - * if creation fails. + * @return The canonical RegisteredType instance, or nullptr if creation + * fails or a cached instance has a different registered type. */ static std::shared_ptr create( const std::string& path, @@ -229,13 +251,19 @@ class RegisteredType : public std::enable_shared_from_this bool fallbackToBase = false); /** - * @brief Factory method to create an instance of a subclass of RegisteredType - * by type. + * @brief Create or retrieve a canonical instance of a RegisteredType subtype. + * + * If a cached object exists at path, it is returned only when it can be cast + * to T. A type mismatch is reported and returns nullptr rather than creating + * a second instance for the same path. Non-registered typed facades, such as + * DataTyped and VectorDataTyped, provide their own create methods and are not + * cached by this factory. * * @tparam T The subclass of RegisteredType to instantiate. * @param path The path of the container. * @param io A shared pointer to the IO object. - * @return A unique_ptr to the created instance of the subclass. + * @return The canonical instance of T, or nullptr if a cached instance has an + * incompatible type. */ template static inline std::shared_ptr create(const std::string& path, @@ -243,6 +271,17 @@ class RegisteredType : public std::enable_shared_from_this { static_assert(std::is_base_of::value, "T must be a derived class of RegisteredType"); + auto existing = getExistingRecordingObject(path, io); + if (existing) { + auto casted = std::dynamic_pointer_cast(existing); + if (casted) { + return casted; + } + std::cerr << "RegisteredType::create: cached object at path " << path + << " has type " << existing->getFullTypeName() + << ", which does not match the requested type." << std::endl; + return nullptr; + } auto result = std::shared_ptr(new T(path, io)); result->registerRecordingObject(); return result; @@ -380,6 +419,15 @@ class RegisteredType : public std::enable_shared_from_this virtual std::string findOwnedObject(const std::string& name) const; protected: + /** + * @brief Helper to get an existing recording object from the IO object. + * @param path The path of the object. + * @param io A shared pointer to the IO object. + * @return A shared pointer to the existing object, or nullptr if not found. + */ + static std::shared_ptr getExistingRecordingObject( + const std::string& path, std::shared_ptr io); + /** * @brief Constructor. * @@ -433,21 +481,6 @@ class RegisteredType : public std::enable_shared_from_this * pointer to the IO object before using it. */ std::weak_ptr m_io; - - /** - * @brief Cache for BaseRecordingData objects for datasets to retain recording - * state. - * - * This map stores shared pointers to BaseRecordingData objects that have been - * previously requested, using the field path as the key. This allows us to - * reuse the same object when it is requested multiple times, improving - * performance and more importantly, retaining the recording position so that - * we can append to the dataset from the last position that we recorded to. - * This is important for writing data to the dataset in a streaming fashion. - * The cache is mutable to allow modification in const methods. - */ - std::unordered_map> - m_recordingDataCache; }; /** @@ -632,25 +665,13 @@ public: \ false) \ { \ std::string fullPath = AQNWB::mergePaths(m_path, fieldPath); \ - if (!reset) { \ - /* Check if the dataset is already in the cache */ \ - auto it = m_recordingDataCache.find(fullPath); \ - if (it != m_recordingDataCache.end()) { \ - return it->second; \ - } \ - } \ - /* Get the dataset from IO and cache it */ \ auto ioPtr = getIO(); \ if (!ioPtr) { \ std::cerr << "IO object has been deleted. Can't access: " << fullPath \ << std::endl; \ return nullptr; \ } \ - auto dataset = ioPtr->getDataSet(fullPath); \ - if (dataset) { \ - m_recordingDataCache[fullPath] = dataset; \ - } \ - return dataset; \ + return ioPtr->getDataSet(fullPath, reset); \ } /** @@ -694,7 +715,7 @@ public: \ auto ioPtr = getIO(); \ if (ioPtr != nullptr) { \ if (ioPtr->objectExists(objectPath)) { \ - return RegisteredType::create(objectPath, ioPtr); \ + return RTYPE::create(objectPath, ioPtr); \ } \ } \ return nullptr; \ @@ -751,7 +772,7 @@ public: \ return nullptr; \ } \ if (ioPtr->objectExists(objectPath)) { \ - return RegisteredType::create(objectPath, ioPtr); \ + return RTYPE::create(objectPath, ioPtr); \ } \ return nullptr; \ } \ @@ -780,7 +801,7 @@ public: \ << objectPath << std::endl; \ return nullptr; \ } \ - return RegisteredType::create(objectPath, ioPtr); \ + return RTYPE::create(objectPath, ioPtr); \ } /** @@ -827,7 +848,7 @@ public: \ if (ioPtr != nullptr) { \ std::string objectPath = ioPtr->readReferenceAttribute(attrPath); \ if (ioPtr->objectExists(objectPath)) { \ - return RegisteredType::create(objectPath, ioPtr); \ + return RTYPE::create(objectPath, ioPtr); \ } \ } \ } catch (const std::exception& e) { \ diff --git a/src/nwb/base/TimeSeries.hpp b/src/nwb/base/TimeSeries.hpp index f76e4c748..aa736ca31 100644 --- a/src/nwb/base/TimeSeries.hpp +++ b/src/nwb/base/TimeSeries.hpp @@ -88,7 +88,7 @@ class TimeSeries : public NWBDataInterface * @param conversion Scalar to multiply each element in data to convert it to * the specified ‘unit’ * @param resolution Smallest meaningful difference between values in data, - * stored in the specified by unit + * stored in the specified unit * @param offset Scalar to add to the data after scaling by ‘conversion’ to * finalize its coercion to the specified ‘unit' * @param continuity Continuity of the data diff --git a/src/nwb/ecephys/ElectricalSeries.hpp b/src/nwb/ecephys/ElectricalSeries.hpp index e38a4fe90..bc6df1c8f 100644 --- a/src/nwb/ecephys/ElectricalSeries.hpp +++ b/src/nwb/ecephys/ElectricalSeries.hpp @@ -51,7 +51,7 @@ class ElectricalSeries : public TimeSeries * @param conversion Scalar to multiply each element in data to convert it to * the specified 'unit' * @param resolution Smallest meaningful difference between values in data, - * stored in the specified by unit + * stored in the specified unit * @param offset Scalar to add to the data after scaling by 'conversion' to * finalize its coercion to the specified 'unit' * @return The status of the initialization operation. diff --git a/src/nwb/ecephys/SpikeEventSeries.hpp b/src/nwb/ecephys/SpikeEventSeries.hpp index d8837b42d..6d06aa3ad 100644 --- a/src/nwb/ecephys/SpikeEventSeries.hpp +++ b/src/nwb/ecephys/SpikeEventSeries.hpp @@ -52,7 +52,7 @@ class SpikeEventSeries : public ElectricalSeries * @param conversion Scalar to multiply each element in data to convert it to * the specified 'unit' * @param resolution Smallest meaningful difference between values in data, - * stored in the specified by unit + * stored in the specified unit * @param offset Scalar to add to the data after scaling by 'conversion' to * finalize its coercion to the specified 'unit' */ diff --git a/src/nwb/epoch/TimeIntervals.cpp b/src/nwb/epoch/TimeIntervals.cpp new file mode 100644 index 000000000..1640a539d --- /dev/null +++ b/src/nwb/epoch/TimeIntervals.cpp @@ -0,0 +1,124 @@ +#include "nwb/epoch/TimeIntervals.hpp" + +#include "Utils.hpp" +#include "nwb/NWBFile.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(TimeIntervals) + +// Constructor +TimeIntervals::TimeIntervals(const std::string& path, + std::shared_ptr io) + : DynamicTable(path, io) +{ +} + +// TODO add addTimeseries parameter to also support construction of the +// timeseries and timeseries_index columns +std::vector TimeIntervals::createDefaultDataSpecs( + const std::string& timeIntervalsPath, + const SizeType rowChunkSize, + const bool addTagsColumn) +{ + std::vector specs = + DynamicTable::createDefaultDataSpecs(rowChunkSize); + + // Add the required start_time column + IO::ArrayDataSetConfig startTimeConfig( + IO::BaseDataType::T_F32, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "start_time", startTimeConfig, "Start time of epoch, in seconds")); + + // Add the required stop_time column + IO::ArrayDataSetConfig stopTimeConfig( + IO::BaseDataType::T_F32, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "stop_time", stopTimeConfig, "Stop time of epoch, in seconds")); + + // add the tags and tags_index columns if requested + if (addTagsColumn) { + // add the tags column spec + IO::ArrayDataSetConfig tagsConfig( + IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "tags", + tagsConfig, + "User-defined tags that identify or categorize events")); + + // add the tags_index column + IO::ArrayDataSetConfig tagsIndexConfig( + IO::BaseDataType::T_U32, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "tags_index", + tagsIndexConfig, + "Index for user-defined tags that identify or categorize events", + AQNWB::mergePaths(timeIntervalsPath, "tags"))); + } + + // TODO add the timeseries and timeseries_index columns + // if (addTimeseriesColumn) + // { + // // TODO Add the timeseries TimeSeriesReferenceVectorData column + // // TODO Add the timesries_index VectorIndex column + // } + + return specs; +} + +Status TimeIntervals::validateDataSpecs( + const std::vector& dataSpecs) const +{ + Status status = DynamicTable::validateDataSpecs(dataSpecs); + if (status != Status::Success) { + return status; + } + + status = checkRequiredColumnSpec( + "start_time", dataSpecs, IO::BaseDataType::F32); + if (status != Status::Success) { + return status; + } + + return checkRequiredColumnSpec( + "stop_time", dataSpecs, IO::BaseDataType::F32); +} + +// Initialize the object +Status TimeIntervals::initialize(const std::string& description, + const std::vector& columnSpecs) +{ + std::vector specsToUse = columnSpecs; + if (specsToUse.empty()) { + specsToUse = createDefaultDataSpecs(this->getPath()); + } + + // Retrieve the IO object + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr << "IO object has been deleted. Can't initialize TimeIntervals: " + << m_path << std::endl; + return Status::Failure; + } + + // Ensure the intervals group exists if this table is being created in the + // intervals group + if (AQNWB::isPathOrDescendant(m_path, NWBFile::INTERVALS_PATH)) { + Status requireStatus = NWBFile::requireIntervalsGroup(ioPtr); + if (requireStatus != Status::Success) { + std::cerr + << "Failed to create or verify intervals group for TimeIntervals: " + << m_path << std::endl; + return Status::Failure; + } + } + + // create group. This configures the "start_time" and "stop_time" columns + // (among others) via the DataSpec mechanism, creating and registering the + // corresponding VectorData recording objects exactly once. + Status dtStatus = DynamicTable::initialize(description, specsToUse); + + return dtStatus; +} diff --git a/src/nwb/epoch/TimeIntervals.hpp b/src/nwb/epoch/TimeIntervals.hpp new file mode 100644 index 000000000..67c6d4aea --- /dev/null +++ b/src/nwb/epoch/TimeIntervals.hpp @@ -0,0 +1,136 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/hdmf/table/DynamicTable.hpp" +// Includes for types that are referenced and used +#include "nwb/hdmf/table/ElementIdentifiers.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "nwb/hdmf/table/VectorIndex.hpp" +// #include "core/base/TimeSeriesReferenceVectorData.hpp" + +#include "spec/core.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief A container for aggregating epoch data and the TimeSeries that each + * epoch applies to. + */ +class TimeIntervals : public AQNWB::NWB::DynamicTable +{ +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + TimeIntervals(const std::string& path, std::shared_ptr io); + +public: + /** + * @brief Virtual destructor. + */ + virtual ~TimeIntervals() override {} + + /** + * @brief Validates the provided data specifications for the TimeIntervals + * table. + * @param dataSpecs The data specifications to validate. + * @return Status::Success if the specifications are valid, otherwise + * Status::Failure. + */ + Status validateDataSpecs( + const std::vector& dataSpecs) const override; + + /** + * @brief Creates the default data specs for the TimeIntervals table. + * @param timeIntervalsPath Path of the TimeIntervals table. Need to construct + * target for VectorIndex + * @param rowChunkSize The chunk size for the rows of the table. + * @param addTagsColumn Add the tags column to the spec + * @return A vector of DataSpecPtr containing the default specs. + */ + static std::vector createDefaultDataSpecs( + const std::string& timeIntervalsPath, + const SizeType rowChunkSize = 100, + const bool addTagsColumn = false); + + /** + * @brief Initializes the TimeIntervals table + * + * Initializes the TimeIntervals table by creating NWB related attributes and + * adding required columns. + * + * \note + * **Handling of Optional Paths:** + * If the TimeIntervals table is being created in the `/intervals` group + * (e.g., + * `/intervals/epochs`), this method will automatically create the + * `/intervals` group if it does not already exist using the + * `NWBFile::requireIntervalsGroup()` helper method. This is because the + * `/intervals` group is optional in the NWB file structure and as such needs + * to be created on-demand. If the TimeIntervals table is being created in a + * different path, this method will not create any parent groups, and it is + * the caller's responsibility to ensure that the parent group exists. + * + * @param description The description of the table + * @param columnSpecs The column specifications to use for initialization. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initialize(const std::string& description, + const std::vector& columnSpecs = {}); + + // Define read methods + DEFINE_REGISTERED_FIELD(readStartTime, + VectorDataTyped, + "start_time", + "Start time of epoch - in seconds.") + + DEFINE_REGISTERED_FIELD(readStopTime, + VectorDataTyped, + "stop_time", + "Stop time of epoch - in seconds.") + + DEFINE_REGISTERED_FIELD( + readTags, + VectorDataTyped, + "tags", + "User-defined tags that identify or categorize events.") + + DEFINE_REGISTERED_FIELD(readTagsIndex, + VectorIndex, + "tags_index", + "Index for tags.") + + /* + // TODO Add support for the timeseries columns requires support for + TimeSeriesReferenceVectorData DEFINE_REGISTERED_FIELD( readTimeseries, + CORE::TimeSeriesReferenceVectorData, + "timeseries", + "An index into a TimeSeries object.") + */ + /* + // TODO Add support for timeseries_index requires support for VectorIndex + DEFINE_REGISTERED_FIELD( + readTimeseriesIndex, + HDMF_COMMON::VectorIndex, + "timeseries_index", + "Index for timeseries.") + */ + + REGISTER_SUBCLASS(TimeIntervals, + DynamicTable, + AQNWB::SPEC::CORE::namespaceName) +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/event/DurationVectorData.cpp b/src/nwb/event/DurationVectorData.cpp new file mode 100644 index 000000000..258570a61 --- /dev/null +++ b/src/nwb/event/DurationVectorData.cpp @@ -0,0 +1,80 @@ +#include "DurationVectorData.hpp" + +#include "Utils.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(DurationVectorData) + +// Constructor +DurationVectorData::DurationVectorData(const std::string& path, + std::shared_ptr io) + : VectorData(path, io) +{ +} + +// Initialize the object +Status DurationVectorData::initialize(const AQNWB::IO::ArrayDataSetConfig& data, + const std::string& description, + std::optional resolution) +{ + Status initStatus = Status::Success; + + // Retrieve the IO object + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr + << "IO object has been deleted. Can't initialize DurationVectorData: " + << m_path << std::endl; + return Status::Failure; + } + + // Call parent initialize method, which initializes the description attribute + // and the data dataset. + Status parentInitStatus = VectorData::initialize(data, description); + initStatus = initStatus && parentInitStatus; + + // Initialize attributes, datasets, and groups + // Initialize unit attribute with fixed value const std::string& "seconds" + const std::string& unit = "seconds"; + Status unitStatus = ioPtr->createAttribute(unit, m_path, "unit"); + // Initialize resolution attribute + Status resolutionStatus = Status::Success; + if (resolution.has_value()) { + float resValue = resolution.value(); + resolutionStatus = ioPtr->createAttribute( + AQNWB::IO::BaseDataType::F32, &resValue, m_path, "resolution"); + } + + // Update the status to reflect the success or failure of the initialization + initStatus = initStatus && parentInitStatus && unitStatus && resolutionStatus; + // Return the final status of the initialization process + return initStatus; +} + +Status DurationVectorData::DataSpec::initialize(Data& data) const +{ + auto* durationData = dynamic_cast(&data); + if (!durationData) { + std::cerr << "DurationVectorData::DataSpec::initialize received " + "incompatible Data object" + << std::endl; + return Status::Failure; + } + return durationData->initialize( + static_cast(*this), + description, + resolution); +} + +std::shared_ptr +DurationVectorData::createDataSpec( + const std::string& name, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& description, + std::optional resolution) +{ + return std::make_shared(name, dataConfig, description, resolution); +} diff --git a/src/nwb/event/DurationVectorData.hpp b/src/nwb/event/DurationVectorData.hpp new file mode 100644 index 000000000..6ae5b955c --- /dev/null +++ b/src/nwb/event/DurationVectorData.hpp @@ -0,0 +1,104 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/hdmf/table/VectorData.hpp" +// Include for the namespace schema header +#include "spec/core.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief A 1-D VectorData that stores durations in seconds. + */ +class DurationVectorData : public AQNWB::NWB::VectorData +{ +public: + struct DataSpec : public Data::DataSpec + { + DataSpec(const std::string& datasetName, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& columnDescription, + std::optional columnResolution = std::nullopt) + : Data::DataSpec(datasetName, dataConfig) + , description(columnDescription) + , resolution(columnResolution) + { + } + + std::string description; + std::optional resolution; + + Status initialize(Data& data) const override; + }; + + /** + * @brief Virtual destructor. + */ + virtual ~DurationVectorData() override {} + + /** + * @brief Initialize the object + * @param data The configuration for the dataset + * @param description Description of the dataset + * @param resolution The temporal resolution of the durations - in seconds. + * This is typically the sampling period (1 / sampling_rate), also known as + * the clock period, of the data acquisition system from which the durations + * were recorded or derived. + */ + Status initialize(const AQNWB::IO::ArrayDataSetConfig& data, + const std::string& description, + std::optional resolution = std::nullopt); + + static std::shared_ptr createDataSpec( + const std::string& name, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& description, + std::optional resolution = std::nullopt); + + // Define read methods + DEFINE_ATTRIBUTE_FIELD( + readUnit, + std::string, + "unit", + "The unit of measurement for the durations - fixed to seconds.") + + DEFINE_ATTRIBUTE_FIELD( + readResolution, + float, + "resolution", + "The temporal resolution of the durations - in seconds. This is " + "typically the sampling period (1 / sampling_rate) - also known as the " + "clock period - of the data acquisition system from which the durations " + "were recorded or derived.") + + DEFINE_DATASET_FIELD(readData, + recordData, + float, + "", + "A 1-D VectorData that stores durations in seconds.") + + REGISTER_SUBCLASS(DurationVectorData, + VectorData, + AQNWB::SPEC::CORE::namespaceName) + +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + DurationVectorData(const std::string& path, + std::shared_ptr io); +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/event/EventsTable.cpp b/src/nwb/event/EventsTable.cpp new file mode 100644 index 000000000..1f041644e --- /dev/null +++ b/src/nwb/event/EventsTable.cpp @@ -0,0 +1,124 @@ +#include "nwb/event/EventsTable.hpp" + +#include "Utils.hpp" +#include "nwb/NWBFile.hpp" +// Includes for referenced types +#include "nwb/hdmf/table/ElementIdentifiers.hpp" +#include "nwb/hdmf/table/MeaningsTable.hpp" +#include "nwb/hdmf/table/VectorData.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(EventsTable) + +// Constructor +EventsTable::EventsTable(const std::string& path, + std::shared_ptr io) + : DynamicTable(path, io) +{ +} + +Status EventsTable::validateDataSpecs( + const std::vector& dataSpecs) const +{ + Status status = DynamicTable::validateDataSpecs(dataSpecs); + if (status != Status::Success) { + return status; + } + + return checkRequiredColumnSpec( + "timestamp", dataSpecs, IO::BaseDataType::F32); +} + +std::vector EventsTable::createDefaultDataSpecs( + std::optional timestampResolution, + bool createDurationColumn, + std::optional durationResolution, + const bool createAnnotationColumn, + const SizeType rowChunkSize) +{ + std::vector specs = + DynamicTable::createDefaultDataSpecs(rowChunkSize); + + IO::ArrayDataSetConfig timestampConfig( + IO::BaseDataType::F32, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "timestamp", + timestampConfig, + "Column containing the time that each event occurred " + "- in seconds - from the session start time.", + timestampResolution)); + + if (createDurationColumn) { + IO::ArrayDataSetConfig durationConfig( + IO::BaseDataType::F32, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "duration", + durationConfig, + "Optional column containing the duration of each event - in seconds. A " + "value of NaN can be used for events without a duration or with a " + "duration that is not yet specified.", + durationResolution)); + } + + if (createAnnotationColumn) { + IO::ArrayDataSetConfig annotationConfig( + IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "annotation", + annotationConfig, + "Column containing user annotations about events.")); + } + + return specs; +} + +// Initialize the object +Status EventsTable::initialize( + const std::string& description, + const std::optional& sourceDescription, + const std::vector& columnSpecs) +{ + Status initStatus = Status::Success; + + // Retrieve the IO object + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr << "IO object has been deleted. Can't initialize EventsTable: " + << m_path << std::endl; + return Status::Failure; + } + + std::vector specsToUse = columnSpecs; + if (specsToUse.empty()) { + specsToUse = createDefaultDataSpecs(); + } + + // Ensure the events group exists if this table is being created in the events + // group + if (AQNWB::isPathOrDescendant(m_path, NWBFile::EVENTS_PATH)) { + Status requireStatus = NWBFile::requireEventsGroup(ioPtr); + if (requireStatus != Status::Success) { + std::cerr << "Failed to create or verify events group for EventsTable: " + << m_path << std::endl; + return Status::Failure; + } + } + + // Call parent initialize method. + Status parentInitStatus = DynamicTable::initialize(description, specsToUse); + + // Initialize attributes, datasets, and groups + // Create the source_description attribute if provided + Status sourceDescStatus = Status::Success; + if (sourceDescription.has_value() && !sourceDescription.value().empty()) { + sourceDescStatus = ioPtr->createAttribute( + sourceDescription.value(), m_path, "source_description"); + } + + // Combine all statuses and return the final status + initStatus = initStatus && parentInitStatus && sourceDescStatus; + return initStatus; +} diff --git a/src/nwb/event/EventsTable.hpp b/src/nwb/event/EventsTable.hpp new file mode 100644 index 000000000..22d68f254 --- /dev/null +++ b/src/nwb/event/EventsTable.hpp @@ -0,0 +1,160 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/event/DurationVectorData.hpp" +#include "nwb/event/TimestampVectorData.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" + +// Include for the namespace schema header +#include "spec/core.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief A column-based table to store information about events, one event per + * row. Use `EventsTable` when each row is anchored at a single timestamp and + * duration is absent, optional, or mixed across rows. Additional columns may be + * added to store metadata about each event, such as the duration of the event. + * Examples include TTL pulses, licks, rewards, stimulus onsets, and detected + * ripples. Each `EventsTable` should hold events of a single type, so that all + * rows share the same set of per-event metadata columns. Events of different + * types (e.g., licks and stimulus presentations) should be stored in separate + * `EventsTable` instances. + */ +class EventsTable : public AQNWB::NWB::DynamicTable +{ +public: + /** + * @brief Virtual destructor. + */ + virtual ~EventsTable() override {} + + /** + * @brief Validates the provided data specifications for the EventsTable. + * @param dataSpecs The data specifications to validate. + * @return Status::Success if the specifications are valid, otherwise + * Status::Failure. + */ + Status validateDataSpecs( + const std::vector& dataSpecs) const override; + + /** + * @brief Creates the default data specs for the EventsTable. + * @param timestampResolution The temporal resolution of the timestamps - in + * seconds. See `TimestampVectorData::initialize()` for more details. + * @param createDurationColumn Whether to create the optional duration column + * (default: false). If true, the duration column will be created with the + * specified `durationResolution`. If false, the duration column will not be + * created. + * @param durationResolution The temporal resolution of the optional duration + * column in seconds. See `DurationVectorData::initialize()` for more details. + * If a std::nullopt is provided, the duration.resolution attribute of the + * duration column will not be created. Note, if `createDurationColumn` is + * false, this parameter is ignored. + * @param createAnnotationColumn Whether to create the annotation column + * (default: false) + * @param rowChunkSize The chunk size for the rows of the table. + * @return A vector of DataSpecPtr containing the default specs. + */ + static std::vector createDefaultDataSpecs( + std::optional timestampResolution = std::nullopt, + bool createDurationColumn = false, + std::optional durationResolution = std::nullopt, + const bool createAnnotationColumn = false, + const SizeType rowChunkSize = 100); + + /** + * @brief Initialize the object + * + * \note + * **Handling of Optional Paths:** + * If the EventsTable is being created in the `/events` group (e.g., + * `/events/my_table`), this method will automatically create the `/events` + * group if it does not already exist using the + * `NWBFile::requireEventsGroup()` helper method. This is because the + * `/events` group is optional in the NWB file structure and as such needs to + * be created on-demand. If the EventsTable is being created in a different + * path, this method will not create any parent groups, and it is the caller's + * responsibility to ensure that the parent group exists. + * + * @param description Description of the table + * @param sourceDescription Optional short text description of where the + * events came from, applying to every row in the table. If empty string is + * provided (default), then the attribute will not be created. + * @param columnSpecs The column specifications to use for initialization. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initialize( + const std::string& description, + const std::optional& sourceDescription = std::nullopt, + const std::vector& columnSpecs = {}); + + // Define read methods + // description overrides inherited field from parent neurodata_type + DEFINE_ATTRIBUTE_FIELD( + readDescription, + std::string, + "description", + "A description of the events stored in the table - including information " + "about how the event times were computed - especially if the times are " + "the result of processing or filtering raw data. For example - if the " + "experimenter is encoding different types of events using a strobed or " + "N-bit encoding - then the description should describe which channels " + "were used and how the event time is computed - e.g. - as the rise time " + "of the first bit.") + + DEFINE_ATTRIBUTE_FIELD( + readSourceDescription, + std::string, + "source_description", + "Optional short text description of where the events came from - " + "applying to every row in the table. For example - Acquisition system " + "for events emitted directly by the acquisition system (e.g. - TTL edges " + "or hardware event channels); Thresholding of analog signal ANALOG1 at 3 " + "V for events produced by a detection algorithm run on acquired data; or " + "Manual video review for events added by a human annotator. This is a " + "free-text label of origin only; use `description` for the longer " + "narrative of how the event times were computed (channels used - " + "encoding scheme - algorithm parameters - etc.).") + + DEFINE_REGISTERED_FIELD(readTimestampColumn, + AQNWB::NWB::TimestampVectorData, + "timestamp", + "Column containing the time that each event occurred " + "- in seconds - from the session start time.") + + DEFINE_REGISTERED_FIELD( + readDurationColumn, + AQNWB::NWB::DurationVectorData, + "duration", + "Optional column containing the duration of each event - in seconds. A " + "value of NaN can be used for events without a duration or with a " + "duration that is not yet specified.") + + DEFINE_REGISTERED_FIELD(readAnnotationColumn, + AQNWB::NWB::VectorData, + "annotation", + "Column containing user annotations about events.") + + REGISTER_SUBCLASS(EventsTable, DynamicTable, AQNWB::SPEC::CORE::namespaceName) + +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + EventsTable(const std::string& path, std::shared_ptr io); +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/event/TimestampVectorData.cpp b/src/nwb/event/TimestampVectorData.cpp new file mode 100644 index 000000000..a740410c1 --- /dev/null +++ b/src/nwb/event/TimestampVectorData.cpp @@ -0,0 +1,78 @@ +#include "TimestampVectorData.hpp" + +#include "Utils.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(TimestampVectorData) + +// Constructor +TimestampVectorData::TimestampVectorData(const std::string& path, + std::shared_ptr io) + : VectorData(path, io) +{ +} + +// Initialize the object +Status TimestampVectorData::initialize( + const AQNWB::IO::ArrayDataSetConfig& data, + const std::string& description, + std::optional resolution) +{ + Status initStatus = Status::Success; + + // Retrieve the IO object + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr + << "IO object has been deleted. Can't initialize TimestampVectorData: " + << m_path << std::endl; + return Status::Failure; + } + + // Call parent initialize method, which initializes the description attribute + // and the data dataset. + Status parentInitStatus = VectorData::initialize(data, description); + // Initialize attributes, datasets, and groups + // Initialize unit attribute with fixed value const std::string& "seconds" + const std::string& unit = "seconds"; + Status unitStatus = ioPtr->createAttribute(unit, m_path, "unit"); + // Initialize resolution attribute + Status resolutionStatus = Status::Success; + if (resolution.has_value()) { + float resValue = resolution.value(); + resolutionStatus = ioPtr->createAttribute( + AQNWB::IO::BaseDataType::F32, &resValue, m_path, "resolution"); + } + // Update the status to reflect the success or failure of the initialization + initStatus = initStatus && parentInitStatus && unitStatus && resolutionStatus; + // Return the final status of the initialization process + return initStatus; +} + +Status TimestampVectorData::DataSpec::initialize(Data& data) const +{ + auto* timestampData = dynamic_cast(&data); + if (!timestampData) { + std::cerr << "TimestampVectorData::DataSpec::initialize received " + "incompatible Data object" + << std::endl; + return Status::Failure; + } + return timestampData->initialize( + static_cast(*this), + description, + resolution); +} + +std::shared_ptr +TimestampVectorData::createDataSpec( + const std::string& name, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& description, + std::optional resolution) +{ + return std::make_shared(name, dataConfig, description, resolution); +} diff --git a/src/nwb/event/TimestampVectorData.hpp b/src/nwb/event/TimestampVectorData.hpp new file mode 100644 index 000000000..fdf61f79e --- /dev/null +++ b/src/nwb/event/TimestampVectorData.hpp @@ -0,0 +1,107 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/hdmf/table/VectorData.hpp" +// Include for the namespace schema header +#include "spec/core.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief A 1-D VectorData that stores timestamps in seconds from the session + * start time. Timestamps are not required to be sorted in time. + */ +class TimestampVectorData : public AQNWB::NWB::VectorData +{ +public: + struct DataSpec : public Data::DataSpec + { + DataSpec(const std::string& datasetName, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& columnDescription, + std::optional columnResolution = std::nullopt) + : Data::DataSpec(datasetName, dataConfig) + , description(columnDescription) + , resolution(columnResolution) + { + } + + std::string description; + std::optional resolution; + + Status initialize(Data& data) const override; + }; + + /** + * @brief Virtual destructor. + */ + virtual ~TimestampVectorData() override {} + + /** + * @brief Initialize the object + * @param data The configuration for the dataset + * @param description Description of the dataset + * @param resolution The temporal resolution of the timestamps - in seconds. + * This is typically the sampling period (1 / sampling_rate), also known as + * the clock period, of the data acquisition system from which the timestamps + * were recorded or derived. + */ + Status initialize(const AQNWB::IO::ArrayDataSetConfig& data, + const std::string& description, + std::optional resolution = std::nullopt); + + static std::shared_ptr createDataSpec( + const std::string& name, + const AQNWB::IO::ArrayDataSetConfig& dataConfig, + const std::string& description, + std::optional resolution = std::nullopt); + + // Define read methods + DEFINE_ATTRIBUTE_FIELD( + readUnit, + std::string, + "unit", + "The unit of measurement for the timestamps - fixed to seconds.") + + DEFINE_ATTRIBUTE_FIELD( + readResolution, + float, + "resolution", + "The temporal resolution of the timestamps - in seconds. This is " + "typically the sampling period (1 / sampling_rate) - also known as the " + "clock period - of the data acquisition system from which the timestamps " + "were recorded or derived.") + + DEFINE_DATASET_FIELD( + readData, + recordData, + float, + "", + "A 1-D VectorData that stores timestamps in seconds from the session " + "start time. Timestamps are not required to be sorted in time.") + + REGISTER_SUBCLASS(TimestampVectorData, + VectorData, + AQNWB::SPEC::CORE::namespaceName) + +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + TimestampVectorData(const std::string& path, + std::shared_ptr io); +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/file/ElectrodeGroup.hpp b/src/nwb/file/ElectrodeGroup.hpp index c6bc20dbb..e77387403 100644 --- a/src/nwb/file/ElectrodeGroup.hpp +++ b/src/nwb/file/ElectrodeGroup.hpp @@ -56,6 +56,6 @@ class ElectrodeGroup : public NWBContainer recordPosition, std::any, "position", - Stereotaxis or common framework coordinates) + Stereotaxic or common framework coordinates) }; } // namespace AQNWB::NWB diff --git a/src/nwb/file/ElectrodesTable.cpp b/src/nwb/file/ElectrodesTable.cpp index dff8bc618..f24692d54 100644 --- a/src/nwb/file/ElectrodesTable.cpp +++ b/src/nwb/file/ElectrodesTable.cpp @@ -13,20 +13,12 @@ REGISTER_SUBCLASS_IMPL(ElectrodesTable) ElectrodesTable::ElectrodesTable(std::shared_ptr io) : DynamicTable(electrodesTablePath, // use the electrodesTablePath io) - , m_groupNamesVectorData(VectorData::create( - AQNWB::mergePaths(electrodesTablePath, "group_name"), io)) - , m_locationsVectorData(VectorData::create( - AQNWB::mergePaths(electrodesTablePath, "location"), io)) { } ElectrodesTable::ElectrodesTable(const std::string& path, std::shared_ptr io) : DynamicTable(electrodesTablePath, io) - , m_groupNamesVectorData(VectorData::create( - AQNWB::mergePaths(electrodesTablePath, "group_name"), io)) - , m_locationsVectorData(VectorData::create( - AQNWB::mergePaths(electrodesTablePath, "location"), io)) { if (path != this->electrodesTablePath) { std::cerr << "WARNING: ElectrodesTable object is required to appear at " @@ -38,31 +30,71 @@ ElectrodesTable::ElectrodesTable(const std::string& path, /** Destructor */ ElectrodesTable::~ElectrodesTable() {} -/** Initialization function*/ -Status ElectrodesTable::initialize(const std::string& description) +std::vector ElectrodesTable::createDefaultDataSpecs( + const SizeType rowChunkSize) { - // create group - DynamicTable::initialize(description); - // IO::BaseDataType vstrType(IO::BaseDataType::Type::V_STR, - // 0); // 0 indicates variable length + std::vector specs = + DynamicTable::createDefaultDataSpecs(rowChunkSize); - IO::ArrayDataSetConfig electrodeConfig( - IO::BaseDataType::I32, SizeArray {1}, SizeArray {1}); - Status electrodeStatus = m_rowElementIdentifiers->initialize(electrodeConfig); + IO::ArrayDataSetConfig locationConfig( + IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "location", + locationConfig, + "the location of channel within the subject e.g. brain region")); IO::ArrayDataSetConfig groupNameConfig( - IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {1}); - Status groupNameStatus = m_groupNamesVectorData->initialize( + IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "group_name", groupNameConfig, - "the name of the ElectrodeGroup this electrode is a part of"); + "the name of the ElectrodeGroup this electrode is a part of")); - IO::ArrayDataSetConfig locationConfig( - IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {1}); - Status locationStatus = m_locationsVectorData->initialize( - locationConfig, - "the location of channel within the subject e.g. brain region"); + // Note: "group" is a reference column, which is currently added dynamically + // in finalize() We don't add it to the specs here because reference columns + // are not yet supported in DataSpec. Also, reference columns can currently + // not be configured with chunking to support resize/append. + // TODO: Add support for reference columns in DataSpec and configure them + // here. + // TODO: Add support for creating reference columns with chunking to support + // resize/append. + + return specs; +} + +Status ElectrodesTable::validateDataSpecs( + const std::vector& dataSpecs) const +{ + Status status = DynamicTable::validateDataSpecs(dataSpecs); + if (status != Status::Success) { + return status; + } + + status = checkRequiredColumnSpec( + "location", dataSpecs, IO::BaseDataType(IO::BaseDataType::V_STR)); + if (status != Status::Success) { + return status; + } + + return checkRequiredColumnSpec( + "group_name", dataSpecs, IO::BaseDataType(IO::BaseDataType::V_STR)); +} - return electrodeStatus && groupNameStatus && locationStatus; +/** Initialization function*/ +Status ElectrodesTable::initialize(const std::string& description, + const std::vector& columnSpecs) +{ + std::vector specsToUse = columnSpecs; + if (specsToUse.empty()) { + specsToUse = createDefaultDataSpecs(); + } + + // create group. This configures the "location" and "group_name" columns + // (among others) via the DataSpec mechanism, creating and registering the + // corresponding VectorData recording objects exactly once. + Status dtStatus = DynamicTable::initialize(description, specsToUse); + + return dtStatus; } void ElectrodesTable::addElectrodes(const std::vector& channelsInput) @@ -83,16 +115,48 @@ Status ElectrodesTable::finalize() // Check if new values have been added for the columns and update them // Updated electrode numbers if (m_electrodeNumbers.size() > 0) { - Status rowIdStatus = setRowIDs(m_rowElementIdentifiers, m_electrodeNumbers); + Status rowIdStatus = setRowIDs(m_electrodeNumbers); m_electrodeNumbers.clear(); // clear after writing status = status && rowIdStatus; } // Add the location names if (m_locationNames.size() > 0) { - Status locationColStatus = - addColumn(m_locationsVectorData, m_locationNames); - m_locationNames.clear(); // clear after writing - status = status && locationColStatus; + auto locationColumn = getConfiguredColumn("location"); + if (!locationColumn) { + std::cerr << "ElectrodesTable::finalize failed to get location column." + << std::endl; + status = Status::Failure; + } else { + // Write all strings in a single block + auto dataset = locationColumn->recordData(); + Status writeStatus = + dataset->writeDataBlock(SizeArray {m_locationNames.size()}, + SizeArray {0}, + IO::BaseDataType::V_STR, + m_locationNames); + + m_locationNames.clear(); // clear after writing + status = status && writeStatus; + } + } + // Add the group names + if (m_groupNames.size() > 0) { + auto groupNameColumn = getConfiguredColumn("group_name"); + if (!groupNameColumn) { + std::cerr << "ElectrodesTable::finalize failed to get group_name column." + << std::endl; + status = Status::Failure; + } else { + // Write all strings in a single block + auto dataset = groupNameColumn->recordData(); + Status writeStatus = + dataset->writeDataBlock(SizeArray {m_groupNames.size()}, + SizeArray {0}, + IO::BaseDataType::V_STR, + m_groupNames); + m_groupNames.clear(); // clear after writing + status = status && writeStatus; + } } // Add the group references if (m_groupReferences.size() > 0) { @@ -104,13 +168,7 @@ Status ElectrodesTable::finalize() status = status && groupColStatus; m_groupReferences.clear(); // clear after writing } - // Add the group names - if (m_groupNames.size() > 0) { - Status groupNameColStatus = addColumn(m_groupNamesVectorData, m_groupNames); - m_groupNames.clear(); // clear after writing - status = status && groupNameColStatus; - } - // finalize the parent class to write the col names + // finalize the parent class // This must be done after all columns have been added Status dtStatus = DynamicTable::finalize(); status = status && dtStatus; diff --git a/src/nwb/file/ElectrodesTable.hpp b/src/nwb/file/ElectrodesTable.hpp index cc1b4cef6..0263dab14 100644 --- a/src/nwb/file/ElectrodesTable.hpp +++ b/src/nwb/file/ElectrodesTable.hpp @@ -13,7 +13,7 @@ namespace AQNWB::NWB /** * @brief Represents a table containing electrode metadata. */ -class ElectrodesTable : public DynamicTable +class ElectrodesTable : public AQNWB::NWB::DynamicTable { public: // Register the ElectrodesTable as a subclass of Container @@ -50,6 +50,23 @@ class ElectrodesTable : public DynamicTable */ ~ElectrodesTable() override; + /** + * @brief Validates the provided data specifications for the ElectrodesTable. + * @param dataSpecs The data specifications to validate. + * @return Status::Success if the specifications are valid, otherwise + * Status::Failure. + */ + Status validateDataSpecs( + const std::vector& dataSpecs) const override; + + /** + * @brief Creates the default data specs for the ElectrodesTable. + * @param rowChunkSize The chunk size for the rows of the table. + * @return A vector of DataSpecPtr containing the default specs. + */ + static std::vector createDefaultDataSpecs( + const SizeType rowChunkSize = 100); + /** * @brief Initializes the ElectrodesTable. * @@ -58,10 +75,12 @@ class ElectrodesTable : public DynamicTable * * @param description The description of the table (default: "metadata about * extracellular electrodes") + * @param columnSpecs The column specifications to use for initialization. * @return Status::Success if successful, otherwise Status::Failure. */ Status initialize(const std::string& description = - "metadata about extracellular electrodes"); + "metadata about extracellular electrodes", + const std::vector& columnSpecs = {}); /** * @brief Finalizes the ElectrodesTable. @@ -123,15 +142,6 @@ class ElectrodesTable : public DynamicTable */ inline const static std::string m_groupPathBase = "/general/extracellular_ephys"; - - /** - * @brief The group names column for write - */ - std::shared_ptr m_groupNamesVectorData; - - /** - * @brief The locations column for write - */ - std::shared_ptr m_locationsVectorData; }; + } // namespace AQNWB::NWB diff --git a/src/nwb/file/Subject.cpp b/src/nwb/file/Subject.cpp new file mode 100644 index 000000000..0e7ab0f80 --- /dev/null +++ b/src/nwb/file/Subject.cpp @@ -0,0 +1,160 @@ +#include "nwb/file/Subject.hpp" + +#include "Utils.hpp" +#include "nwb/NWBFile.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(Subject) + +Subject::Subject(std::shared_ptr io) + : NWBContainer(mergePaths(NWBFile::GENERAL_PATH, "subject"), io) +{ +} + +// Constructor +Subject::Subject(const std::string& path, std::shared_ptr io) + : NWBContainer(mergePaths(NWBFile::GENERAL_PATH, "subject"), io) +{ + if (path != mergePaths(NWBFile::GENERAL_PATH, "subject")) { + std::cerr << "WARNING: Subject object path must be /general/subject. " + "Ignoring provided path." + << std::endl; + } +} + +std::shared_ptr Subject::create(std::shared_ptr io) +{ + return RegisteredType::create( + mergePaths(NWBFile::GENERAL_PATH, "subject"), io); +} + +// Initialize the object +Status Subject::initialize(const SubjectSpec& subjectSpec) +{ + // Get the IO object + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "Subject::initialize IO object has been deleted." << std::endl; + return Status::Failure; + } + if (!ioPtr->canModifyObjects()) { + return Status::Failure; + } + + // Call parent initialize method. + Status initStatus = Status::Success; + Status parentInitStatus = NWBContainer::initialize(); + initStatus = initStatus && parentInitStatus; + + // Initialize attributes, datasets, and groups + // Initialize age dataset and age/reference attribute if age is provided + if (subjectSpec.age.has_value()) { + Status ageStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "age"), subjectSpec.age.value()); + initStatus = initStatus && ageStatus; + if (ageStatus != Status::Success) { + std::cerr << "Failed to create age dataset." << std::endl; + } else { + std::string ageReference = subjectSpec.ageReference.value_or("birth"); + Status ageRefStatus = ioPtr->createAttribute( + ageReference, mergePaths(this->m_path, "age"), "reference"); + initStatus = initStatus && ageRefStatus; + if (ageRefStatus != Status::Success) { + std::cerr << "Failed to create age reference attribute." << std::endl; + } + } + } + + // Initialize date_of_birth dataset if date_of_birth is provided + if (subjectSpec.dateOfBirth.has_value()) { + if (isISO8601Date(subjectSpec.dateOfBirth.value()) == false) { + std::cerr << "Warning: date_of_birth is not in ISO8601 format: " + << subjectSpec.dateOfBirth.value() << std::endl; + initStatus = Status::Failure; + } else { + Status dobStatus = + ioPtr->createStringDataSet(mergePaths(this->m_path, "date_of_birth"), + subjectSpec.dateOfBirth.value()); + initStatus = initStatus && dobStatus; + if (dobStatus != Status::Success) { + std::cerr << "Failed to create date_of_birth dataset." << std::endl; + } + } + } + // Initialize description dataset if description is provided + if (subjectSpec.description.has_value()) { + Status descStatus = + ioPtr->createStringDataSet(mergePaths(this->m_path, "description"), + subjectSpec.description.value()); + initStatus = initStatus && descStatus; + if (descStatus != Status::Success) { + std::cerr << "Failed to create description dataset." << std::endl; + } + } + + // Initialize genotype dataset if genotype is provided + if (subjectSpec.genotype.has_value()) { + Status genotypeStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "genotype"), subjectSpec.genotype.value()); + initStatus = initStatus && genotypeStatus; + if (genotypeStatus != Status::Success) { + std::cerr << "Failed to create genotype dataset." << std::endl; + } + } + + // Initialize sex dataset if sex is provided + if (subjectSpec.sex.has_value()) { + Status sexStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "sex"), subjectSpec.sex.value()); + initStatus = initStatus && sexStatus; + if (sexStatus != Status::Success) { + std::cerr << "Failed to create sex dataset." << std::endl; + } + } + + // Initialize species dataset if species is provided + if (subjectSpec.species.has_value()) { + Status speciesStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "species"), subjectSpec.species.value()); + initStatus = initStatus && speciesStatus; + if (speciesStatus != Status::Success) { + std::cerr << "Failed to create species dataset." << std::endl; + } + } + + // Initialize strain dataset if strain is provided + if (subjectSpec.strain.has_value()) { + Status strainStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "strain"), subjectSpec.strain.value()); + initStatus = initStatus && strainStatus; + if (strainStatus != Status::Success) { + std::cerr << "Failed to create strain dataset." << std::endl; + } + } + + // Initialize subject_id dataset if subject_id is provided + if (subjectSpec.subjectId.has_value()) { + Status subjectIdStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "subject_id"), subjectSpec.subjectId.value()); + initStatus = initStatus && subjectIdStatus; + if (subjectIdStatus != Status::Success) { + std::cerr << "Failed to create subject_id dataset." << std::endl; + } + } + + // Initialize weight dataset if weight is provided + if (subjectSpec.weight.has_value()) { + Status weightStatus = ioPtr->createStringDataSet( + mergePaths(this->m_path, "weight"), subjectSpec.weight.value()); + initStatus = initStatus && weightStatus; + if (weightStatus != Status::Success) { + std::cerr << "Failed to create weight dataset." << std::endl; + } + } + + // Return the overall status of the initialization + return initStatus; +} diff --git a/src/nwb/file/Subject.hpp b/src/nwb/file/Subject.hpp new file mode 100644 index 000000000..534460db7 --- /dev/null +++ b/src/nwb/file/Subject.hpp @@ -0,0 +1,266 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/base/NWBContainer.hpp" +// Include for the namespace schema header +#include "spec/core.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief Information about the animal or person from which the data was + * measured. + */ +class Subject : public AQNWB::NWB::NWBContainer +{ +public: + REGISTER_SUBCLASS(Subject, NWBContainer, AQNWB::SPEC::CORE::namespaceName) + + /** + * @brief Metadata about the experimental subject. + * + * All fields are optional. Pass an instance of this struct to + * Subject::initialize() to create a Subject group in the NWB file. + */ + struct SubjectSpec + { + /// @brief Age of subject (e.g., "P90D" for 90 days post-natal). + std::optional age = std::nullopt; + /// @brief Age is with reference to this event. Can be ‘birth’ or + /// ‘gestational’. If reference is omitted, ‘birth’ is implied. + std::optional ageReference = std::nullopt; + /// @brief Date of birth of subject as iso formatted date string + std::optional dateOfBirth = std::nullopt; + /// @brief Description of subject and where subject came from. + std::optional description = std::nullopt; + /// @brief Genetic strain. If absent, assume Wild Type (WT). + std::optional genotype = std::nullopt; + /// @brief Biological sex of subject. + std::optional sex = std::nullopt; + /// @brief Species of subject. + std::optional species = std::nullopt; + /// @brief Strain of subject. + std::optional strain = std::nullopt; + /// @brief ID of animal/person used/participating in experiment (lab + /// convention). + std::optional subjectId = std::nullopt; + /// @brief Weight at time of experiment. + std::optional weight = std::nullopt; + + /** + * @brief Set the age of the subject. + * @param value Age of subject (e.g., "P90D" for 90 days post-natal). + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withAge(std::string value) + { + age = std::move(value); + return *this; + } + + /** + * @brief Set the age reference event of the subject. + * @param value Age is with reference to this event. Can be ‘birth’ or + * ‘gestational’. If reference is omitted, ‘birth’ is implied. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withAgeReference(std::string value) + { + ageReference = std::move(value); + return *this; + } + + /** + * @brief Set the date of birth of the subject. + * @param value Date of birth of subject as iso formatted date string. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withDateOfBirth(std::string value) + { + dateOfBirth = std::move(value); + return *this; + } + + /** + * @brief Set the description of the subject. + * @param value Description of subject and where subject came from. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withDescription(std::string value) + { + description = std::move(value); + return *this; + } + + /** + * @brief Set the genotype of the subject. + * @param value Genetic strain. If absent, assume Wild Type (WT). + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withGenotype(std::string value) + { + genotype = std::move(value); + return *this; + } + + /** + * @brief Set the biological sex of the subject. + * @param value Biological sex of subject. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withSex(std::string value) + { + sex = std::move(value); + return *this; + } + + /** + * @brief Set the species of the subject. + * @param value Species of subject. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withSpecies(std::string value) + { + species = std::move(value); + return *this; + } + + /** + * @brief Set the strain of the subject. + * @param value Strain of subject. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withStrain(std::string value) + { + strain = std::move(value); + return *this; + } + + /** + * @brief Set the ID of the subject. + * @param value ID of animal/person used/participating in experiment (lab + * convention). + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withSubjectId(std::string value) + { + subjectId = std::move(value); + return *this; + } + + /** + * @brief Set the weight of the subject. + * @param value Weight at time of experiment. + * @return Reference to this SubjectSpec, to allow method chaining. + */ + SubjectSpec& withWeight(std::string value) + { + weight = std::move(value); + return *this; + } + }; + + /** \brief Convenience factor method since the path is fixed to + * '/general/subject' + * @param io A shared pointer to the IO object. + * @return A shared pointer to the created NWBFile object, or nullptr if + * creation failed. + */ + static std::shared_ptr create(std::shared_ptr io); + + /** + * @brief Virtual destructor. + */ + virtual ~Subject() override {} + + /** + * @brief Initialize the object + * @param subjectSpec The SubjectSpec object with the subject metadata + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initialize(const SubjectSpec& subjectSpec); + + // Define read methods + DEFINE_DATASET_FIELD( + readAge, + recordAge, + std::string, + "age", + "Age of subject. Can be supplied instead of date_of_birth.") + + DEFINE_DATASET_FIELD( + readDateOfBirth, + recordDateOfBirth, + std::string, + "date_of_birth", + "Date of birth of subject. Can be supplied instead of age.") + + DEFINE_DATASET_FIELD(readDescription, + recordDescription, + std::string, + "description", + "Description of subject and where subject came from " + "(e.g. - breeder - if animal).") + + DEFINE_DATASET_FIELD(readGenotype, + recordGenotype, + std::string, + "genotype", + "Genetic strain. If absent - assume Wild Type (WT).") + + DEFINE_DATASET_FIELD( + readSex, recordSex, std::string, "sex", "Gender of subject.") + + DEFINE_DATASET_FIELD( + readSpecies, recordSpecies, std::string, "species", "Species of subject.") + + DEFINE_DATASET_FIELD( + readStrain, recordStrain, std::string, "strain", "Strain of subject.") + + DEFINE_DATASET_FIELD( + readSubjectId, + recordSubjectId, + std::string, + "subject_id", + "ID of animal/person used/participating in experiment (lab convention).") + + DEFINE_DATASET_FIELD(readWeight, + recordWeight, + std::string, + "weight", + "Weight at time of experiment - at time of surgery and " + "at other important times.") + + DEFINE_ATTRIBUTE_FIELD( + readAgeReference, + std::string, + "age/reference", + "Age is with reference to this event. Can be birth or gestational. If " + "reference is omitted - birth is implied.") + +protected: + /** + * @brief Constructor for Subject. + * @param io The shared pointer to the IO object. + */ + explicit Subject(std::shared_ptr io); + + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + Subject(const std::string& path, std::shared_ptr io); +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/hdmf/base/Data.cpp b/src/nwb/hdmf/base/Data.cpp index 6fcbadfcb..8f4b1a191 100644 --- a/src/nwb/hdmf/base/Data.cpp +++ b/src/nwb/hdmf/base/Data.cpp @@ -1,5 +1,9 @@ +#include + #include "nwb/hdmf/base/Data.hpp" +#include "io/RecordingObjects.hpp" + using namespace AQNWB::NWB; // Register the base Data class @@ -48,9 +52,146 @@ Status Data::initialize(const IO::BaseArrayDataSetConfig& dataConfig) return Status::Success; } +std::vector Data::readCellValues( + const SizeArray& start, + const SizeArray& count, + const SizeArray& stride, + const SizeArray& block) +{ + std::vector result; + + // Read the data block + auto dataBlock = readData()->valuesGeneric(start, count, stride, block); + + // Get the number of elements read + size_t numElements = std::accumulate(dataBlock.shape.begin(), + dataBlock.shape.end(), + 1ULL, + std::multiplies()); + + if (numElements == 0) { + return result; + } + + result.reserve(numElements); + + // Convert the data block to CellValues based on the data type + switch (dataBlock.baseDataType.type) { + case IO::BaseDataType::T_U8: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_U16: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_U32: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_U64: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_I8: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_I16: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_I32: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_I64: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_F32: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_F64: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + case IO::BaseDataType::T_STR: + case IO::BaseDataType::V_STR: { + auto data = IO::DataBlock::fromGeneric(dataBlock); + for (size_t i = 0; i < numElements; ++i) { + result.emplace_back(data.data[i]); + } + break; + } + default: + throw std::runtime_error("Unsupported data type in Data::readCellValues"); + } + + return result; +} + namespace AQNWB::NWB { +template +std::shared_ptr> DataTyped::create( + const std::string& path, std::shared_ptr io) +{ + if (io) { + auto recordingObjects = io->getRecordingObjects(); + if (recordingObjects) { + auto existingObj = recordingObjects->getRecordingObject(path); + if (existingObj) { + auto typedObj = + std::dynamic_pointer_cast>(existingObj); + if (typedObj) { + return typedObj; + } + // If an object with the same path exists but is not of this exact type + // (e.g., it's a base Data), we create a new DataTyped + // instance but do NOT add it to the recording objects cache to avoid + // overwriting the existing Data object or causing conflicts. + return std::shared_ptr>( + new DataTyped(path, io)); + } + } + return std::shared_ptr>(new DataTyped(path, io)); + } + return nullptr; +} + // Explicitly instantiate the DataTyped template for all common data types. // This ensures that these specializations are generated by the compiler, // reducing compile times and ensuring availability throughout the program. diff --git a/src/nwb/hdmf/base/Data.hpp b/src/nwb/hdmf/base/Data.hpp index dbe8f1777..84b24b32f 100644 --- a/src/nwb/hdmf/base/Data.hpp +++ b/src/nwb/hdmf/base/Data.hpp @@ -3,12 +3,16 @@ #include #include +#include "Types.hpp" #include "io/BaseIO.hpp" #include "nwb/RegisteredType.hpp" #include "spec/hdmf_common.hpp" namespace AQNWB::NWB { + +using CellValue = AQNWB::Types::CellValue; + /** * @brief An abstract data type for a dataset. * @tparam DTYPE The data type of the data managed by Data @@ -16,6 +20,88 @@ namespace AQNWB::NWB class Data : public RegisteredType { public: + /** + * @brief Non-templated base class for runtime Data configuration. + * + * DynamicTable needs to store a heterogeneous list of specs for different + * Data subclasses (e.g., ElementIdentifiers, VectorData, + * TimestampVectorData). This non-templated base provides a common storage + * type while still allowing derived typed specs to implement concrete object + * creation. + */ + struct DataSpecBase : public IO::ArrayDataSetConfig + { + /** + * @brief Construct the DataSpecBase. + * @param datasetName The dataset name relative to the owning + * table/container. + * @param dataConfig Dataset configuration for the object to create. + */ + DataSpecBase(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig) + : IO::ArrayDataSetConfig(dataConfig) + , name(datasetName) + { + } + + virtual ~DataSpecBase() = default; + + /** + * @brief Create the concrete Data object represented by this spec. + * @param path Full path of the dataset to create. + * @param io The IO object to associate with the created instance. + * @return The created concrete Data object. + */ + virtual std::shared_ptr create( + const std::string& path, std::shared_ptr io) const = 0; + + /** + * @brief Initialize the created concrete Data object from this spec. + * @param data The object to initialize. + * @return Status::Success if successful, otherwise Status::Failure. + */ + virtual Status initialize(Data& data) const = 0; + + /// @brief Dataset name relative to the owning table/container. + std::string name; + }; + + /** + * @brief Typed runtime configuration for a concrete Data subclass. + * + * The template parameter DataT supplies the `create(path, io)` factory used + * to instantiate the correct concrete subclass from the stored runtime spec. + * + * @tparam DataT Concrete subclass of Data represented by this spec. + */ + template + struct DataSpec : public DataSpecBase + { + /** + * @brief Construct the typed DataSpec. + * @param datasetName The dataset name relative to the owning + * table/container. + * @param dataConfig Dataset configuration for the object to create. + */ + DataSpec(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig) + : DataSpecBase(datasetName, dataConfig) + { + } + + /** + * @brief Create the concrete Data object represented by this typed spec. + * @param path Full path of the dataset to create. + * @param io The IO object to associate with the created instance. + * @return The created concrete Data object. + */ + std::shared_ptr create(const std::string& path, + std::shared_ptr io) const override + { + return DataT::create(path, io); + } + }; + // Register the Data class with the type registry REGISTER_SUBCLASS(Data, RegisteredType, "hdmf-common") @@ -49,6 +135,29 @@ class Data : public RegisteredType */ inline bool isInitialized() { return this->readData()->exists(); } + /** + * @brief Reads a range of cell values from the dataset. + * + * The function reads a slice of data from the dataset based on the provided + * start, count, stride, and block parameters. It returns a vector of + * CellValue objects, where each CellValue contains the data for a cell. This + * is used for reading data from the VectorData column in a DynamicTable. For + * regular data read of values used readData()->values() or + * readData()->valuesGeneric() instead. + * + * @param start The starting indices for the slice (optional). + * @param count The number of elements to read for each dimension (optional). + * @param stride The stride for each dimension (optional). + * @param block The block size for each dimension (optional). + * @return A vector of CellValue, where each CellValue contains the data for a + * cell. + */ + virtual std::vector readCellValues( + const SizeArray& start = {}, + const SizeArray& count = {}, + const SizeArray& stride = {}, + const SizeArray& block = {}); + // Define the data fields to expose for lazy read access DEFINE_DATASET_FIELD(readData, recordData, std::any, "", The main data) @@ -108,10 +217,7 @@ class DataTyped : public Data * creation failed. */ static std::shared_ptr create( - const std::string& path, std::shared_ptr io) - { - return RegisteredType::create(path, io); - } + const std::string& path, std::shared_ptr io); /** * @brief Virtual destructor. @@ -123,9 +229,10 @@ class DataTyped : public Data * * This function is useful when the type of the data is known and we want * read data in a typed manner where the type is stored in the DTYPE template - * parameter. NOTE: The original Data object retains ownership of the - * Data.m_dataset recording dataset object if it was initialized, i.e., - * the returned DataTyped object will have a nullptr m_dataset. + * parameter. NOTE: The BaseRecordingData object used for recording to the + * dataset is cached in the BaseIO object and is, hence, shared between the + * Data and the DataTyped, such that recording to either object + * will be reflected in the other. * * @param data The Data object to convert * @return A DataTyped object with the same path and IO object as the input diff --git a/src/nwb/hdmf/table/DynamicTable.cpp b/src/nwb/hdmf/table/DynamicTable.cpp index c0929a6c0..71d1b4d5c 100644 --- a/src/nwb/hdmf/table/DynamicTable.cpp +++ b/src/nwb/hdmf/table/DynamicTable.cpp @@ -1,9 +1,16 @@ +#include +#include + #include "nwb/hdmf/table/DynamicTable.hpp" #include "Utils.hpp" +#include "nwb/hdmf/table/MeaningsTable.hpp" +#include "nwb/hdmf/table/VectorIndex.hpp" using namespace AQNWB::NWB; +#include + // DynamicTable // Initialize the static registered_ member to trigger registration REGISTER_SUBCLASS_IMPL(DynamicTable) @@ -13,9 +20,7 @@ DynamicTable::DynamicTable(const std::string& path, std::shared_ptr io) : Container(path, io) , m_colNames({}) - , m_recordingColumns(std::make_unique()) - , m_rowElementIdentifiers( - ElementIdentifiers::create(AQNWB::mergePaths(path, "id"), io)) + , m_rowElementIdentifiers(nullptr) { // Read the colNames attribute if it exists such that any columns // we may add append to the existing list of columns rather than @@ -36,7 +41,8 @@ DynamicTable::DynamicTable(const std::string& path, DynamicTable::~DynamicTable() {} /** Initialization function*/ -Status DynamicTable::initialize(const std::string& description) +Status DynamicTable::initialize(const std::string& description, + const std::vector& dataSpecs) { auto ioPtr = getIO(); if (!ioPtr) { @@ -49,7 +55,59 @@ Status DynamicTable::initialize(const std::string& description) if (description != "") { ioPtr->createAttribute(description, m_path, "description"); } - return containerStatus; + + const auto effectiveSpecs = + dataSpecs.empty() ? createDefaultDataSpecs() : dataSpecs; + + Status validationStatus = validateDataSpecs(effectiveSpecs); + if (validationStatus != Status::Success) { + throw std::invalid_argument( + "DynamicTable::initialize: provided dataSpecs are invalid."); + } + + clearColumns(); + Status configureStatus = configureDataObjects(effectiveSpecs); + return containerStatus && configureStatus; +} + +Status DynamicTable::validateDataSpecs( + const std::vector& dataSpecs) const +{ + return checkRequiredColumnSpec( + "id", dataSpecs, IO::BaseDataType::I32); +} + +std::vector DynamicTable::createDefaultDataSpecs( + const SizeType rowChunkSize) +{ + return {ElementIdentifiers::createDataSpec( + "id", + IO::ArrayDataSetConfig( + IO::BaseDataType::I32, SizeArray {0}, SizeArray {rowChunkSize}))}; +} + +void DynamicTable::setColNames(const std::vector& newColNames) +{ + if (newColNames == m_colNames) { + return; + } + + // Ensure that the new column names are a permutation of the existing column + // names. This check ensures that all existing columns are present in the new + // list, but allows for reordering. + if (newColNames.size() != m_colNames.size() + || !std::is_permutation( + newColNames.begin(), newColNames.end(), m_colNames.begin())) + { + std::cerr << "New column names do not match existing column names. " + << "All columns must be present in the newColNames vector." + << std::endl; + throw std::invalid_argument( + "New column names do not match existing column names."); + } + + m_colNames = newColNames; + flushColNames(); } SizeType DynamicTable::addColumnName(const std::string& colName) @@ -61,10 +119,20 @@ SizeType DynamicTable::addColumnName(const std::string& colName) } else { // Column name does not exist, add it and return new index m_colNames.push_back(colName); + flushColNames(); return m_colNames.size() - 1; } } +Status DynamicTable::addColumn(const DataSpecPtr& dataSpec) +{ + if (!dataSpec) { + std::cerr << "DynamicTable::addColumn received null DataSpec." << std::endl; + return Status::Failure; + } + return configureDataObject(*dataSpec); +} + /** Add column to table */ Status DynamicTable::addColumn(const std::shared_ptr& vectorData, const std::vector& values) @@ -80,17 +148,30 @@ Status DynamicTable::addColumn(const std::shared_ptr& vectorData, SizeArray {0}, IO::BaseDataType::V_STR, values); - addColumnName(vectorData->getName()); - m_recordingColumns->addRecordingObject(vectorData); + registerColumn(vectorData); return writeStatus; } } -Status DynamicTable::setRowIDs( - const std::shared_ptr& elementIDs, - const std::vector& values) +Status DynamicTable::addColumn(const std::shared_ptr& vectorData) { - if (!elementIDs->isInitialized()) { + if (!vectorData) { + std::cerr << "VectorData column is null" << std::endl; + return Status::Failure; + } + if (!vectorData->isInitialized()) { + std::cerr << "VectorData dataset is not initialized " + << vectorData->getPath() << std::endl; + return Status::Failure; + } + + registerColumn(vectorData); + return Status::Success; +} + +Status DynamicTable::setRowIDs(const std::vector& values) +{ + if (!m_rowElementIdentifiers) { std::cerr << "ElementIdentifiers dataset is not initialized" << std::endl; return Status::Failure; } else { @@ -101,14 +182,439 @@ Status DynamicTable::setRowIDs( return Status::Failure; } - Status writeDataStatus = elementIDs->recordData()->writeDataBlock( - SizeArray(1, values.size()), IO::BaseDataType::I32, &values[0]); - Status createAttrsStatus = - ioPtr->createCommonNWBAttributes(AQNWB::mergePaths(m_path, "id"), - elementIDs->getNamespace(), - elementIDs->getTypeName()); - return writeDataStatus && createAttrsStatus; + if (values.empty()) { + return Status::Success; + } + auto idData = m_rowElementIdentifiers->recordData(); + SizeArray positionOffset = {0}; + auto currentShape = idData->getShape(); + if (!currentShape.empty()) { + positionOffset[0] = currentShape[0]; + } + Status writeDataStatus = idData->writeDataBlock(SizeArray {values.size()}, + positionOffset, + IO::BaseDataType::I32, + &values[0]); + return writeDataStatus; + } +} + +Status DynamicTable::addRow(const AQNWB::Types::RowData& row, + const std::optional& rowId) +{ + if (rowId.has_value()) { + return addRows(std::vector {row}, + std::vector {*rowId}); + } + return addRows(std::vector {row}); +} + +SizeType DynamicTable::getNumberOfRows() const +{ + auto idCol = readIdColumn(); + if (!idCol) { + return 0; + } + auto shape = idCol->readData()->getShape(); + if (shape.empty()) { + return 0; + } + return shape[0]; +} + +std::vector DynamicTable::readRows( + SizeType start, + SizeType count, + const std::vector& colNames, + bool includeId) +{ + std::vector rows; + + if (m_colNames.empty()) { + std::cerr << "DynamicTable::readRows no columns available." << std::endl; + return rows; + } + + // Determine which columns to read + std::vector columnsToRead = + colNames.empty() ? m_colNames : colNames; + + // Add "id" to columnsToRead if it's not already there and includeId is true + if (includeId) { + auto it = std::find(columnsToRead.begin(), columnsToRead.end(), "id"); + if (it == columnsToRead.end()) { + columnsToRead.insert(columnsToRead.begin(), "id"); + } + } + + // 1. Determine the actual number of rows to read + SizeType totalRows = getNumberOfRows(); + if (totalRows == 0) { + std::cerr << "DynamicTable::readRows: Could not determine row count." + << std::endl; + return rows; + } + + // Raise an error if start is out of bounds + if (start >= totalRows) { + throw std::invalid_argument( + "DynamicTable::readRows: start index out of bounds."); + } + + // Adjust count if it exceeds the available rows + SizeType actualCount = count; + if (count == AQNWB::Types::SizeTypeNotSet || start + count > totalRows) { + actualCount = totalRows - start; + } + + // 2. Read data for each column + std::unordered_map> + columnData; + + for (const auto& colName : columnsToRead) { + if (colName == "id") { + auto idCol = readIdColumn(); + if (idCol) { + SizeArray startArray = {start}; + SizeArray countArray = {actualCount}; + columnData["id"] = idCol->readCellValues(startArray, countArray); + } + continue; + } + + // Check if there is a VectorIndex for this column + auto vectorIndex = readColumn(colName + "_index"); + + if (vectorIndex) { + // Read ragged array data using the VectorIndex + columnData[colName] = + vectorIndex->readIndexedCellValues(start, actualCount); + } else { + // Read regular column data + auto col = readColumn(colName); + if (col) { + SizeArray startArray = {start}; + SizeArray countArray = {actualCount}; + columnData[colName] = col->readCellValues(startArray, countArray); + } else { + std::cerr << "DynamicTable::readRows: Could not read column '" + << colName << "'." << std::endl; + } + } + } + + // 3. Assemble the rows + rows.resize(actualCount); + for (SizeType i = 0; i < actualCount; ++i) { + for (const auto& [colName, data] : columnData) { + if (i < data.size()) { + rows[i][colName] = data[i]; + } + } + } + + return rows; +} + +std::string DynamicTable::toString(SizeType start, + SizeType count, + const std::vector& colNames, + bool includeId) +{ + std::vector rows = + readRows(start, count, colNames, includeId); + if (rows.empty()) { + return "Empty Table"; + } + + std::string result; + + // Determine columns to print + std::vector columnsToPrint = + colNames.empty() ? m_colNames : colNames; + if (includeId) { + auto it = std::find(columnsToPrint.begin(), columnsToPrint.end(), "id"); + if (it == columnsToPrint.end()) { + columnsToPrint.insert(columnsToPrint.begin(), "id"); + } + } + + // Print header + for (size_t i = 0; i < columnsToPrint.size(); ++i) { + result += columnsToPrint[i]; + if (i < columnsToPrint.size() - 1) { + result += ","; + } + } + result += "\n"; + + // Helper to escape fields for CSV (RFC 4180) + auto csvEscape = [](std::string_view field) -> std::string + { + const bool needsQuotes = + field.find_first_of(",\"\n\r") != std::string_view::npos + || (!field.empty() && field.front() == '[' && field.back() == ']'); + if (!needsQuotes) + return std::string(field); + + std::string out; + out.reserve(field.size() + 2); + out.push_back('"'); + for (char c : field) { + if (c == '"') + out.push_back('"'); // double it + out.push_back(c); + } + out.push_back('"'); + return out; + }; + + // Print rows + for (const auto& row : rows) { + for (size_t i = 0; i < columnsToPrint.size(); ++i) { + const auto& colName = columnsToPrint[i]; + auto it = row.find(colName); + if (it != row.end()) { + result += csvEscape(it->second.toString()); + } else { + result += "NULL"; + } + + if (i < columnsToPrint.size() - 1) { + result += ","; + } + } + result += "\n"; + } + + return result; +} + +Status DynamicTable::addRows(const std::vector& rows, + const std::vector& rowIds) +{ + if (rows.empty()) { + return Status::Success; + } + Status loadStatus = ensureConfiguredColumnsLoaded(); + if (loadStatus != Status::Success) { + return Status::Failure; + } + if (m_configuredColumns.empty()) { + std::cerr << "DynamicTable::addRows no configured columns available." + << std::endl; + return Status::Failure; + } + if (!rowIds.empty() && rowIds.size() != rows.size()) { + std::cerr << "DynamicTable::addRows rowIds size must match rows size." + << std::endl; + return Status::Failure; + } + + SizeType rowCount = rows.size(); + + // 1. Identify target columns and cache them in VectorIndices + std::unordered_set targetColumns; + for (const auto& col : m_configuredColumns) { + auto vectorIndex = std::dynamic_pointer_cast(col.column); + if (vectorIndex) { + auto target = vectorIndex->readTarget(); + if (target) { + targetColumns.insert(target->getName()); + } + } + } + + // 2. Validate that all rows have the required columns + for (const auto& row : rows) { + for (const auto& col : m_configuredColumns) { + if (targetColumns.find(col.name) != targetColumns.end()) { + // Target columns must be present in the row + if (row.find(col.name) == row.end()) { + std::cerr + << "DynamicTable::addRows: Missing value for target column '" + << col.name << "'." << std::endl; + return Status::Failure; + } + } else { + auto vectorIndex = std::dynamic_pointer_cast(col.column); + if (!vectorIndex) { + // Regular columns must be present in the row + auto cell = row.find(col.name); + if (cell == row.end()) { + std::cerr << "DynamicTable::addRows: Missing value for column '" + << col.name << "'." << std::endl; + return Status::Failure; + } + if (std::holds_alternative( + cell->second.value)) + { + std::cerr << "DynamicTable::addRows: Regular column '" << col.name + << "' does not accept vector-valued cells." << std::endl; + return Status::Failure; + } + } + } + } + } + + // 3. Buffer values for each column + std::unordered_map + columnBuffers; + std::unordered_map> indexBuffers; + + for (const auto& col : m_configuredColumns) { + auto vectorIndex = std::dynamic_pointer_cast(col.column); + if (vectorIndex) { + auto target = vectorIndex->readTarget(); + if (!target) + return Status::Failure; + std::string targetName = target->getName(); + columnBuffers[targetName] = IO::BaseDataType::createEmptyVectorVariant( + target->readData()->getDataType()); + indexBuffers[col.name] = std::vector(); + } else if (targetColumns.find(col.name) == targetColumns.end()) { + columnBuffers[col.name] = + IO::BaseDataType::createEmptyVectorVariant(col.dataType); + } + } + + for (const auto& row : rows) { + for (const auto& col : m_configuredColumns) { + auto vectorIndex = std::dynamic_pointer_cast(col.column); + if (vectorIndex) { + auto target = vectorIndex->readTarget(); + if (!target) + return Status::Failure; + std::string targetName = target->getName(); + auto it = row.find(targetName); + if (it != row.end()) { + // Also need to account for existing elements in the target column + size_t existingTargetElements = 0; + auto targetData = target->readData(); + if (targetData) { + auto shape = targetData->getShape(); + if (!shape.empty()) { + existingTargetElements = shape[0]; + } + } + + Status appendStatus = + appendCellValueToBuffer(columnBuffers[targetName], it->second); + if (appendStatus != Status::Success) { + std::cerr << "DynamicTable::addRows: Failed to append row to " + "VectorIndex '" + << col.name << "'." << std::endl; + return Status::Failure; + } + + size_t newTargetSize = std::visit( + [](const auto& vec) -> size_t + { + using VecType = std::decay_t; + if constexpr (std::is_same_v) + return 0; + else + return vec.size(); + }, + columnBuffers[targetName]); + + uint32_t newIndex = + static_cast(existingTargetElements + newTargetSize); + indexBuffers[col.name].push_back(newIndex); + } + } else if (targetColumns.find(col.name) == targetColumns.end()) { + // Regular column (not a target of a VectorIndex) + auto it = row.find(col.name); + if (it != row.end()) { + Status appendStatus = + appendCellValueToBuffer(columnBuffers[col.name], it->second); + if (appendStatus != Status::Success) { + std::cerr + << "DynamicTable::addRows: Failed to append value to column '" + << col.name << "'." << std::endl; + return Status::Failure; + } + } + } + } + } + + // 4. Write buffers to columns + for (const auto& col : m_configuredColumns) { + auto vectorIndex = std::dynamic_pointer_cast(col.column); + if (vectorIndex) { + auto target = vectorIndex->readTarget(); + if (!target) + return Status::Failure; + std::string targetName = target->getName(); + + // Write target buffer + if (columnBuffers.find(targetName) != columnBuffers.end()) { + Status writeStatus = target->appendBuffer(columnBuffers[targetName]); + if (writeStatus != Status::Success) { + std::cerr << "DynamicTable::addRows: Failed to write buffer for " + "target column '" + << targetName << "'." << std::endl; + return Status::Failure; + } + // Clear buffer so we don't write it again if multiple indices point to + // it + columnBuffers.erase(targetName); + } + + // Write index buffer + IO::BaseDataType::BaseDataVectorVariant indexVariant = + indexBuffers[col.name]; + Status writeIndexStatus = vectorIndex->appendBuffer(indexVariant); + if (writeIndexStatus != Status::Success) { + std::cerr << "DynamicTable::addRows: Failed to write buffer for index " + "column '" + << col.name << "'." << std::endl; + return Status::Failure; + } + } else if (targetColumns.find(col.name) == targetColumns.end()) { + // Write regular column buffer + Status writeStatus = col.column->appendBuffer(columnBuffers[col.name]); + if (writeStatus != Status::Success) { + std::cerr + << "DynamicTable::addRows: Failed to write buffer for column '" + << col.name << "'." << std::endl; + return Status::Failure; + } + } + } + + // 5. Handle IDs + std::vector finalRowIds = rowIds; + if (finalRowIds.empty()) { + finalRowIds = generateRowIDs(rowCount); + } else if (finalRowIds.size() != rowCount) { + std::cerr << "DynamicTable::addRows: Number of provided row IDs does not " + "match number of rows." + << std::endl; + return Status::Failure; } + + if (m_rowElementIdentifiers) { + auto idData = m_rowElementIdentifiers->recordData(); + SizeArray positionOffset = {0}; + auto currentShape = idData->getShape(); + if (!currentShape.empty()) { + positionOffset[0] = currentShape[0]; + } + Status idWriteStatus = idData->writeDataBlock(SizeArray {rowCount}, + positionOffset, + IO::BaseDataType::I32, + finalRowIds.data()); + if (idWriteStatus != Status::Success) { + std::cerr << "DynamicTable::addRows: Failed to write row IDs." + << std::endl; + return Status::Failure; + } + } + + return Status::Success; } Status DynamicTable::addReferenceColumn(const std::string& name, @@ -138,17 +644,16 @@ Status DynamicTable::addReferenceColumn(const std::string& name, std::cerr << "Failed to create reference column" << std::endl; return Status::Failure; } - addColumnName(name); - m_recordingColumns->addRecordingObject(refColumn); + registerColumn(refColumn); return Status::Success; } } -Status DynamicTable::finalize() +Status DynamicTable::flushColNames() { auto ioPtr = getIO(); if (!ioPtr) { - std::cerr << "DynamicTable::finalize IO object has been deleted." + std::cerr << "DynamicTable::flushColNames IO object has been deleted." << std::endl; return Status::Failure; } @@ -158,6 +663,256 @@ Status DynamicTable::finalize() "colnames", true // overwrite the attribute if it already exists ); + return colNamesStatus; +} + +Status DynamicTable::finalize() +{ Status parentStatus = Container::finalize(); - return colNamesStatus && parentStatus; + return parentStatus; +} + +std::shared_ptr DynamicTable::createMeaningsTable( + const std::string& columnName, const SizeType rowChunkSize) +{ + // Get the I/O object an ensure it is valid + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "DynamicTable::createMeaningsTable IO object has been deleted." + << std::endl; + return nullptr; + } + + // Check that the column VectorData exists in the DynamicTable + auto columnVectorData = readColumn(columnName); + if (!columnVectorData) { + std::cerr << "Column VectorData '" << columnName + << "' does not exist in DynamicTable '" << m_path + << "'. Cannot create MeaningsTable." << std::endl; + return nullptr; + } + auto valueDataType = columnVectorData->readData()->getDataType(); + + // Check the meanings_tables group exists, if not create it + std::string meaningsTablesGroupPath = + AQNWB::mergePaths(m_path, "meanings_tables"); + if (!ioPtr->objectExists(meaningsTablesGroupPath)) { + Status createGroupStatus = ioPtr->createGroup(meaningsTablesGroupPath); + if (createGroupStatus != Status::Success) { + std::cerr << "Failed to create meanings_tables group at '" + << meaningsTablesGroupPath << "'." << std::endl; + return nullptr; + } + } + + // Create the MeaningsTable for the specified column + std::string meaningsTablePath = + AQNWB::mergePaths(meaningsTablesGroupPath, columnName + "_meanings"); + auto meaningsTable = MeaningsTable::create(meaningsTablePath, ioPtr); + if (!meaningsTable) { + std::cerr << "Failed to create MeaningsTable at '" << meaningsTablePath + << "'." << std::endl; + return nullptr; + } + + // Initialize the MeaningsTable with the target VectorData and value data type + auto specs = + MeaningsTable::createDefaultDataSpecs(valueDataType, rowChunkSize); + Status initStatus = + meaningsTable->initialize(*columnVectorData, + valueDataType, + "Meanings table for column: " + columnName, + specs); + + // Report error if initialization failed + if (initStatus != Status::Success) { + std::cerr << "Failed to initialize MeaningsTable at '" << meaningsTablePath + << "'." << std::endl; + return nullptr; + } + + // Return the created MeaningsTable + return meaningsTable; +} + +std::shared_ptr DynamicTable::readMeaningsTable( + const std::string& objectName) const +{ + std::string prefixPath = AQNWB::mergePaths(m_path, "meanings_tables"); + std::string objectPath = AQNWB::mergePaths(prefixPath, objectName); + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "IO object has been deleted. Can't read field: " << objectPath + << std::endl; + return nullptr; + } + if (ioPtr->objectExists(objectPath)) { + return MeaningsTable::create(objectPath, ioPtr); + } + return nullptr; +} + +std::shared_ptr DynamicTable::createMeaningsTableInstance( + const std::string& objectName) const +{ + std::string prefixPath = AQNWB::mergePaths(m_path, "meanings_tables"); + std::string objectPath = AQNWB::mergePaths(prefixPath, objectName); + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "IO object has been deleted. Can't create field: " + << objectPath << std::endl; + return nullptr; + } + return MeaningsTable::create(objectPath, ioPtr); +} + +std::shared_ptr DynamicTable::getConfiguredColumn( + const std::string& name) const +{ + auto it = m_configuredColumnIndices.find(name); + if (it != m_configuredColumnIndices.end()) { + return m_configuredColumns[it->second].column; + } + return nullptr; +} + +void DynamicTable::clearColumns() +{ + m_configuredColumns.clear(); + m_configuredColumnIndices.clear(); + m_colNames.clear(); + m_rowElementIdentifiers.reset(); +} + +Status DynamicTable::configureDataObjects( + const std::vector& dataSpecs) +{ + for (const auto& spec : dataSpecs) { + if (!spec) { + std::cerr << "DynamicTable::configureDataObjects received null spec." + << std::endl; + return Status::Failure; + } + Status status = configureDataObject(*spec); + if (status != Status::Success) { + return status; + } + } + return Status::Success; +} + +Status DynamicTable::configureDataObject(const DataSpec& dataSpec) +{ + auto ioPtr = getIO(); + if (!ioPtr) { + return Status::Failure; + } + std::string columnPath = AQNWB::mergePaths(m_path, dataSpec.name); + auto dataObj = dataSpec.create(columnPath, ioPtr); + if (!dataObj) { + return Status::Failure; + } + Status initStatus = dataSpec.initialize(*dataObj); + if (initStatus != Status::Success) { + return Status::Failure; + } + + if (dataSpec.name == "id") { + m_rowElementIdentifiers = + std::dynamic_pointer_cast(dataObj); + return Status::Success; + } + + auto vectorData = std::dynamic_pointer_cast(dataObj); + if (!vectorData) { + return Status::Failure; + } + + registerColumn(vectorData); + + return Status::Success; +} + +SizeType DynamicTable::registerColumn(const std::shared_ptr& column) +{ + if (!column) { + std::cerr << "DynamicTable::registerColumn received null column." + << std::endl; + return static_cast(-1); + } + + std::string name = column->getName(); + + // If already registered, return its index. + auto it = m_configuredColumnIndices.find(name); + if (it != m_configuredColumnIndices.end()) { + return it->second; + } + + // Register column + ConfiguredColumn config; + config.name = name; + config.dataType = column->readData()->getDataType(); + config.column = column; + + m_configuredColumns.push_back(config); + SizeType index = m_configuredColumns.size() - 1; + m_configuredColumnIndices[name] = index; + + // If the column is not a VectorIndex, add its name to the list of column + // names Since we are here writing string values to the columns, our + // vectorData should not be able to be a VectorIndex, but we check just in + // case. + bool isVectorIndex = + (std::dynamic_pointer_cast(column) != nullptr); + if (!isVectorIndex) { + addColumnName(name); + } + // Return the index of the registered column. + return index; +} + +Status DynamicTable::ensureConfiguredColumnsLoaded() +{ + if (!m_configuredColumns.empty()) { + return Status::Success; + } + return loadConfiguredColumnsFromFile(); +} + +Status DynamicTable::loadConfiguredColumnsFromFile() +{ + auto ioPtr = getIO(); + if (!ioPtr) { + return Status::Failure; + } + + for (const auto& colName : m_colNames) { + auto col = readColumn(colName); + if (col) { + registerColumn(col); + } + } + + m_rowElementIdentifiers = readIdColumn(); + return Status::Success; +} + +std::vector DynamicTable::generateRowIDs(SizeType rowCount) +{ + std::vector ids(rowCount); + int startId = 0; + if (m_rowElementIdentifiers) { + auto idData = m_rowElementIdentifiers->recordData(); + if (idData) { + const auto& currentShape = idData->getShape(); + if (!currentShape.empty()) { + startId = static_cast(currentShape.front()); + } + } + } + for (SizeType i = 0; i < rowCount; ++i) { + ids[i] = startId + static_cast(i); + } + return ids; } diff --git a/src/nwb/hdmf/table/DynamicTable.hpp b/src/nwb/hdmf/table/DynamicTable.hpp index ea6dec833..a97339f97 100644 --- a/src/nwb/hdmf/table/DynamicTable.hpp +++ b/src/nwb/hdmf/table/DynamicTable.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include "Utils.hpp" #include "io/BaseIO.hpp" @@ -12,6 +14,11 @@ #include "nwb/hdmf/table/VectorData.hpp" #include "spec/hdmf_common.hpp" +namespace AQNWB::NWB +{ +class MeaningsTable; +} // namespace AQNWB::NWB + namespace AQNWB::NWB { /** @@ -24,6 +31,13 @@ namespace AQNWB::NWB class DynamicTable : public Container { public: + using DataSpec = Data::DataSpecBase; + using DataSpecPtr = std::shared_ptr; + +public: + using CellValue = AQNWB::NWB::CellValue; + using RowData = std::unordered_map; + // Register the TimeSeries as a subclass of Container REGISTER_SUBCLASS(DynamicTable, Container, @@ -45,18 +59,49 @@ class DynamicTable : public Container /** * @brief Initializes the `DynamicTable` object by creating NWB attributes and - * column names. + * optional configured data columns. * * @param description The description of the table (optional). + * @param dataSpecs Optional ordered configuration for the data objects to + * create. If empty, only the default id column is created. * @return Status::Success if successful, otherwise Status::Failure. + * @throw std::invalid_argument if the provided dataSpecs are invalid. + * + */ + Status initialize(const std::string& description, + const std::vector& dataSpecs = {}); + + /** + * @brief Validates the provided data specifications for the table. + * + * This method checks whether the provided data specifications are valid + * for the current table type. It can be called during initialization or + * separately by the user to validate a proposed schema. + * + * @param dataSpecs The data specifications to validate. + * @return Status::Success if the specifications are valid, otherwise + * Status::Failure. + */ + virtual Status validateDataSpecs( + const std::vector& dataSpecs) const; + + /** + * @brief Create the default data specs for a DynamicTable. + * + * The default DynamicTable schema consists only of the built-in + * ElementIdentifiers `id` dataset. Callers may use this as a starting point, + * modify the returned vector, and pass the result to initialize(). + * + * @param rowChunkSize Chunk size to use for the default id dataset. + * @return Ordered default data specs for the table. */ - Status initialize(const std::string& description); + static std::vector createDefaultDataSpecs( + const SizeType rowChunkSize = 100); /** * @brief Finalizes writing the DynamicTable. * - * Finalizes the DynamicTable by writing the column names - * as a single write once the table has been set up + * Finalizes the DynamicTable once the table has been set up. * * @return Status::Success if successful, otherwise Status::Failure. */ @@ -75,6 +120,34 @@ class DynamicTable : public Container Status addColumn(const std::shared_ptr& vectorData, const std::vector& values); + /** + * @brief Adds a column to the table from a DataSpecPtr. + * + * This is the preferred overload for adding columns after table creation + * (i.e. after initialize() has been called but before startRecording()). + * It is consistent with the DataSpec-based approach used by initialize() / + * configureDataObjects(): the spec bundles the column name, dataset + * configuration, and description, and the table creates and initializes the + * VectorData internally. Because DataSpec::initialize() is virtual, this + * overload works for any VectorData subtype whose DataSpec is registered. + * + * @param dataSpec The DataSpec describing the column to add. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status addColumn(const DataSpecPtr& dataSpec); + + /** + * @brief Adds an already-initialized column to the table without writing + * data. + * + * Kept for backward compatibility. Prefer addColumn(DataSpecPtr) for new + * code, which is consistent with the DataSpec-based API used by initialize(). + * + * @param vectorData A shared pointer to the already-initialized `VectorData`. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status addColumn(const std::shared_ptr& vectorData); + /** * @brief Adds a column of references to the table. * @param name The name of the column. @@ -87,32 +160,133 @@ class DynamicTable : public Container const std::vector& dataset); /** - * @brief Adds a column of element identifiers to the table. - * @param elementIDs A unique pointer to the `ElementIdentifiers` dataset. + * @brief Adds a single row to the table. + * @param row The row data to add. + * @param rowId The ID of the row (optional). + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status addRow(const AQNWB::Types::RowData& row, + const std::optional& rowId = std::nullopt); + + /** + * @brief Adds multiple rows to the table. + * @param rows The vector of row data to add. + * @param rowIds The vector of row IDs (optional). + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status addRows(const std::vector& rows, + const std::vector& rowIds = {}); + + /** + * @brief Get the number of rows in the table. + * @return The number of rows. + */ + SizeType getNumberOfRows() const; + + /** + * @brief Read rows from the table. + * + * @param start The starting row index. + * @param count The number of rows to read. If SizeTypeNotSet, reads to the + * end of the table. May be adjusted if start + count exceeds the number of + * rows. + * @param colNames The names of the columns to read. If empty, reads all + * columns. + * @param includeId Whether to include the 'id' column in the result. + * @return A vector of RowData objects. + * @throws std::invalid_argument if start index is out of bounds. + */ + std::vector readRows( + SizeType start = 0, + SizeType count = Types::SizeTypeNotSet, + const std::vector& colNames = {}, + bool includeId = true); + + /** + * @brief Convert the table to a string representation. + * + * @param start The starting row index. + * @param count The number of rows to read. + * @param colNames The names of the columns to read. If empty, reads all + * columns. + * @param includeId Whether to include the 'id' column in the result. + * @return A string representation of the table. + */ + std::string toString(SizeType start = 0, + SizeType count = Types::SizeTypeNotSet, + const std::vector& colNames = {}, + bool includeId = true); + + /** + * @brief Sets the values of the element identifiers on the table. * @param values The vector of id values. * @return Status::Success if successful, otherwise Status::Failure. */ - Status setRowIDs(const std::shared_ptr& elementIDs, - const std::vector& values); + Status setRowIDs(const std::vector& values); /** * @brief Sets the column names of the DynamicTable * - * ..note:: - * For this change to take affect in the file we need to call - * finalize() after setting the column names to write the data to the file. - * - * .. warning:: - * This will overwrite any existing column names. It is up to - * the caller to ensure that all existing columns are included in the new - * list. + * All changes to the column names will be flushed to the file. + * If the newColNames is identical to the existing column names, + * then no changes will be made. * + * @exception Throws std::invalid_argument if the newColNames vector does not + * contain all columns of the table. I.e., the newColNames vector + * must be a permutation of the existing column names. * @param newColNames The vector of new column names. */ - virtual void setColNames(const std::vector& newColNames) + virtual void setColNames(const std::vector& newColNames); + + /** + * @brief Create a MeaningsTable for a specific VectorData column in this + * DynamicTable. + * + * The function creates and fully initializes a MeaningsTable for the + * specified VectorData column in this DynamicTable. The MeaningsTable will + * be created at the path "meanings_tables/_meanings" within this + * DynamicTable. + * + * @param columnName The name of the VectorData column for which to create the + * MeaningsTable. + * @param rowChunkSize The chunk size for the rows in the MeaningsTable + * (optional, default: 100). + * @return A shared pointer to the created MeaningsTable, or nullptr if + * creation failed. + */ + std::shared_ptr createMeaningsTable( + const std::string& columnName, const SizeType rowChunkSize = 100); + + /** + * @brief Type trait to determine the return type of readColumn. + * + * If T is a subclass of VectorData, the return type is T. + * Otherwise, the return type is VectorDataTyped. + */ + template + struct ColumnReturnType { - m_colNames = newColNames; - } + using type = + typename std::conditional::value, + T, + VectorDataTyped>::type; + }; + + /** + * @brief Type trait identifying non-registered typed VectorData facades. + * + * VectorDataTyped instances provide compile-time value typing without + * becoming the single canonical RegisteredType cached for a dataset path. + */ + template + struct IsVectorDataTyped : std::false_type + { + }; + + template + struct IsVectorDataTyped> : std::true_type + { + }; /** * @brief Read an arbitrary column of the DynamicTable @@ -121,12 +295,20 @@ class DynamicTable : public Container * read functions are preferred because they help avoid the need for * specifying the specific name of the column and data type to use. * + * If the template parameter T is a subclass of VectorData (e.g., + * TimestampVectorData), the function returns a shared pointer to T. + * Otherwise, it returns a shared pointer to VectorDataTyped. + * + * @tparam T The data type of the column or the specific VectorData subclass. + * @param colName The name of the column to read. * @return The VectorData object representing the column or a nullptr if the - * column doesn't exists + * column doesn't exist. */ - template - std::shared_ptr> readColumn(const std::string& colName) + template + std::shared_ptr::type> readColumn( + const std::string& colName) { + using ReturnType = typename ColumnReturnType::type; std::string columnPath = AQNWB::mergePaths(m_path, colName); auto ioPtr = getIO(); if (ioPtr != nullptr) { @@ -134,7 +316,15 @@ class DynamicTable : public Container if (ioPtr->getStorageObjectType(columnPath) == StorageObjectType::Dataset) { - return VectorDataTyped::create(columnPath, ioPtr); + if constexpr (std::is_base_of::value + && !IsVectorDataTyped::value) + { + // Resolve the concrete on-disk registered type before caching it. + auto column = RegisteredType::create(columnPath, ioPtr); + return std::dynamic_pointer_cast(column); + } else { + return ReturnType::create(columnPath, ioPtr); + } } } } else { @@ -160,26 +350,246 @@ class DynamicTable : public Container "id", "unique identifiers for the rows of this dynamic table") + /** + * @brief Returns the instance of the class representing the named + * MeaningsTable field. + * + * This overload is declared here and defined in DynamicTable.cpp so callers + * that include only DynamicTable.hpp do not need MeaningsTable.hpp to use the + * default MeaningsTable accessor. + * + * @param objectName The name of the object to retrieve. + * @return A shared pointer to a MeaningsTable representing the object. May + * return nullptr if the path does not exist. + */ + std::shared_ptr readMeaningsTable( + const std::string& objectName) const; + + /** + * @brief Returns the instance of the class representing the named + * MeaningsTable field for write. + * + * This overload is declared here and defined in DynamicTable.cpp so callers + * that include only DynamicTable.hpp do not need MeaningsTable.hpp to use the + * default MeaningsTable accessor. + * + * **NOTE:** Use createMeaningsTable() to create and initialize a new + * MeaningsTable for a specific column. + * + * @param objectName The name of the object to retrieve. + * @return A shared pointer to a MeaningsTable representing the object. + */ + std::shared_ptr createMeaningsTableInstance( + const std::string& objectName) const; + protected: /** * @brief Add a column name to m_colNames while preventing duplicates + * + * If the column name already exists in m_colNames, it will not be added + * again. The function returns the index of the column name in m_colNames, + * whether it was newly added or already existed. If the column name is new, + * it will be appended to the end of m_colNames. Any changes to m_colNames + * will be flushed to the file. + * @param colName The name of the column to add * @return Index of the column name in m_colNames */ SizeType addColumnName(const std::string& colName); /** - * @brief Names of the columns in the table. + * @brief Flush the column names to the file. + * + * This function writes the current column names in m_colNames to the file as + * an attribute. + * + * @return Status::Success if successful, otherwise Status::Failure. */ - std::vector m_colNames; + + Status flushColNames(); + + /** + * @brief Internal state structure tracking a configured table column. + */ + struct ConfiguredColumn + { + std::string name; ///< The column name + IO::BaseDataType dataType; ///< The underlying data type + std::shared_ptr + column; ///< Shared pointer to the column dataset + }; + + /** + * @brief Look up an already-configured column by name. + * + * Configured columns are created and registered once during initialize() + * via the DataSpec mechanism. This accessor allows subclasses (and finalize() + * logic) to write data into the existing VectorData objects rather than + * creating new ones, which would register duplicate recording objects. + * + * @param name The name of the configured column to retrieve. + * @return A shared pointer to the configured VectorData column, or nullptr if + * no column with the given name has been configured. + */ + std::shared_ptr getConfiguredColumn( + const std::string& name) const; + + /** + * @brief Register a column in the table's column registry. + * + * Adds the column to the ordered list of configured columns, records its + * name-to-index lookup, and adds its name to the colnames attribute. If a + * column of the same name is already registered, returns the existing index + * and modifies no state. + */ + SizeType registerColumn(const std::shared_ptr& column); + + /** + * @brief Clear all registered columns and column names. + * + * Resets the internal registry of configured columns, the column names list, + * and the row identifiers dataset. This is used by initialize() before + * rebuilding the table's column configuration from data specs. + */ + void clearColumns(); + + /** + * @brief Configure multiple data objects. + * + * @param dataSpecs The specifications for the data objects to configure. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status configureDataObjects(const std::vector& dataSpecs); + + /** + * @brief Configure a single data object. + * + * @param dataSpec The specification for the data object to configure. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status configureDataObject(const DataSpec& dataSpec); + + /** + * @brief Ensure that all configured columns are loaded from the file. + * + * This function checks whether all configured columns have been loaded into + * memory. If any configured column is not yet loaded, it attempts to load it + * from the file. + * + * @return Status::Success if all configured columns are loaded successfully, + * otherwise Status::Failure. + */ + Status ensureConfiguredColumnsLoaded(); + + /** + * @brief Load all configured columns and row element identifiers from the + * file. + * + * Invoked automatically when rows are added to a previously initialized but + * newly loaded table. It registers existing columns based on colnames. + * + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status loadConfiguredColumnsFromFile(); /** - * @brief The columns added for recording + * @brief Validate the concrete specification, shape, and optionally data + * type of a required column. + * + * @tparam RequiredDataSpec The concrete DataSpec type required by the + * column. + * @param requiredName Name of the required column. + * @param dataSpecs Data specifications to inspect. + * @param expectedDataType Required data type, if the column has a fixed + * schema data type. + * @return Status::Success when every spec for the required column has the + * required concrete type, is one-dimensional, and has the expected data + * type when specified; otherwise Status::Failure. */ - std::unique_ptr m_recordingColumns; + template + Status checkRequiredColumnSpec(const std::string& requiredName, + const std::vector& dataSpecs, + const std::optional& + expectedDataType = std::nullopt) const + { + bool found = false; + for (const auto& spec : dataSpecs) { + if (!spec || spec->name != requiredName) { + continue; + } + + found = true; + if (!std::dynamic_pointer_cast(spec)) { + std::cerr << "DynamicTable::checkRequiredColumnSpec: required column '" + << requiredName + << "' does not use the required concrete DataSpec type." + << std::endl; + return Status::Failure; + } + if (spec->getShape().size() != 1) { + std::cerr << "DynamicTable::checkRequiredColumnSpec: required column '" + << requiredName << "' must be one-dimensional." << std::endl; + return Status::Failure; + } + if (expectedDataType.has_value() + && !(spec->getType() == expectedDataType.value())) + { + std::cerr << "DynamicTable::checkRequiredColumnSpec: required column '" + << requiredName << "' has an invalid data type." << std::endl; + return Status::Failure; + } + } + + if (!found) { + std::cerr << "DynamicTable::checkRequiredColumnSpec: required column '" + << requiredName << "' not found." << std::endl; + return Status::Failure; + } + return Status::Success; + } + + /** + * @brief Write a buffer of data to a configured column. + * + * This function writes the provided buffer of data to the specified + * configured column. The column must have been previously configured via + * the DataSpec mechanism during initialize(). + * + * @param configuredColumn The configured column to which to write data. + * @param buffer The buffer of data to write. + * @param rowCount The number of rows in the buffer. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status writeColumnBuffer( + const ConfiguredColumn& configuredColumn, + const IO::BaseDataType::BaseDataVectorVariant& buffer, + SizeType rowCount); + + /** + * @brief Generate row IDs for the table. + * + * @param rowCount The number of rows for which to generate IDs. + * @return A vector of generated row IDs. + */ + std::vector generateRowIDs(SizeType rowCount); + + /** + * @brief Names of the columns in the table. + */ + std::vector m_colNames; /** * @brief The row ids data object for write */ std::shared_ptr m_rowElementIdentifiers; + + /** + * @brief Ordered list of configured columns managed by the table. + */ + std::vector m_configuredColumns; + + /** + * @brief Lookup map from column name to its index in m_configuredColumns. + */ + std::unordered_map m_configuredColumnIndices; }; } // namespace AQNWB::NWB diff --git a/src/nwb/hdmf/table/ElementIdentifiers.hpp b/src/nwb/hdmf/table/ElementIdentifiers.hpp index 33e053675..cb58c3396 100644 --- a/src/nwb/hdmf/table/ElementIdentifiers.hpp +++ b/src/nwb/hdmf/table/ElementIdentifiers.hpp @@ -12,6 +12,28 @@ namespace AQNWB::NWB class ElementIdentifiers : public Data { public: + struct DataSpec : public Data::DataSpec + { + DataSpec(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig) + : Data::DataSpec(datasetName, dataConfig) + { + } + + Status initialize(Data& data) const override + { + auto* elementIdentifiers = dynamic_cast(&data); + if (!elementIdentifiers) { + std::cerr << "ElementIdentifiers::DataSpec::initialize received " + "incompatible Data object" + << std::endl; + return Status::Failure; + } + return elementIdentifiers->initialize( + static_cast(*this)); + } + }; + // Register ElementIdentifiers class as a registered type REGISTER_SUBCLASS(ElementIdentifiers, Data, @@ -34,6 +56,18 @@ class ElementIdentifiers : public Data using RegisteredType::m_io; using RegisteredType::m_path; + + static std::shared_ptr createDataSpec( + const std::string& name, const IO::ArrayDataSetConfig& dataConfig) + { + return std::make_shared(name, dataConfig); + } + + Status initialize(const IO::ArrayDataSetConfig& dataConfig) + { + return Data::initialize(dataConfig); + } + DEFINE_DATASET_FIELD(readData, recordData, int, "", The main data) }; } // namespace AQNWB::NWB diff --git a/src/nwb/hdmf/table/MeaningsTable.cpp b/src/nwb/hdmf/table/MeaningsTable.cpp new file mode 100644 index 000000000..b65c97491 --- /dev/null +++ b/src/nwb/hdmf/table/MeaningsTable.cpp @@ -0,0 +1,93 @@ +#include "MeaningsTable.hpp" + +#include "Utils.hpp" +// Includes for referenced types +#include "nwb/hdmf/table/VectorData.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(MeaningsTable) + +// Constructor +MeaningsTable::MeaningsTable(const std::string& path, + std::shared_ptr io) + : DynamicTable(path, io) +{ +} + +Status MeaningsTable::validateDataSpecs( + const std::vector& dataSpecs) const +{ + Status status = DynamicTable::validateDataSpecs(dataSpecs); + if (status != Status::Success) { + return status; + } + + status = checkRequiredColumnSpec("value", dataSpecs); + if (status != Status::Success) { + return status; + } + + return checkRequiredColumnSpec( + "meaning", dataSpecs, IO::BaseDataType(IO::BaseDataType::V_STR)); +} + +std::vector MeaningsTable::createDefaultDataSpecs( + const AQNWB::IO::BaseDataType& valueDataType, const SizeType rowChunkSize) +{ + std::vector specs = + DynamicTable::createDefaultDataSpecs(rowChunkSize); + + IO::ArrayDataSetConfig valueConfig( + valueDataType, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "value", + valueConfig, + "The value of a row in the referenced VectorData object.")); + + IO::ArrayDataSetConfig meaningConfig( + IO::BaseDataType::V_STR, SizeArray {0}, SizeArray {rowChunkSize}); + specs.push_back(std::make_shared( + "meaning", + meaningConfig, + "The meaning of the value in the referenced VectorData object.")); + + return specs; +} + +// Initialize the object +Status MeaningsTable::initialize(const VectorData& targetVectorData, + const AQNWB::IO::BaseDataType& valueDataType, + const std::string& description, + const std::vector& columnSpecs) +{ + Status initStatus = Status::Success; + + // Retrieve the IO object + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr << "IO object has been deleted. Can't initialize MeaningsTable: " + << m_path << std::endl; + return Status::Failure; + } + + std::vector specsToUse = columnSpecs; + if (specsToUse.empty()) { + specsToUse = createDefaultDataSpecs(valueDataType); + } + + // Call parent initialize method. This initializes the DynamicTable and + // creates the description attribute. + Status parentInitStatus = DynamicTable::initialize(description, specsToUse); + initStatus = initStatus && parentInitStatus; + + // Set the target VectorData reference as an attribute + Status targetStatus = ioPtr->createReferenceAttribute( + targetVectorData.getPath(), m_path, "target"); + + // Update the overall status and return the final status + initStatus = initStatus && targetStatus; + return initStatus; +} diff --git a/src/nwb/hdmf/table/MeaningsTable.hpp b/src/nwb/hdmf/table/MeaningsTable.hpp new file mode 100644 index 000000000..139be4e09 --- /dev/null +++ b/src/nwb/hdmf/table/MeaningsTable.hpp @@ -0,0 +1,113 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/hdmf/table/DynamicTable.hpp" +// Forward declarations for referenced types +namespace AQNWB::NWB +{ +class ElementIdentifiers; +class VectorData; +} // namespace AQNWB::NWB +// Include for the namespace schema header +#include "spec/hdmf_common.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief A table to store information about the meanings of values in a + * referenced VectorData object. All possible values of the referenced + * VectorData object should be present in the 'value' column of this table, even + * if the value is not observed in the data. Additional columns may be added to + * store additional metadata about each value. The name of the MeaningsTable + * should correspond to the name of the referenced VectorData object with a + * "_meanings" suffix. e.g., if the referenced VectorData object is named + * "stimulus_type", the corresponding MeaningsTable should be named + * "stimulus_type_meanings". + */ +class MeaningsTable : public AQNWB::NWB::DynamicTable +{ +public: + /** + * @brief Virtual destructor. + */ + virtual ~MeaningsTable() override {} + + /** + * @brief Validates the provided data specifications for the MeaningsTable. + * @param dataSpecs The data specifications to validate. + * @return Status::Success if the specifications are valid, otherwise + * Status::Failure. + */ + Status validateDataSpecs( + const std::vector& dataSpecs) const override; + + /** + * @brief Creates the default data specs for the MeaningsTable. + * @param valueDataType The data type of the 'value' column in the + * MeaningsTable. + * @param rowChunkSize The chunk size for the rows of the table. + * @return A vector of DataSpecPtr containing the default specs. + */ + static std::vector createDefaultDataSpecs( + const AQNWB::IO::BaseDataType& valueDataType, + const SizeType rowChunkSize = 100); + + /** + * @brief Initialize the object + * @param targetVectorData The VectorData object that this MeaningsTable is + * associated with. + * @param valueDataType The data type of the 'value' column in the + * MeaningsTable. + * @param description Description of the table (optional) + * @param columnSpecs The column specifications to use for initialization. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initialize(const VectorData& targetVectorData, + const AQNWB::IO::BaseDataType& valueDataType, + const std::string& description = + "A table to store information about the meanings of " + "values in a referenced VectorData object.", + const std::vector& columnSpecs = {}); + + DEFINE_REGISTERED_FIELD( + readValueColumn, + AQNWB::NWB::VectorData, + "value", + "The value of a row in the referenced VectorData object.") + + DEFINE_REGISTERED_FIELD( + readMeaningColumn, + AQNWB::NWB::VectorDataTyped, + "meaning", + "The meaning of the value in the referenced VectorData object.") + + DEFINE_REFERENCED_REGISTERED_FIELD(readTarget, + AQNWB::NWB::VectorData, + "target", + "Reference to the VectorData object for " + "which this table provides meanings.") + + REGISTER_SUBCLASS(MeaningsTable, + DynamicTable, + AQNWB::SPEC::HDMF_COMMON::namespaceName) + +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + MeaningsTable(const std::string& path, std::shared_ptr io); +}; + +} // namespace AQNWB::NWB diff --git a/src/nwb/hdmf/table/VectorData.cpp b/src/nwb/hdmf/table/VectorData.cpp index 39ee49bf2..b83d28227 100644 --- a/src/nwb/hdmf/table/VectorData.cpp +++ b/src/nwb/hdmf/table/VectorData.cpp @@ -1,5 +1,7 @@ #include "nwb/hdmf/table/VectorData.hpp" +#include "io/RecordingObjects.hpp" + using namespace AQNWB::NWB; // Register the base VectorData class @@ -10,8 +12,286 @@ VectorData::VectorData(const std::string& path, std::shared_ptr io) { } +VectorData::DataSpec::DataSpec(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig, + const std::string& columnDescription) + : Data::DataSpec(datasetName, dataConfig) + , description(columnDescription) +{ +} + +Status VectorData::DataSpec::initialize(Data& data) const +{ + auto* vectorData = dynamic_cast(&data); + if (!vectorData) { + std::cerr << "VectorData::DataSpec::initialize received incompatible Data " + "object" + << std::endl; + return Status::Failure; + } + return vectorData->initialize( + static_cast(*this), description); +} + +std::shared_ptr VectorData::createReferenceVectorData( + const std::string& path, + std::shared_ptr io, + const std::string& description, + const std::vector& references) +{ + Status dataStatus = io->createReferenceDataSet(path, references); + if (dataStatus != Status::Success) { + return nullptr; + } + + auto vectorData = VectorData::create(path, io); + Status commonAttrsStatus = io->createCommonNWBAttributes( + path, vectorData->getNamespace(), vectorData->getTypeName()); + Status attrStatus = io->createAttribute(description, path, "description"); + if ((attrStatus && commonAttrsStatus) != Status::Success) { + return nullptr; + } + + return vectorData; +} + +std::shared_ptr VectorData::createDataSpec( + const std::string& name, + const IO::ArrayDataSetConfig& dataConfig, + const std::string& description) +{ + return std::make_shared(name, dataConfig, description); +} + +Status VectorData::initialize(const IO::BaseArrayDataSetConfig& dataConfig, + const std::string& description) +{ + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr << "IO object has been deleted. Can't initialize VectorData: " + << m_path << std::endl; + return Status::Failure; + } + + Status dataStatus = Data::initialize(dataConfig); + if (dataConfig.isLink()) { + const auto* linkConfig = + dynamic_cast(&dataConfig); + if (linkConfig) { + return dataStatus + && linkConfig->validateTarget(*ioPtr, {}, {}, {"description"}); + } + return dataStatus; + } + + Status attrStatus = + ioPtr->createAttribute(description, m_path, "description"); + return dataStatus && attrStatus; +} + +Status VectorData::appendData(const AQNWB::Types::CellValue& cellValue, + size_t& elementsAppended) +{ + elementsAppended = 0; + + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr + << "IO object has been deleted. Can't append value to VectorData: " + << m_path << std::endl; + return Status::Failure; + } + + auto dataset = recordData(); + if (!dataset) { + std::cerr << "VectorData::appendValue: dataset is not initialized." + << std::endl; + return Status::Failure; + } + + auto dataType = ioPtr->getStorageObjectDataType(this->getPath()); + if (!dataType.isCompatibleCellValue(cellValue)) { + std::cerr << "VectorData::appendData: value type does not match dataset " + "type." + << std::endl; + return Status::Failure; + } + + SizeArray positionOffset = {0}; + auto currentShape = dataset->getShape(); + if (!currentShape.empty()) { + positionOffset[0] = currentShape[0]; + } + + // The cellValue contains a variant that can hold either a scalar or a vector. + // The vector case is used for ragged arrays where the VectorData is indexed + // by a VectorIndex, in which case a single cell of a table can contain + // multiple elements. We use std::visit to handle both cases. + return std::visit( + [&](auto&& arg) -> Status + { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return Status::Failure; + } else { + // Check if the value is a scalar + if (auto* scalarVariant = + std::get_if( + &cellValue.value)) + { + elementsAppended = 1; + // Visit the inner scalar variant to get the actual value + return std::visit( + [&](auto&& scalarArg) -> Status + { + using ScalarT = std::decay_t; + if constexpr (std::is_same_v) { + return Status::Failure; + } else { + // Strings need to be passed as a vector to writeDataBlock + if constexpr (std::is_same_v) { + std::vector strVec = {scalarArg}; + return dataset->writeDataBlock( + SizeArray {1}, positionOffset, dataType, strVec); + } else { + // Other scalar types can be passed by pointer + return dataset->writeDataBlock( + SizeArray {1}, positionOffset, dataType, &scalarArg); + } + } + }, + *scalarVariant); + } + // Check if the value is a vector (for ragged arrays) + else if (auto* vectorVariant = + std::get_if( + &cellValue.value)) + { + // Visit the inner vector variant to get the actual vector + return std::visit( + [&](auto&& vectorArg) -> Status + { + using VectorT = std::decay_t; + if constexpr (std::is_same_v) { + return Status::Failure; + } else { + elementsAppended = vectorArg.size(); + if (elementsAppended == 0) { + return Status::Success; // Nothing to append + } + // Strings are passed directly as a vector + if constexpr (std::is_same_v>) { + return dataset->writeDataBlock( + SizeArray {static_cast(elementsAppended)}, + positionOffset, + dataType, + vectorArg); + } else { + // Other vector types are passed by pointer to their + // underlying data + return dataset->writeDataBlock( + SizeArray {static_cast(elementsAppended)}, + positionOffset, + dataType, + vectorArg.data()); + } + } + }, + *vectorVariant); + } + return Status::Failure; + } + }, + cellValue.value); +} + +Status VectorData::appendBuffer( + const IO::BaseDataType::BaseDataVectorVariant& buffer) +{ + auto ioPtr = getIO(); + if (ioPtr == nullptr) { + std::cerr + << "IO object has been deleted. Can't append buffer to VectorData: " + << m_path << std::endl; + return Status::Failure; + } + + auto dataset = recordData(); + if (!dataset) { + std::cerr << "VectorData::appendBuffer: dataset is not initialized." + << std::endl; + return Status::Failure; + } + + auto dataType = ioPtr->getStorageObjectDataType(this->getPath()); + if (!dataType.isCompatibleVector(buffer)) { + std::cerr << "VectorData::appendBuffer: buffer type does not match dataset " + "type." + << std::endl; + return Status::Failure; + } + + SizeArray positionOffset = {0}; + auto currentShape = dataset->getShape(); + if (!currentShape.empty()) { + positionOffset[0] = currentShape[0]; + } + + return std::visit( + [&](const auto& vec) -> Status + { + using VecType = std::decay_t; + if constexpr (std::is_same_v) { + return Status::Failure; + } else { + if (vec.empty()) { + return Status::Success; + } + SizeArray dataShape = {static_cast(vec.size())}; + if constexpr (std::is_same_v>) { + return dataset->writeDataBlock( + dataShape, positionOffset, dataType, vec); + } else { + return dataset->writeDataBlock( + dataShape, positionOffset, dataType, vec.data()); + } + } + }, + buffer); +} + namespace AQNWB::NWB { + +template +std::shared_ptr> VectorDataTyped::create( + const std::string& path, std::shared_ptr io) +{ + if (io) { + auto recordingObjects = io->getRecordingObjects(); + if (recordingObjects) { + auto existingObj = recordingObjects->getRecordingObject(path); + if (existingObj) { + auto typedObj = + std::dynamic_pointer_cast>(existingObj); + if (typedObj) { + return typedObj; + } + // If an object with the same path exists but is not of this exact type + // (e.g., it's a base VectorData), we create a new VectorDataTyped + // instance but do NOT add it to the recording objects cache to avoid + // overwriting the existing VectorDataobject or causing conflicts. + return std::shared_ptr>( + new VectorDataTyped(path, io)); + } + } + return std::shared_ptr>( + new VectorDataTyped(path, io)); + } + return nullptr; +} + // Explicitly instantiate the VectorDataTyped template for all common data // types. This ensures that these specializations are generated by the compiler, // reducing compile times and ensuring availability throughout the program. diff --git a/src/nwb/hdmf/table/VectorData.hpp b/src/nwb/hdmf/table/VectorData.hpp index 4cfe0cada..3282d5a8d 100644 --- a/src/nwb/hdmf/table/VectorData.hpp +++ b/src/nwb/hdmf/table/VectorData.hpp @@ -1,7 +1,11 @@ #pragma once +#include #include +#include +#include +#include "Types.hpp" #include "Utils.hpp" #include "io/ReadIO.hpp" #include "nwb/hdmf/base/Data.hpp" @@ -9,12 +13,32 @@ namespace AQNWB::NWB { + +using CellValue = AQNWB::Types::CellValue; + /** * @brief An n-dimensional dataset representing a column of a DynamicTable. */ class VectorData : public Data { public: + /** + * @brief Runtime configuration for creating and initializing a VectorData + * column. + */ + struct DataSpec : public Data::DataSpec + { + DataSpec(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig, + const std::string& columnDescription); + + virtual ~DataSpec() = default; + + std::string description; + + Status initialize(Data& data) const override; + }; + REGISTER_SUBCLASS(VectorData, Data, AQNWB::SPEC::HDMF_COMMON::namespaceName) protected: @@ -46,23 +70,21 @@ class VectorData : public Data const std::string& path, std::shared_ptr io, const std::string& description, - const std::vector& references) - { - Status dataStatus = io->createReferenceDataSet(path, references); - if (dataStatus != Status::Success) { - return nullptr; - } - - auto vectorData = VectorData::create(path, io); - Status commonAttrsStatus = io->createCommonNWBAttributes( - path, vectorData->getNamespace(), vectorData->getTypeName()); - Status attrStatus = io->createAttribute(description, path, "description"); - if ((attrStatus && commonAttrsStatus) != Status::Success) { - return nullptr; - } - - return vectorData; - } + const std::vector& references); + + /** + * @brief Create a ColumnSpec for configuring this type as a DynamicTable + * column. + * + * @param name The column name. + * @param dataConfig Dataset configuration for the column. + * @param description The column description attribute. + * @return Shared pointer to the ColumnSpec for this column type. + */ + static std::shared_ptr createDataSpec( + const std::string& name, + const IO::ArrayDataSetConfig& dataConfig, + const std::string& description); /** * @brief Initialize the dataset for the VectorData object @@ -74,32 +96,27 @@ class VectorData : public Data * @return Status::Success if successful, otherwise Status::Failure. */ Status initialize(const IO::BaseArrayDataSetConfig& dataConfig, - const std::string& description) - { - auto ioPtr = getIO(); - if (ioPtr == nullptr) { - std::cerr << "IO object has been deleted. Can't initialize VectorData: " - << m_path << std::endl; - return Status::Failure; - } - Status dataStatus = Data::initialize(dataConfig); - if (dataConfig.isLink()) { - // For links, we don't set attributes since there is no dataset to attach - // them to. Validate that the target has the required "description" - // attribute. - const auto* linkConfig = - dynamic_cast(&dataConfig); - if (linkConfig) { - return dataStatus - && linkConfig->validateTarget(*ioPtr, {}, {}, {"description"}); - } - return dataStatus; - } else { - Status attrStatus = - ioPtr->createAttribute(description, m_path, "description"); - return dataStatus && attrStatus; - } - } + const std::string& description); + + /** + * @brief Appends a cell value (scalar or vector) to the dataset. + * + * @param cellValue The value to append. + * @param elementsAppended Output parameter that will be set to the number of + * elements appended. + * @return Status::Success if successful, otherwise Status::Failure. + */ + virtual Status appendData(const AQNWB::Types::CellValue& cellValue, + size_t& elementsAppended); + + /** + * @brief Appends a buffer of values to the dataset. + * + * @param buffer The buffer of values to append. + * @return Status::Success if successful, otherwise Status::Failure. + */ + virtual Status appendBuffer( + const IO::BaseDataType::BaseDataVectorVariant& buffer); DEFINE_ATTRIBUTE_FIELD(readDescription, std::string, @@ -152,10 +169,7 @@ class VectorDataTyped : public VectorData * creation failed. */ static std::shared_ptr create( - const std::string& path, std::shared_ptr io) - { - return RegisteredType::create(path, io); - } + const std::string& path, std::shared_ptr io); /** * @brief Virtual destructor. @@ -167,9 +181,10 @@ class VectorDataTyped : public VectorData * * This function is useful when the type of the data is known and we want * read data in a typed manner where the type is stored in the DTYPE template - * parameter. NOTE: The original Data object retains ownership of the - * Data.m_dataset recording dataset object if it was initialized, i.e., - * the returned VectorDataTyped object will have a nullptr m_dataset. + * parameter. NOTE: The BaseRecordingData object used for recording to the + * dataset is cached in the BaseIO object and is, hence, shared between the + * VectorData and the VectorDataTyped, such that recording to either object + * will be reflected in the other. * * @param data The Data object to convert * @return A shared pointer for VectorDataTyped object with the same path and diff --git a/src/nwb/hdmf/table/VectorIndex.cpp b/src/nwb/hdmf/table/VectorIndex.cpp new file mode 100644 index 000000000..932550012 --- /dev/null +++ b/src/nwb/hdmf/table/VectorIndex.cpp @@ -0,0 +1,528 @@ +#include + +#include "nwb/hdmf/table/VectorIndex.hpp" + +#include "Utils.hpp" + +using namespace AQNWB::NWB; +using namespace AQNWB::IO; + +// Initialize the static registered_ member to trigger registration +REGISTER_SUBCLASS_IMPL(VectorIndex) + +// Constructor +VectorIndex::VectorIndex(const std::string& path, + std::shared_ptr io) + : VectorData(path, io) +{ +} + +// Initialize the object +Status VectorIndex::initialize(const IO::BaseArrayDataSetConfig& dataConfig, + const std::string& description, + const std::string& targetPath) +{ + Status initStatus = Status::Success; + + // Get the io pointer. + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr << "VectorIndex:initialize IO object has been deleted." + << std::endl; + return Status::Failure; + } + if (!ioPtr->canModifyObjects()) { + std::cerr << "VectorIndex::initialize IO object cannot modify objects." + << std::endl; + return Status::Failure; + } + + // Make sure that dataConfig uses a unsigned int data type as required + // Extract shape, chunking, and data type from the config + SizeArray shape, chunking; + BaseDataType dataType; + Status propStatus = + dataConfig.getProperties(ioPtr.get(), shape, chunking, dataType); + if (propStatus != Status::Success) { + std::cerr << "VectorIndex::initialize could not retrieve dataset properties" + << std::endl; + return Status::Failure; + } + + if (dataType.type != IO::BaseDataType::Type::T_U8 + && dataType.type != IO::BaseDataType::Type::T_U16 + && dataType.type != IO::BaseDataType::Type::T_U32 + && dataType.type != IO::BaseDataType::Type::T_U64) + { + throw std::invalid_argument( + "VectorIndex::initialize invalid dataConfig requires a unsigned int " + "type"); + } + if (shape.size() != 1) { + throw std::invalid_argument( + "VectorIndex::initialize invalid dataConfig requires 1D shape"); + } + + // Call parent initialize method. + Status parentInitStatus = VectorData::initialize(dataConfig, description); + initStatus = initStatus && parentInitStatus; + + // Initialize the target attribute + Status targetStatus = + ioPtr->createReferenceAttribute(targetPath, getPath(), "target"); + initStatus = initStatus && targetStatus; + + // Save the data type and current index for use in appendRow + m_dataType = dataType; + m_dataTypeInitialized = true; + m_currentIndex = 0; + m_currentIndexInitialized = true; + + return initStatus; +} + +std::shared_ptr VectorIndex::getTargetColumn() +{ + if (!m_targetColumn) { + m_targetColumn = readTarget(); + } + return m_targetColumn; +} + +uint64_t VectorIndex::extractIndexValue( + const AQNWB::Types::CellValue& cell) const +{ + uint64_t index = 0; + // A CellValue contains a variant (cell.value) which can be a scalar or a + // vector. We expect an index to be a scalar value. + std::visit( + [&](auto&& arg) + { + using T = std::decay_t; + // Check if the variant holds a scalar value + if constexpr (std::is_same_v) { + // The scalar value itself is another variant holding the actual type + // (e.g., uint32_t, uint64_t) + std::visit( + [&](auto&& scalarArg) + { + using ScalarT = std::decay_t; + // Ensure the scalar is an integral type before casting to + // uint64_t + if constexpr (std::is_integral_v) { + index = static_cast(scalarArg); + } + }, + arg); + } + }, + cell.value); + return index; +} + +AQNWB::Types::CellValue VectorIndex::combineCellsToVector( + const std::vector& cells, + size_t offset, + size_t count) const +{ + // Handle edge cases where the requested slice is empty or out of bounds + if (offset >= cells.size() || count == 0) { + return AQNWB::Types::CellValue(std::vector {}); + } + + AQNWB::Types::CellValue result(std::vector {}); + + // We need to determine the underlying data type of the cells to create a + // vector of the same type. We inspect the first cell in the requested slice + // to find its type. + std::visit( + [&](auto&& arg) + { + using T = std::decay_t; + // We expect the individual cells to be scalar values + if constexpr (std::is_same_v) { + std::visit( + [&](auto&& scalarArg) + { + using ScalarT = std::decay_t; + // Ensure the scalar is not empty (monostate) + if constexpr (!std::is_same_v) { + // Create a vector of the discovered type to hold the combined + // elements + std::vector vec; + vec.reserve(count); + + // Iterate over the requested slice of cells + for (size_t i = 0; i < count; ++i) { + if (offset + i >= cells.size()) { + break; + } + + // Extract the value from each cell and add it to our vector + std::visit( + [&](auto&& cellArg) + { + using CellT = std::decay_t; + if constexpr (std::is_same_v< + CellT, + AQNWB::Types::ScalarDataVariant>) + { + std::visit( + [&](auto&& cellScalarArg) + { + using CellScalarT = + std::decay_t; + // Only add the value if its type matches the + // type of the first cell + if constexpr (std::is_same_v) { + vec.push_back(cellScalarArg); + } + }, + cellArg); + } + }, + cells[offset + i].value); + } + // Wrap the populated vector in a CellValue + result = AQNWB::Types::CellValue(std::move(vec)); + } + }, + arg); + } + }, + cells[offset].value); + + return result; +} + +std::vector VectorIndex::readIndexedCellValues( + SizeType start, SizeType count, SizeType stride, SizeType block) +{ + if (count == 0) { + return {}; + } + if (stride == 0) { + throw std::invalid_argument( + "VectorIndex::readIndexedCellValues: stride must be greater than 0"); + } + if (block == 0) { + throw std::invalid_argument( + "VectorIndex::readIndexedCellValues: block must be greater than 0"); + } + + auto ioPtr = getIO(); + if (!ioPtr || !ioPtr->isOpen()) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: IO object has been deleted or is " + "closed."); + } + + auto dataset = readData(); + if (!dataset || !dataset->exists()) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: dataset is not initialized or " + "does not exist."); + } + + // First, read the indices from this VectorIndex + SizeArray startArray = {start}; + SizeArray countArray = + count != AQNWB::Types::SizeTypeNotSet ? SizeArray {count} : SizeArray {}; + SizeArray strideArray = {stride}; + SizeArray blockArray = {block}; + + std::vector indices = VectorData::readCellValues( + startArray, countArray, strideArray, blockArray); + + if (indices.empty()) { + return {}; + } + + // Get the target column + auto target = getTargetColumn(); + if (!target) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: target VectorData is not " + "available."); + } + + // We need to read the previous index to know where the first vector starts + // If start == 0, the previous index is 0 + uint64_t prevIndex = 0; + if (start > 0) { + SizeArray prevStart = {start - 1}; + SizeArray prevCount = {1}; + std::vector prevIndices = + VectorData::readCellValues(prevStart, prevCount); + if (!prevIndices.empty()) { + prevIndex = extractIndexValue(prevIndices[0]); + } + } + + // Extract all current indices + std::vector currIndices; + currIndices.reserve(indices.size()); + std::transform(indices.begin(), + indices.end(), + std::back_inserter(currIndices), + [this](const auto& indexCell) + { return extractIndexValue(indexCell); }); + + // Validate indices and find total range to read + uint64_t totalElementsToRead = 0; + uint64_t lastIndex = prevIndex; + for (uint64_t currIndex : currIndices) { + if (currIndex < lastIndex) { + throw std::invalid_argument( + "VectorIndex::readIndexedCellValues: Invalid index data, " + "currIndex (" + + std::to_string(currIndex) + ") < prevIndex (" + + std::to_string(lastIndex) + ")"); + } + lastIndex = currIndex; + } + + if (!currIndices.empty()) { + totalElementsToRead = currIndices.back() - prevIndex; + } + + // Validate that the indices do not exceed the target dataset size + if (totalElementsToRead > 0) { + SizeArray targetShape; + bool targetExists = ioPtr->objectExists(target->getPath()); + if (!targetExists) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: Target dataset does not exist."); + } + + try { + targetShape = ioPtr->getStorageObjectShape(target->getPath()); + } catch (const std::exception& e) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: Failed to get target dataset " + "shape: " + + std::string(e.what())); + } + + if (!targetShape.empty() && currIndices.back() > targetShape[0]) { + throw std::out_of_range( + "VectorIndex::readIndexedCellValues: Index out of bounds. " + "Max index (" + + std::to_string(currIndices.back()) + + ") exceeds target dataset size (" + std::to_string(targetShape[0]) + + ")."); + } else if (targetShape.empty() && currIndices.back() > 0) { + throw std::out_of_range( + "VectorIndex::readIndexedCellValues: Index out of bounds. " + "Max index (" + + std::to_string(currIndices.back()) + + ") exceeds target dataset size (0)."); + } + } + + // Empty rows have no values from which to infer their element type, so use + // the target dataset metadata to create correctly typed empty vectors. + auto targetData = target->readData(); + if (!targetData) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: target data is unavailable."); + } + const auto emptyTargetValues = + IO::BaseDataType::createEmptyVectorVariant(targetData->getDataType()); + if (std::holds_alternative(emptyTargetValues)) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: target data type is unsupported."); + } + + // Read nonempty target values in one operation before slicing them into rows. + std::vector allTargetCells; + if (totalElementsToRead > 0) { + SizeArray targetStart = {static_cast(prevIndex)}; + SizeArray targetCount = {static_cast(totalElementsToRead)}; + try { + allTargetCells = target->readCellValues(targetStart, targetCount); + } catch (const std::out_of_range&) { + throw; // Re-throw out_of_range + } catch (const std::exception& e) { + throw std::runtime_error( + "VectorIndex::readIndexedCellValues: Failed to read target cells: " + + std::string(e.what())); + } + } + + std::vector result; + result.reserve(indices.size()); + + // Slice the bulk-read target cells into individual vectors + size_t targetCellOffset = 0; + for (uint64_t currIndex : currIndices) { + uint64_t numElements = currIndex - prevIndex; + + if (numElements == 0) { + result.emplace_back(emptyTargetValues); + } else { + result.push_back( + combineCellsToVector(allTargetCells, targetCellOffset, numElements)); + targetCellOffset += numElements; + } + + prevIndex = currIndex; + } + + return result; +} + +Status VectorIndex::initializeAppendState() +{ + if (m_currentIndexInitialized && m_dataTypeInitialized) { + return Status::Success; + } + + auto dataset = readData(); + if (!dataset || !dataset->exists()) { + std::cerr + << "VectorIndex::initializeAppendState: dataset is not initialized." + << std::endl; + return Status::Failure; + } + + auto ioPtr = getIO(); + if (!ioPtr) { + std::cerr + << "VectorIndex::initializeAppendState: IO object has been deleted." + << std::endl; + return Status::Failure; + } + + if (!m_dataTypeInitialized) { + try { + m_dataType = ioPtr->getStorageObjectDataType(dataset->getPath()); + m_dataTypeInitialized = true; + } catch (const std::exception& e) { + std::cerr + << "VectorIndex::initializeAppendState: error getting data type: " + << e.what() << std::endl; + return Status::Failure; + } + } + + if (m_currentIndexInitialized) { + return Status::Success; + } + + auto shape = dataset->getShape(); + if (shape.empty() || shape[0] == 0) { + m_currentIndex = 0; + m_currentIndexInitialized = true; + return Status::Success; + } + + // Read the last value + SizeArray positionOffset = {shape[0] - 1}; + SizeArray readShape = {1}; + uint64_t lastValue = 0; + + Status readStatus = Status::Failure; + + try { + auto dataBlock = dataset->valuesGeneric(positionOffset, readShape); + auto variantData = dataBlock.as_variant(); + + if (m_dataType.type == IO::BaseDataType::Type::T_U8) { + if (auto* vec = std::get_if>(&variantData)) { + if (!vec->empty()) { + lastValue = vec->front(); + readStatus = Status::Success; + } + } + } else if (m_dataType.type == IO::BaseDataType::Type::T_U16) { + if (auto* vec = std::get_if>(&variantData)) { + if (!vec->empty()) { + lastValue = vec->front(); + readStatus = Status::Success; + } + } + } else if (m_dataType.type == IO::BaseDataType::Type::T_U32) { + if (auto* vec = std::get_if>(&variantData)) { + if (!vec->empty()) { + lastValue = vec->front(); + readStatus = Status::Success; + } + } + } else if (m_dataType.type == IO::BaseDataType::Type::T_U64) { + if (auto* vec = std::get_if>(&variantData)) { + if (!vec->empty()) { + lastValue = vec->front(); + readStatus = Status::Success; + } + } + } else { + std::cerr << "VectorIndex::initializeAppendState: unsupported data type." + << std::endl; + return Status::Failure; + } + } catch (const std::exception& e) { + std::cerr << "VectorIndex::initializeAppendState: error reading data: " + << e.what() << std::endl; + return Status::Failure; + } + + if (readStatus == Status::Success) { + m_currentIndex = lastValue; + m_currentIndexInitialized = true; + } + + return readStatus; +} + +Status VectorIndex::appendData(const AQNWB::Types::CellValue& targetValues, + size_t& elementsAppended) +{ + if (!m_currentIndexInitialized || !m_dataTypeInitialized) { + Status initStatus = initializeAppendState(); + if (initStatus != Status::Success) { + return initStatus; + } + } + + auto target = getTargetColumn(); + if (!target) { + std::cerr << "VectorIndex::appendData: target VectorData is not available." + << std::endl; + return Status::Failure; + } + + elementsAppended = 0; + Status appendStatus = target->appendData(targetValues, elementsAppended); + if (appendStatus != Status::Success) { + return appendStatus; + } + + m_currentIndex += elementsAppended; + + // Now append the new index to this VectorIndex + // We need to append the value based on the actual data type of the dataset + auto ioPtr = getIO(); + if (!ioPtr) { + return Status::Failure; + } + size_t indexElementsAppended = 0; + + if (m_dataType.type == IO::BaseDataType::Type::T_U8) { + CellValue indexValue(static_cast(m_currentIndex)); + return VectorData::appendData(indexValue, indexElementsAppended); + } else if (m_dataType.type == IO::BaseDataType::Type::T_U16) { + CellValue indexValue(static_cast(m_currentIndex)); + return VectorData::appendData(indexValue, indexElementsAppended); + } else if (m_dataType.type == IO::BaseDataType::Type::T_U32) { + CellValue indexValue(static_cast(m_currentIndex)); + return VectorData::appendData(indexValue, indexElementsAppended); + } else if (m_dataType.type == IO::BaseDataType::Type::T_U64) { + CellValue indexValue(m_currentIndex); + return VectorData::appendData(indexValue, indexElementsAppended); + } else { + std::cerr << "VectorIndex::appendData: unsupported data type." << std::endl; + return Status::Failure; + } +} diff --git a/src/nwb/hdmf/table/VectorIndex.hpp b/src/nwb/hdmf/table/VectorIndex.hpp new file mode 100644 index 000000000..d60129dd1 --- /dev/null +++ b/src/nwb/hdmf/table/VectorIndex.hpp @@ -0,0 +1,253 @@ +#pragma once + +// Common STL includes +#include +#include +#include +#include +// Base AqNWB includes for IO and RegisteredType +#include "io/BaseIO.hpp" +#include "io/ReadIO.hpp" +#include "nwb/RegisteredType.hpp" +// Include for parent type +#include "nwb/hdmf/table/VectorData.hpp" +// Includes for types that are referenced and used +#include "nwb/hdmf/base/Data.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +// Include for the namespace schema header +#include "spec/hdmf_common.hpp" + +namespace AQNWB::NWB +{ + +/** + * @brief Used with VectorData to encode a ragged array. An array of indices + * into the first dimension of the target VectorData, and forming a map between + * the rows of a DynamicTable and the indices of the VectorData. The name of the + * VectorIndex is expected to be the name of the target VectorData object + * followed by "_index". + * + * In a ragged array, each row can have a different number of elements. The + * first vector is at VectorData[0:VectorIndex[0]]. The second vector is at + * VectorData[VectorIndex[0]:VectorIndex[1]], and so on. This is using + * half-open intervals, so the first index is inclusive and the second index is + * exclusive. So for example, if VectorIndex = [3, 5, 8], then the first vector + * is at VectorData[0:3], the second vector is at VectorData[3:5], and the third + * vector is at VectorData[5:8], while the total number of elements in the + * VectorData is 8. + */ +class VectorIndex : public AQNWB::NWB::VectorData +{ +protected: + /** + * @brief Constructor + * @param path Path to the object in the file + * @param io IO object for reading/writing + */ + VectorIndex(const std::string& path, std::shared_ptr io); + +public: + /** + * @brief Runtime configuration for creating and initializing a VectorIndex + * column. + */ + struct DataSpec : public Data::DataSpec + { + DataSpec(const std::string& datasetName, + const IO::ArrayDataSetConfig& dataConfig, + const std::string& columnDescription, + const std::string& columnTargetPath) + : Data::DataSpec(datasetName, dataConfig) + , description(columnDescription) + , targetPath(columnTargetPath) + { + } + + virtual ~DataSpec() = default; + + std::string description; + std::string targetPath; + + Status initialize(Data& data) const override + { + auto* vectorIndex = dynamic_cast(&data); + if (!vectorIndex) { + std::cerr << "VectorIndex::DataSpec::initialize received incompatible " + "VectorIndex object" + << std::endl; + return Status::Failure; + } + return vectorIndex->initialize( + static_cast(*this), + description, + targetPath); + } + }; + + /** + * @brief Virtual destructor. + */ + virtual ~VectorIndex() override {} + + /** + * @brief Initialize the dataset for the VectorIndex object + * + * This function creates a dataset using the provided configuration + * + * @param dataConfig The configuration for the dataset. Must use an + * unsigned int data type. + * @param description The description of the VectorIndex + * @param targetPath The path to the target VectorData + * @throw std::invalid_argument if the provided dataSpecs are invalid. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initialize(const IO::BaseArrayDataSetConfig& dataConfig, + const std::string& description, + const std::string& targetPath); + + /** + * @brief Appends data to the target VectorData and updates this + * index. + * + * This method delegates the actual appending of values to the target + * VectorData, determines how many elements were appended, increments its + * internal current index, and writes the new index value to its own dataset. + * + * @param targetValues The values to append to the target VectorData. + * @param elementsAppended Output parameter that will be set to the number of + * elements appended. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status appendData(const AQNWB::Types::CellValue& targetValues, + size_t& elementsAppended) override; + + /** + * @brief Gets the cached target VectorData column, or reads it if not cached. + * + * @return The target VectorData column, or nullptr if not available. + */ + std::shared_ptr getTargetColumn(); + + /** + * @brief Reads a range of ragged array vectors from the target VectorData. + * + * For VectorIndex, this reads the indices and then uses them to read the + * corresponding ragged array vectors from the target VectorData. + * + * @param start The starting index for the slice (optional). + * @param count The number of elements to read (optional). + * @param stride The stride (optional). + * @param block The block size (optional). + * @return A vector of CellValue, where each CellValue contains the data for a + * cell. + * @throws std::out_of_range if the requested range is invalid (e.g., start + + * count exceeds the number of rows), or if the indices point outside the + * target dataset. + * @throws std::runtime_error if the VectorData cannot be read, if the target + * column is not available, or if the IO object is closed. + * @throws std::invalid_argument if the indices stored in the VectorIndex are + * invalid (e.g., not monotonically increasing). + */ + std::vector readIndexedCellValues( + SizeType start = 0, + SizeType count = AQNWB::Types::SizeTypeNotSet, + SizeType stride = 1, + SizeType block = 1); + + // Define read methods + DEFINE_REFERENCED_REGISTERED_FIELD( + readTarget, + VectorData, + "target", + "Reference to the target dataset that this index applies to.") + + DEFINE_ATTRIBUTE_FIELD(readDescription, + std::string, + "description", + "Description of what these vectors represent.") + + DEFINE_DATASET_FIELD( + readData, + recordData, + uint32_t, + "", + "Used with VectorData to encode a ragged array. An array of indices into " + "the first dimension of the target VectorData - and forming a map " + "between the rows of a DynamicTable and the indices of the VectorData. " + "The name of the VectorIndex is expected to be the name of the target " + "VectorData object followed by _index.") + + REGISTER_SUBCLASS(VectorIndex, + VectorData, + AQNWB::SPEC::HDMF_COMMON::namespaceName) + +private: + /** + * @brief The current index value, representing the total number of elements + * currently in the target VectorData. This is the same as the last value in + * this VectorIndex. + */ + uint64_t m_currentIndex = 0; + + /** + * @brief Flag indicating whether m_currentIndex has been initialized from the + * file. + */ + bool m_currentIndexInitialized = false; + + /** + * @brief Cached target VectorData column to avoid reading it from the file + * repeatedly. + */ + std::shared_ptr m_targetColumn; + + /** + * @brief Cached data type of the VectorIndex dataset. + */ + IO::BaseDataType m_dataType; + + /** + * @brief Flag indicating whether m_dataType has been initialized. + */ + bool m_dataTypeInitialized = false; + + /** + * @brief Initializes m_currentIndex and m_dataType by reading the last value + * from the dataset, or setting it to 0 if the dataset is empty. This is used + * for resuming appending of index values to the VectorIndex. + * @return Status::Success if successful, otherwise Status::Failure. + */ + Status initializeAppendState(); + + /** + * @brief Extracts the integer index value from a CellValue variant. + * + * This helper function safely extracts the underlying integer value from a + * CellValue that is expected to contain a scalar integer. + * + * @param cell The CellValue containing the index. + * @return The extracted index as a uint64_t. Returns 0 if the cell does not + * contain a valid integer. + */ + uint64_t extractIndexValue(const AQNWB::Types::CellValue& cell) const; + + /** + * @brief Combines multiple scalar CellValues into a single vector CellValue. + * + * This helper function takes a slice of scalar CellValues (representing + * individual elements of a ragged array vector) and combines them into a + * single CellValue containing a std::vector of the appropriate type. + * + * @param cells The vector of scalar CellValues to combine. + * @param offset The starting index in the cells vector. + * @param count The number of elements to combine. + * @return A CellValue containing a vector of the combined elements. Returns + * an empty vector CellValue if count is 0 or offset is out of bounds. + */ + AQNWB::Types::CellValue combineCellsToVector( + const std::vector& cells, + size_t offset, + size_t count) const; +}; + +} // namespace AQNWB::NWB diff --git a/src/spec/core.hpp b/src/spec/core.hpp index b94720c9b..8ed0e0101 100644 --- a/src/spec/core.hpp +++ b/src/spec/core.hpp @@ -10,7 +10,7 @@ /*! * \namespace AQNWB::SPEC::CORE * @brief Namespace for the core schema specification. - * @version 2.9.0 + * @version 2.11.0 * * This file is auto-generated from the schema files. * Do not edit this file manually. @@ -20,23 +20,26 @@ namespace AQNWB::SPEC::CORE const std::string namespaceName = "core"; -const std::string version = "2.9.0"; +const std::string version = "2.11.0"; // clang-format off constexpr std::string_view nwb_base = R"delimiter( -{"datasets":[{"neurodata_type_def":"NWBData","neurodata_type_inc":"Data","doc":"An abstract data type for a dataset."},{"neurodata_type_def":"TimeSeriesReferenceVectorData","neurodata_type_inc":"VectorData","default_name":"timeseries","dtype":[{"name":"idx_start","dtype":"int32","doc":"Start index into the TimeSeries 'data' and 'timestamp' datasets of the referenced TimeSeries. The first dimension of those arrays is always time."},{"name":"count","dtype":"int32","doc":"Number of data samples available in this time series, during this epoch"},{"name":"timeseries","dtype":{"target_type":"TimeSeries","reftype":"object"},"doc":"The TimeSeries that this index applies to"}],"doc":"Column storing references to a TimeSeries (rows). For each TimeSeries this VectorData column stores the start_index and count to indicate the range in time to be selected as well as an object reference to the TimeSeries."},{"neurodata_type_def":"BaseImage","neurodata_type_inc":"NWBData","doc":"An abstract base type for image data. Parent type for Image and ExternalImage types.","attributes":[{"name":"description","dtype":"text","doc":"Description of the image.","required":false}]},{"neurodata_type_def":"Image","neurodata_type_inc":"BaseImage","dtype":"numeric","dims":[["x","y"],["x","y","r, g, b"],["x","y","r, g, b, a"]],"shape":[[null,null],[null,null,3],[null,null,4]],"doc":"A type for storing image data directly. Shape can be 2-D (x, y), or 3-D where the third dimension can have three or four elements, e.g. (x, y, (r, g, b)) or (x, y, (r, g, b, a)).","attributes":[{"name":"resolution","dtype":"float32","doc":"Pixel resolution of the image, in pixels per centimeter.","required":false}]},{"neurodata_type_def":"ExternalImage","neurodata_type_inc":"BaseImage","doc":"A type for referencing an external image file. The single file path or URI to the external image file should be stored in the dataset. This type should NOT be used if the image is stored in another NWB file and that file is linked to this file.","dtype":"text","attributes":[{"name":"image_mode","dtype":"text","doc":"Image mode (color mode) of the image, e.g., \"RGB\", \"RGBA\", \"grayscale\", and \"LA\".","required":false},{"name":"image_format","dtype":"text","doc":"Common name of the image file format. Only widely readable, open file formats are allowed. Allowed values are \"PNG\", \"JPEG\", and \"GIF\".","required":true}]},{"neurodata_type_def":"ImageReferences","neurodata_type_inc":"NWBData","dtype":{"target_type":"BaseImage","reftype":"object"},"dims":["num_images"],"shape":[null],"doc":"Ordered dataset of references to BaseImage (e.g., Image or ExternalImage) objects."}],"groups":[{"neurodata_type_def":"NWBContainer","neurodata_type_inc":"Container","doc":"An abstract data type for a generic container storing collections of data and metadata. Base type for all data and metadata containers."},{"neurodata_type_def":"NWBDataInterface","neurodata_type_inc":"NWBContainer","doc":"An abstract data type for a generic container storing collections of data, as opposed to metadata."},{"neurodata_type_def":"TimeSeries","neurodata_type_inc":"NWBDataInterface","doc":"General purpose time series.","attributes":[{"name":"description","dtype":"text","default_value":"no description","doc":"Description of the time series.","required":false},{"name":"comments","dtype":"text","default_value":"no comments","doc":"Human-readable comments about the TimeSeries. This second descriptive field can be used to store additional information, or descriptive information if the primary description field is populated with a computer-readable string.","required":false}],"datasets":[{"name":"data","dims":[["num_times"],["num_times","num_DIM2"],["num_times","num_DIM2","num_DIM3"],["num_times","num_DIM2","num_DIM3","num_DIM4"]],"shape":[[null],[null,null],[null,null,null],[null,null,null,null]],"doc":"Data values. Data can be in 1-D, 2-D, 3-D, or 4-D. The first dimension should always represent time. This can also be used to store binary data (e.g., image frames). This can also be a link to data stored in an external file.","attributes":[{"name":"conversion","dtype":"float32","default_value":1.0,"doc":"Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. e.g. if the data acquisition system stores values in this object as signed 16-bit integers (int16 range -32,768 to 32,767) that correspond to a 5V range (-2.5V to 2.5V), and the data acquisition system gain is 8000X, then the 'conversion' multiplier to get from raw data acquisition values to recorded volts is 2.5/32768/8000 = 9.5367e-9.","required":false},{"name":"offset","dtype":"float32","default_value":0.0,"doc":"Scalar to add to the data after scaling by 'conversion' to finalize its coercion to the specified 'unit'. Two common examples of this include (a) data stored in an unsigned type that requires a shift after scaling to re-center the data, and (b) specialized recording devices that naturally cause a scalar offset with respect to the true units.","required":false},{"name":"resolution","dtype":"float32","default_value":-1.0,"doc":"Smallest meaningful difference between values in data, stored in the specified by unit, e.g., the change in value of the least significant bit, or a larger number if signal noise is known to be present. If unknown, use -1.0.","required":false},{"name":"unit","dtype":"text","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."},{"name":"continuity","dtype":"text","doc":"Optionally describe the continuity of the data. Can be \"continuous\", \"instantaneous\", or \"step\". For example, a voltage trace would be \"continuous\", because samples are recorded from a continuous process. An array of lick times would be \"instantaneous\", because the data represents distinct moments in time. Times of image presentations would be \"step\" because the picture remains the same until the next timepoint. This field is optional, but is useful in providing information about the underlying data. It may inform the way this data is interpreted, the way it is visualized, and what analysis methods are applicable.","required":false}]},{"name":"starting_time","dtype":"float64","doc":"Timestamp of the first sample in seconds. When timestamps are uniformly spaced, the timestamp of the first sample can be specified and all subsequent ones calculated from the sampling rate attribute.","quantity":"?","attributes":[{"name":"rate","dtype":"float32","doc":"Sampling rate, in Hz."},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for time, which is fixed to 'seconds'."}]},{"name":"timestamps","dtype":"float64","dims":["num_times"],"shape":[null],"doc":"Timestamps for samples stored in data, in seconds, relative to the common experiment master-clock stored in NWBFile.timestamps_reference_time.","quantity":"?","attributes":[{"name":"interval","dtype":"int32","value":1,"doc":"Value is '1'"},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for timestamps, which is fixed to 'seconds'."}]},{"name":"control","dtype":"uint8","dims":["num_times"],"shape":[null],"doc":"Numerical labels that apply to each time point in data for the purpose of querying and slicing data by these values. If present, the length of this array should be the same size as the first dimension of data.","quantity":"?"},{"name":"control_description","dtype":"text","dims":["num_control_values"],"shape":[null],"doc":"Description of each control value. Must be present if control is present. If present, control_description[0] should describe time points where control == 0.","quantity":"?"}],"groups":[{"name":"sync","doc":"Lab-specific time and sync information as provided directly from hardware devices and that is necessary for aligning all acquired time information to a common timebase. The timestamp array stores time in the common timebase. This group will usually only be populated in TimeSeries that are stored external to the NWB file, in files storing raw data. Once timestamp data is calculated, the contents of 'sync' are mostly for archival purposes.","quantity":"?"}]},{"neurodata_type_def":"ProcessingModule","neurodata_type_inc":"NWBContainer","doc":"A collection of processed data.","attributes":[{"name":"description","dtype":"text","doc":"Description of this collection of processed data."}],"groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Data objects stored in this collection.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tables stored in this collection.","quantity":"*"}]},{"neurodata_type_def":"Images","neurodata_type_inc":"NWBDataInterface","default_name":"Images","doc":"A collection of images with an optional way to specify the order of the images using the \"order_of_images\" dataset. An order must be specified if the images are referenced by index, e.g., from an IndexSeries.","attributes":[{"name":"description","dtype":"text","doc":"Description of this collection of images."}],"datasets":[{"neurodata_type_inc":"BaseImage","doc":"Images stored in this collection.","quantity":"+"},{"name":"order_of_images","neurodata_type_inc":"ImageReferences","doc":"Ordered dataset of references to BaseImage objects stored in the parent group. Each object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.","quantity":"?"}]}]})delimiter"; +{"datasets":[{"neurodata_type_def":"NWBData","neurodata_type_inc":"Data","doc":"An abstract data type for a dataset."},{"neurodata_type_def":"TimeSeriesReferenceVectorData","neurodata_type_inc":"VectorData","default_name":"timeseries","dtype":[{"name":"idx_start","dtype":"int32","doc":"Start index into the TimeSeries 'data' and 'timestamp' datasets of the referenced TimeSeries. The first dimension of those arrays is always time."},{"name":"count","dtype":"int32","doc":"Number of data samples available in this time series, during this epoch."},{"name":"timeseries","dtype":{"target_type":"TimeSeries","reftype":"object"},"doc":"The TimeSeries object that this reference refers to."}],"doc":"Column storing references into one or more TimeSeries objects. Each row of this column stores a start index and count into a referenced TimeSeries and an object reference to that TimeSeries."},{"neurodata_type_def":"BaseImage","neurodata_type_inc":"NWBData","doc":"An abstract base type for image data. Parent type for Image and ExternalImage types.","attributes":[{"name":"description","dtype":"text","doc":"Description of the image.","required":false}]},{"neurodata_type_def":"Image","neurodata_type_inc":"BaseImage","dtype":"numeric","dims":[["x","y"],["x","y","r, g, b"],["x","y","r, g, b, a"]],"shape":[[null,null],[null,null,3],[null,null,4]],"doc":"A type for storing image data directly. Shape can be 2-D (x, y), or 3-D where the third dimension can have three or four elements, e.g., (x, y, (r, g, b)) or (x, y, (r, g, b, a)).","attributes":[{"name":"resolution","dtype":"float32","doc":"Pixel resolution of the image, in pixels per centimeter.","required":false}]},{"neurodata_type_def":"ExternalImage","neurodata_type_inc":"BaseImage","doc":"A type for referencing an external image file. The single file path or URI to the external image file should be stored in the dataset. This type should NOT be used if the image is stored in another NWB file and that file is linked to this file.","dtype":"text","attributes":[{"name":"image_mode","dtype":"text","doc":"Image mode (color mode) of the image, e.g., \"RGB\", \"RGBA\", \"grayscale\", and \"LA\".","required":false},{"name":"image_format","dtype":"text","doc":"Common name of the image file format. Only widely readable, open file formats are allowed. Allowed values are \"PNG\", \"JPEG\", and \"GIF\".","required":true}]},{"neurodata_type_def":"ImageReferences","neurodata_type_inc":"NWBData","dtype":{"target_type":"BaseImage","reftype":"object"},"dims":["num_images"],"shape":[null],"doc":"Ordered dataset of references to BaseImage (e.g., Image or ExternalImage) objects."}],"groups":[{"neurodata_type_def":"NWBContainer","neurodata_type_inc":"Container","doc":"An abstract data type for a generic container storing collections of data and metadata. Base type for all data and metadata containers."},{"neurodata_type_def":"NWBDataInterface","neurodata_type_inc":"NWBContainer","doc":"An abstract data type for a generic container storing collections of data, as opposed to metadata."},{"neurodata_type_def":"TimeSeries","neurodata_type_inc":"NWBDataInterface","doc":"General purpose time series.","attributes":[{"name":"description","dtype":"text","default_value":"no description","doc":"Description of the time series.","required":false},{"name":"comments","dtype":"text","default_value":"no comments","doc":"Human-readable comments about the TimeSeries. This second descriptive field can be used to store additional information, or descriptive information if the primary description field is populated with a computer-readable string.","required":false}],"datasets":[{"name":"data","dims":[["num_times"],["num_times","num_DIM2"],["num_times","num_DIM2","num_DIM3"],["num_times","num_DIM2","num_DIM3","num_DIM4"]],"shape":[[null],[null,null],[null,null,null],[null,null,null,null]],"doc":"Data values. Data can be in 1-D, 2-D, 3-D, or 4-D. The first dimension should always represent time. This can also be used to store binary data (e.g., image frames). This can also be a link to data stored in an external file.","attributes":[{"name":"conversion","dtype":"float32","default_value":1.0,"doc":"Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. For example, if the data acquisition system stores values in this object as signed 16-bit integers (int16 range -32,768 to 32,767) that correspond to a 5V range (-2.5V to 2.5V), and the data acquisition system gain is 8000X, then the 'conversion' multiplier to get from raw data acquisition values to recorded volts is 2.5/32768/8000 = 9.5367e-9.","required":false},{"name":"offset","dtype":"float32","default_value":0.0,"doc":"Scalar to add to the data after scaling by 'conversion' to finalize its coercion to the specified 'unit'. Two common examples of this include (a) data stored in an unsigned type that requires a shift after scaling to re-center the data, and (b) specialized recording devices that naturally cause a scalar offset with respect to the true units.","required":false},{"name":"resolution","dtype":"float32","default_value":-1.0,"doc":"Smallest meaningful difference between values in data, stored in the specified unit, e.g., the change in value of the least significant bit, or a larger number if signal noise is known to be present. If unknown, use -1.0.","required":false},{"name":"unit","dtype":"text","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."},{"name":"continuity","dtype":"text","doc":"Optionally describe the continuity of the data. Can be \"continuous\", \"instantaneous\", or \"step\". For example, a voltage trace would be \"continuous\", because samples are recorded from a continuous process. An array of lick times would be \"instantaneous\", because the data represents distinct moments in time. Times of image presentations would be \"step\" because the picture remains the same until the next timepoint. This field is optional, but is useful in providing information about the underlying data. It may inform the way this data is interpreted, the way it is visualized, and what analysis methods are applicable. For storing instantaneous event information, it is recommended to use an EventsTable instead of a TimeSeries with continuity set to \"instantaneous\".","required":false}]},{"name":"starting_time","dtype":"float64","doc":"Timestamp of the first sample in seconds. When timestamps are uniformly spaced, the timestamp of the first sample can be specified and all subsequent ones calculated from the sampling rate attribute.","quantity":"?","attributes":[{"name":"rate","dtype":"float32","doc":"Sampling rate, in Hz."},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for time, which is fixed to 'seconds'."}]},{"name":"timestamps","dtype":"float64","dims":["num_times"],"shape":[null],"doc":"Timestamps for samples stored in data, in seconds, relative to the common experiment master clock stored in NWBFile.timestamps_reference_time.","quantity":"?","attributes":[{"name":"interval","dtype":"int32","value":1,"doc":"Value is '1'"},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for timestamps, which is fixed to 'seconds'."}]},{"name":"control","dtype":"uint8","dims":["num_times"],"shape":[null],"doc":"Numerical labels that apply to each time point in data for the purpose of querying and slicing data by these values. If present, the length of this array should be the same as the length of the first dimension of data.","quantity":"?"},{"name":"control_description","dtype":"text","dims":["num_control_values"],"shape":[null],"doc":"Description of each control value. Must be present if control is present. If present, control_description[0] should describe time points where control == 0.","quantity":"?"}],"groups":[{"name":"sync","doc":"Lab-specific time and sync information as provided directly from hardware devices and that is necessary for aligning all acquired time information to a common timebase. The timestamp array stores time in the common timebase. This group will usually only be populated in TimeSeries that are stored external to the NWB file, in files storing raw data. Once timestamp data is calculated, the contents of 'sync' are mostly for archival purposes.","quantity":"?"}]},{"neurodata_type_def":"ProcessingModule","neurodata_type_inc":"NWBContainer","doc":"A collection of processed data.","attributes":[{"name":"description","dtype":"text","doc":"Description of this collection of processed data."}],"groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Data objects stored in this collection.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tables stored in this collection.","quantity":"*"}]},{"neurodata_type_def":"Images","neurodata_type_inc":"NWBDataInterface","default_name":"Images","doc":"A collection of images with an optional way to specify the order of the images using the \"order_of_images\" dataset. An order must be specified if the images are referenced by index, e.g., from an IndexSeries.","attributes":[{"name":"description","dtype":"text","doc":"Description of this collection of images."}],"datasets":[{"neurodata_type_inc":"BaseImage","doc":"Images stored in this collection.","quantity":"+"},{"name":"order_of_images","neurodata_type_inc":"ImageReferences","doc":"Ordered dataset of references to BaseImage objects stored in the parent group. Each object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.","quantity":"?"}]}]})delimiter"; constexpr std::string_view nwb_device = R"delimiter( {"groups":[{"neurodata_type_def":"Device","neurodata_type_inc":"NWBContainer","doc":"Metadata about a specific instance of a data acquisition device, e.g., recording system, electrode, microscope. Link to a DeviceModel.model to represent information about the model of the device.","attributes":[{"name":"description","dtype":"text","doc":"Description of the device as free-form text. If there is any software/firmware associated with the device, the names and versions of those can be added to NWBFile.was_generated_by.","required":false},{"name":"manufacturer","dtype":"text","doc":"DEPRECATED. The name of the manufacturer of the device, e.g., Imec, Plexon, Thorlabs. Instead of using this field, store the value in DeviceModel.manufacturer and link to that DeviceModel from this Device.","required":false},{"name":"model_number","dtype":"text","doc":"DEPRECATED. The model number (or part/product number) of the device, e.g., PRB_1_4_0480_1, PLX-VP-32-15SE(75)-(260-80)(460-10)-300-(1)CON/32m-V, BERGAMO. Instead of using this field, store the value in DeviceModel.model_number and link to that DeviceModel from this Device.","required":false},{"name":"model_name","dtype":"text","doc":"DEPRECATED. The model name of the device, e.g., Neuropixels 1.0, V-Probe, Bergamo III. Instead of using this field, create and add a new DeviceModel named the model name and link to that DeviceModel from this Device.","required":false},{"name":"serial_number","dtype":"text","doc":"The serial number of the device.","required":false}],"links":[{"name":"model","target_type":"DeviceModel","doc":"The model of the device.","quantity":"?"}]},{"neurodata_type_def":"DeviceModel","neurodata_type_inc":"NWBContainer","doc":"Model properties of a data acquisition device, e.g., recording system, electrode, microscope. This should be extended for specific types of device models to include additional attributes specific to each type. The name of the DeviceModel should be the most common representation of the model name, e.g., Neuropixels 1.0, V-Probe, Bergamo III.","attributes":[{"name":"manufacturer","dtype":"text","doc":"The name of the manufacturer of the device model, e.g., Imec, Plexon, Thorlabs."},{"name":"model_number","dtype":"text","doc":"The model number (or part/product number) of the device, e.g., PRB_1_4_0480_1, PLX-VP-32-15SE(75)-(260-80)(460-10)-300-(1)CON/32m-V, BERGAMO.","required":false},{"name":"description","dtype":"text","doc":"Description of the device model as free-form text.","required":false}]}]})delimiter"; constexpr std::string_view nwb_epoch = R"delimiter( -{"groups":[{"neurodata_type_def":"TimeIntervals","neurodata_type_inc":"DynamicTable","doc":"A container for aggregating epoch data and the TimeSeries that each epoch applies to.","datasets":[{"name":"start_time","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Start time of epoch, in seconds."},{"name":"stop_time","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Stop time of epoch, in seconds."},{"name":"tags","neurodata_type_inc":"VectorData","dtype":"text","doc":"User-defined tags that identify or categorize events.","quantity":"?"},{"name":"tags_index","neurodata_type_inc":"VectorIndex","doc":"Index for tags.","quantity":"?"},{"name":"timeseries","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"An index into a TimeSeries object.","quantity":"?"},{"name":"timeseries_index","neurodata_type_inc":"VectorIndex","doc":"Index for timeseries.","quantity":"?"}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"TimeIntervals","neurodata_type_inc":"DynamicTable","doc":"A column-based table to store information about time intervals, one interval per row. Use `TimeIntervals` when each row naturally has a start and stop time and represents a contiguous period of time, often encompassing an experimental condition. Rows can optionally reference `TimeSeries` data via the `timeseries` column. Examples include trials, epochs, behavioral states (e.g., \"running\", \"resting\"), and invalid recording windows.","datasets":[{"name":"start_time","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Start time of epoch, in seconds."},{"name":"stop_time","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Stop time of epoch, in seconds."},{"name":"tags","neurodata_type_inc":"VectorData","dtype":"text","doc":"User-defined tags that identify or categorize events.","quantity":"?"},{"name":"tags_index","neurodata_type_inc":"VectorIndex","doc":"Index for tags.","quantity":"?"},{"name":"timeseries","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"An index into a TimeSeries object.","quantity":"?"},{"name":"timeseries_index","neurodata_type_inc":"VectorIndex","doc":"Index for timeseries.","quantity":"?"}]}]})delimiter"; + +constexpr std::string_view nwb_event = R"delimiter( +{"datasets":[{"neurodata_type_def":"TimestampVectorData","neurodata_type_inc":"VectorData","dtype":"float","dims":["num_times"],"shape":[null],"doc":"A 1-D VectorData that stores timestamps in seconds from the session start time. Timestamps are not required to be sorted in time.","attributes":[{"name":"unit","dtype":"text","value":"seconds","doc":"The unit of measurement for the timestamps, fixed to 'seconds'."},{"name":"resolution","dtype":"float","doc":"The temporal resolution of the timestamps, in seconds. This is typically the sampling period (1 / sampling_rate), also known as the clock period, of the data acquisition system from which the timestamps were recorded or derived.","required":false}]},{"neurodata_type_def":"DurationVectorData","neurodata_type_inc":"VectorData","dtype":"float","dims":["num_times"],"shape":[null],"doc":"A 1-D VectorData that stores durations in seconds.","attributes":[{"name":"unit","dtype":"text","value":"seconds","doc":"The unit of measurement for the durations, fixed to 'seconds'."},{"name":"resolution","dtype":"float","doc":"The temporal resolution of the durations, in seconds. This is typically the sampling period (1 / sampling_rate), also known as the clock period, of the data acquisition system from which the durations were recorded or derived.","required":false}]}],"groups":[{"neurodata_type_def":"EventsTable","neurodata_type_inc":"DynamicTable","doc":"A column-based table to store information about events, one event per row. Use `EventsTable` when each row is anchored at a single timestamp and duration is absent, optional, or mixed across rows. Additional columns may be added to store metadata about each event, such as the duration of the event. Examples include TTL pulses, licks, rewards, stimulus onsets, and detected ripples. Each `EventsTable` should hold events of a single type, so that all rows share the same set of per-event metadata columns. Events of different types (e.g., licks and stimulus presentations) should be stored in separate `EventsTable` instances.","attributes":[{"name":"description","dtype":"text","doc":"A description of the events stored in the table, including information about how the event times were computed, especially if the times are the result of processing or filtering raw data. For example, if the experimenter is encoding different types of events using a strobed or N-bit encoding, then the description should describe which channels were used and how the event time is computed, e.g., as the rise time of the first bit."},{"name":"source_description","dtype":"text","doc":"Optional short text description of where the events came from, applying to every row in the table. For example, \"Acquisition system\" for events emitted directly by the acquisition system (e.g., TTL edges or hardware event channels); \"Thresholding of analog signal ANALOG1 at 3 V\" for events produced by a detection algorithm run on acquired data; or \"Manual video review\" for events added by a human annotator. This is a free-text label of origin only; use `description` for the longer narrative of how the event times were computed (channels used, encoding scheme, algorithm parameters, etc.).","required":false}],"datasets":[{"name":"timestamp","neurodata_type_inc":"TimestampVectorData","doc":"Column containing the time that each event occurred, in seconds, from the session start time."},{"name":"duration","neurodata_type_inc":"DurationVectorData","doc":"Optional column containing the duration of each event, in seconds. A value of NaN can be used for events without a duration or with a duration that is not yet specified.","quantity":"?"},{"name":"annotation","neurodata_type_inc":"VectorData","doc":"Column containing user annotations about events.","dtype":"text","dims":["num_times"],"shape":[null],"quantity":"?"}]}]})delimiter"; constexpr std::string_view nwb_image = R"delimiter( -{"datasets":[{"neurodata_type_def":"GrayscaleImage","neurodata_type_inc":"Image","dims":["x","y"],"shape":[null,null],"doc":"A grayscale image.","dtype":"numeric"},{"neurodata_type_def":"RGBImage","neurodata_type_inc":"Image","dims":["x","y","r, g, b"],"shape":[null,null,3],"doc":"A color image.","dtype":"numeric"},{"neurodata_type_def":"RGBAImage","neurodata_type_inc":"Image","dims":["x","y","r, g, b, a"],"shape":[null,null,4],"doc":"A color image with transparency.","dtype":"numeric"}],"groups":[{"neurodata_type_def":"ImageSeries","neurodata_type_inc":"TimeSeries","doc":"General image data that is common between acquisition and stimulus time series. Sometimes the image data is stored in the file in a raw format while other times it will be stored as a series of external image files in the host file system. The data field will either be binary data, if the data is stored in the NWB file, or empty, if the data is stored in an external image stack. [frame][x][y] or [frame][x][y][z].","datasets":[{"name":"data","dtype":"numeric","dims":[["frame","x","y"],["frame","x","y","z"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"Binary data representing images across frames. If data are stored in an external file, this should be an empty 3D array."},{"name":"dimension","dtype":"int32","dims":["rank"],"shape":[null],"doc":"Number of pixels on x, y, (and z) axes.","quantity":"?"},{"name":"external_file","dtype":"text","dims":["num_files"],"shape":[null],"doc":"Paths to one or more external file(s). The field is only present if format='external'. This is only relevant if the image series is stored in the file system as one or more image file(s). This field should NOT be used if the image is stored in another NWB file and that file is linked to this file.","quantity":"?","attributes":[{"name":"starting_frame","dtype":"int32","dims":["num_files"],"shape":[null],"doc":"Each external image may contain one or more consecutive frames of the full ImageSeries. This attribute serves as an index to indicate which frames each file contains, to facilitate random access. The 'starting_frame' attribute, hence, contains a list of frame numbers within the full ImageSeries of the first frame of each file listed in the parent 'external_file' dataset. Zero-based indexing is used (hence, the first element will always be zero). For example, if the 'external_file' dataset has three paths to files and the first file has 5 frames, the second file has 10 frames, and the third file has 20 frames, then this attribute will have values [0, 5, 15]. If there is a single external file that holds all of the frames of the ImageSeries (and so there is a single element in the 'external_file' dataset), then this attribute should have value [0]."}]},{"name":"format","dtype":"text","default_value":"raw","doc":"Format of image. If this is 'external', then the attribute 'external_file' contains the path information to the image files. If this is 'raw', then the raw (single-channel) binary data is stored in the 'data' dataset. If this attribute is not present, then the default format='raw' case is assumed.","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the Device object that was used to capture these images.","quantity":"?"}]},{"neurodata_type_def":"ImageMaskSeries","neurodata_type_inc":"ImageSeries","doc":"DEPRECATED. An alpha mask that is applied to a presented visual stimulus. The 'data' array contains an array of mask values that are applied to the displayed image. Mask values are stored as RGBA. Mask can vary with time. The timestamps array indicates the starting time of a mask, and that mask pattern continues until it's explicitly changed.","links":[{"name":"masked_imageseries","target_type":"ImageSeries","doc":"Link to ImageSeries object that this image mask is applied to."}]},{"neurodata_type_def":"OpticalSeries","neurodata_type_inc":"ImageSeries","doc":"Image data that is presented or recorded. A stimulus template movie will be stored only as an image. When the image is presented as stimulus, additional data is required, such as field of view (e.g., how much of the visual field the image covers, or how what is the area of the target being imaged). If the OpticalSeries represents acquired imaging data, orientation is also important.","datasets":[{"name":"distance","dtype":"float32","doc":"Distance from camera/monitor to target/eye.","quantity":"?"},{"name":"field_of_view","dtype":"float32","dims":[["width, height"],["width, height, depth"]],"shape":[[2],[3]],"doc":"Width, height and depth of image, or imaged area, in meters.","quantity":"?"},{"name":"data","dtype":"numeric","dims":[["frame","x","y"],["frame","x","y","r, g, b"]],"shape":[[null,null,null],[null,null,null,3]],"doc":"Images presented to subject, either grayscale or RGB"},{"name":"orientation","dtype":"text","doc":"Description of image relative to some reference frame (e.g., which way is up). Must also specify frame of reference.","quantity":"?"}]},{"neurodata_type_def":"IndexSeries","neurodata_type_inc":"TimeSeries","doc":"Stores indices that reference images defined in other containers. The primary purpose of the IndexSeries is to allow images stored in an Images container to be referenced in a specific sequence through the 'indexed_images' link. This approach avoids duplicating image data when the same image needs to be presented multiple times or when images need to be shown in a different order than they are stored. Since images in an Images container do not have an inherent order, the Images container needs to include an 'order_of_images' dataset (of type ImageReferences) when being referenced by an IndexSeries. This dataset establishes the ordered sequence that the indices in IndexSeries refer to. The 'data' field stores the index into this ordered sequence, and the 'timestamps' array indicates the precise presentation time of each indexed image during an experiment. This can be used for displaying individual images or creating movie segments by referencing a sequence of images with the appropriate timestamps. While IndexSeries can also reference frames from an ImageSeries through the 'indexed_timeseries' link, this usage is discouraged and will be deprecated in favor of using Images containers with 'order_of_images'.","datasets":[{"name":"data","dtype":"uint32","dims":["num_times"],"shape":[null],"doc":"Index of the image (using zero-indexing) in the linked Images object.","attributes":[{"name":"conversion","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"resolution","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"offset","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"unit","dtype":"text","value":"N/A","doc":"This field is unused by IndexSeries and has the value N/A."}]}],"links":[{"name":"indexed_timeseries","target_type":"ImageSeries","doc":"Link to ImageSeries object containing images that are indexed. Use of this link is discouraged and will be deprecated. Link to an Images type instead.","quantity":"?"},{"name":"indexed_images","target_type":"Images","doc":"Link to Images object containing an ordered set of images that are indexed. The Images object must contain a 'ordered_images' dataset specifying the order of the images in the Images type.","quantity":"?"}]}]})delimiter"; +{"datasets":[{"neurodata_type_def":"GrayscaleImage","neurodata_type_inc":"Image","dims":["x","y"],"shape":[null,null],"doc":"A grayscale image.","dtype":"numeric"},{"neurodata_type_def":"RGBImage","neurodata_type_inc":"Image","dims":["x","y","r, g, b"],"shape":[null,null,3],"doc":"A color image.","dtype":"numeric"},{"neurodata_type_def":"RGBAImage","neurodata_type_inc":"Image","dims":["x","y","r, g, b, a"],"shape":[null,null,4],"doc":"A color image with transparency.","dtype":"numeric"}],"groups":[{"neurodata_type_def":"ImageSeries","neurodata_type_inc":"TimeSeries","doc":"General image data that is common between acquisition and stimulus time series. Sometimes the image data is stored in the file in a raw format while other times it will be stored as a series of external image files in the host file system. The data field will either be binary data, if the data is stored in the NWB file, or empty, if the data is stored in an external image stack. [frame][x][y] or [frame][x][y][z].","datasets":[{"name":"data","dtype":"numeric","dims":[["frame","x","y"],["frame","x","y","z"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"Binary data representing images across frames. If data are stored in an external file, this should be an empty 3-D array."},{"name":"dimension","dtype":"int32","dims":["rank"],"shape":[null],"doc":"Number of pixels on x, y, (and z) axes.","quantity":"?"},{"name":"external_file","dtype":"text","dims":["num_files"],"shape":[null],"doc":"Paths to one or more external file(s). The field is only present if format='external'. This is only relevant if the image series is stored in the file system as one or more image file(s). This field should NOT be used if the image is stored in another NWB file and that file is linked to this file.","quantity":"?","attributes":[{"name":"starting_frame","dtype":"int32","dims":["num_files"],"shape":[null],"doc":"Each external image may contain one or more consecutive frames of the full ImageSeries. This attribute serves as an index to indicate which frames each file contains, to facilitate random access. The 'starting_frame' attribute, hence, contains a list of frame numbers within the full ImageSeries of the first frame of each file listed in the parent 'external_file' dataset. Zero-based indexing is used (hence, the first element will always be zero). For example, if the 'external_file' dataset has three paths to files and the first file has 5 frames, the second file has 10 frames, and the third file has 20 frames, then this attribute will have values [0, 5, 15]. If there is a single external file that holds all of the frames of the ImageSeries (and so there is a single element in the 'external_file' dataset), then this attribute should have value [0]."}]},{"name":"num_samples","dtype":"uint32","doc":"Total number of frames across all external files. This is required when format='external' and timing is described using starting_time and rate, since data is empty and its first dimension cannot be used to determine the number of frames. When timestamps is provided, len(timestamps) already serves this purpose.","quantity":"?"},{"name":"format","dtype":"text","default_value":"raw","doc":"Format of image. If this is 'external', then the attribute 'external_file' contains the path information to the image files. If this is 'raw', then the raw (single-channel) binary data is stored in the 'data' dataset. If this attribute is not present, then the default format='raw' case is assumed.","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the Device object that was used to capture these images.","quantity":"?"}]},{"neurodata_type_def":"ImageMaskSeries","neurodata_type_inc":"ImageSeries","doc":"DEPRECATED. An alpha mask that is applied to a presented visual stimulus. The 'data' array contains an array of mask values that are applied to the displayed image. Mask values are stored as RGBA. Mask can vary with time. The timestamps array indicates the starting time of a mask, and that mask pattern continues until it is explicitly changed.","links":[{"name":"masked_imageseries","target_type":"ImageSeries","doc":"Link to ImageSeries object that this image mask is applied to."}]},{"neurodata_type_def":"OpticalSeries","neurodata_type_inc":"ImageSeries","doc":"Image data that is presented or recorded. A stimulus template movie will be stored only as an image. When the image is presented as stimulus, additional data is required, such as field of view (e.g., how much of the visual field the image covers, or how large the imaged area of the target is). If the OpticalSeries represents acquired imaging data, orientation is also important.","datasets":[{"name":"distance","dtype":"float32","doc":"Distance from camera/monitor to target/eye.","quantity":"?"},{"name":"field_of_view","dtype":"float32","dims":[["width, height"],["width, height, depth"]],"shape":[[2],[3]],"doc":"Width, height and depth of image, or imaged area, in meters.","quantity":"?"},{"name":"data","dtype":"numeric","dims":[["frame","x","y"],["frame","x","y","r, g, b"]],"shape":[[null,null,null],[null,null,null,3]],"doc":"Images presented to subject, either grayscale or RGB"},{"name":"orientation","dtype":"text","doc":"Description of image relative to some reference frame (e.g., which way is up). Must also specify frame of reference.","quantity":"?"}]},{"neurodata_type_def":"IndexSeries","neurodata_type_inc":"TimeSeries","doc":"Stores indices that reference images defined in other containers. The primary purpose of the IndexSeries is to allow images stored in an Images container to be referenced in a specific sequence through the 'indexed_images' link. This approach avoids duplicating image data when the same image needs to be presented multiple times or when images need to be shown in a different order than they are stored. Since images in an Images container do not have an inherent order, the Images container needs to include an 'order_of_images' dataset (of type ImageReferences) whenever it is referenced by an IndexSeries. This dataset establishes the ordered sequence that the indices in IndexSeries refer to. The 'data' field stores the index into this ordered sequence, and the 'timestamps' array indicates the precise presentation time of each indexed image during an experiment. This can be used for displaying individual images or creating movie segments by referencing a sequence of images with the appropriate timestamps. While IndexSeries can also reference frames from an ImageSeries through the 'indexed_timeseries' link, this usage is discouraged and will be deprecated in favor of using Images containers with 'order_of_images'.","datasets":[{"name":"data","dtype":"uint32","dims":["num_times"],"shape":[null],"doc":"Index of the image (using zero-indexing) in the linked Images object.","attributes":[{"name":"conversion","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"resolution","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"offset","dtype":"float32","doc":"This field is unused by IndexSeries.","required":false},{"name":"unit","dtype":"text","value":"N/A","doc":"This field is unused by IndexSeries and has the value N/A."}]}],"links":[{"name":"indexed_timeseries","target_type":"ImageSeries","doc":"Link to ImageSeries object containing images that are indexed. Use of this link is discouraged and will be deprecated. Link to an Images type instead.","quantity":"?"},{"name":"indexed_images","target_type":"Images","doc":"Link to Images object containing an ordered set of images that are indexed. The Images object must contain an 'order_of_images' dataset specifying the order of the images in the Images type.","quantity":"?"}]}]})delimiter"; -constexpr std::string_view nwb_file_part0 = R"delimiter({"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.9.0","doc":"File version string. Use semantic versioning, e.g. 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (eg, video stimulus, sound, voltage, etc) and secondary representations of that data (eg, measurements of something used as a stimulus). This group should be made read-only after experiment complete and timestamps are corrected to common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (eg, sound, video, voltage, etc). Many different experiments can use the same stimuli, and stimuli can be reused during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"NWBDataInterface","doc":"Generic NWB data interfaces, usually from an extension, containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"DynamicTable objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc, are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"was_generated_by","dtype":"text","doc":"Name and version of software package(s) used to generate data contained in this NWB File. For each software package or library, include the name of the software as the first value and the version as the second value.","dims":["num_sources","name, version"],"shape":[null,2],"quantity":"?"},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"neurodata_type_inc":"LabMetaData","doc":"Place-holder than can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"},{"name":"models","doc":"Collection of data acquisition device models.","quantity":"?","groups":[{"neurodata_type_inc":"DeviceModel","doc":"Data acquisition device models.","quantity":"*"}]}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"ElectrodesTable","doc":"A table of all electrodes (i.e. channels) used for recording. Changed in NWB 2.9.0 to use the newly added ElectrodesTable neurodata type instead of a DynamicTable with added columns.","quantity":"?"}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as as part of an experiment. In this case both, the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where the a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental experimental_conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimuluation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether that be logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experiment, or epochs (see epochs subgroup) deriving from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Divisions in time mark)delimiter"; -constexpr std::string_view nwb_file_part1 = R"delimiter(ing experimental stages or sub-divisions of a single recording session.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Repeated experimental events that have a logical grouping.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Gender of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]})delimiter"; +constexpr std::string_view nwb_file_part0 = R"delimiter({"groups":[{"neurodata_type_def":"NWBFile","neurodata_type_inc":"NWBContainer","name":"root","doc":"An NWB file storing cellular-based neurophysiology data from a single experimental session.","attributes":[{"name":"nwb_version","dtype":"text","value":"2.11.0","doc":"File version string. Use semantic versioning, e.g., 1.2.1. This will be the name of the format with trailing major, minor and patch numbers."}],"datasets":[{"name":"file_create_date","dtype":"isodatetime","dims":["num_modifications"],"shape":[null],"doc":"A record of the date the file was created and of subsequent modifications. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted strings: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. The file can be created after the experiment was run, so this may differ from the experiment start time. Each modification to the nwb file adds a new entry to the array."},{"name":"identifier","dtype":"text","doc":"A unique text identifier for the file. For example, concatenated lab name, file creation date/time and experimentalist, or a hash of these and/or other values. The goal is that the string should be unique to all other files."},{"name":"session_description","dtype":"text","doc":"A description of the experimental session and data in the file."},{"name":"session_start_time","dtype":"isodatetime","doc":"Date and time of the experiment/session start. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds."},{"name":"timestamps_reference_time","dtype":"isodatetime","doc":"Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero)."}],"groups":[{"name":"acquisition","doc":"Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.","groups":[{"neurodata_type_inc":"NWBDataInterface","doc":"Acquired, raw data.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to acquisition","quantity":"*"}]},{"name":"analysis","doc":"Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Custom analysis results.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Tabular data that is relevant to data stored in analysis","quantity":"*"}]},{"name":"scratch","doc":"A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.","quantity":"?","groups":[{"neurodata_type_inc":"NWBContainer","doc":"Any one-off containers","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"Any one-off tables","quantity":"*"}],"datasets":[{"neurodata_type_inc":"ScratchData","doc":"Any one-off datasets","quantity":"*"}]},{"name":"processing","doc":"The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.","groups":[{"neurodata_type_inc":"ProcessingModule","doc":"Intermediate analysis of acquired data.","quantity":"*"}]},{"name":"stimulus","doc":"Data pushed into the system (e.g., video stimulus, sound, voltage, etc.) and secondary representations of that data (e.g., measurements of something used as a stimulus). This group should be made read-only after the experiment is complete and timestamps are corrected to a common timebase. Stores both presented stimuli and stimulus templates, the latter in case the same stimulus is presented multiple times, or is pulled from an external stimulus library. Stimuli are here defined as any signal that is pushed into the system as part of the experiment (e.g., sound, video, voltage, etc.). Many different experiments can use the same stimuli, and stimuli can be reused during an experiment. The stimulus group is organized so that one version of template stimuli can be stored and these can be used multiple times. These templates can exist in the present file or can be linked to a remote library file.","groups":[{"name":"presentation","doc":"Stimuli presented during the experiment.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"NWBDataInterface","doc":"Generic NWB data interfaces, usually from an extension, containing data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"DynamicTable","doc":"DynamicTable objects containing data of presented stimuli.","quantity":"*"}]},{"name":"templates","doc":"Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment's time reference frame.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries objects containing template data of presented stimuli.","quantity":"*"},{"neurodata_type_inc":"Images","doc":"Images objects containing images of presented stimuli.","quantity":"*"}]}]},{"name":"general","doc":"Experimental metadata, including protocol, notes and description of hardware device(s). The metadata stored in this section should be used to describe the experiment. Metadata necessary for interpreting the data is stored with the data. General experimental metadata, including animal strain, experimental protocols, experimenter, devices, etc. are stored under 'general'. Core metadata (e.g., that required to interpret data fields) is stored with the data itself, and implicitly defined by the file specification (e.g., time is in seconds). The strategy used here for storing non-core metadata is to use free-form text fields, such as would appear in sentences or paragraphs from a Methods section. Metadata fields are text to enable them to be more general, for example to represent ranges instead of numerical values. Machine-readable metadata is stored as attributes to these free-form datasets. All entries in the below table are to be included when data is present. Unused groups (e.g., intracellular_ephys in an optophysiology experiment) should not be created unless there is data to store within them.","datasets":[{"name":"data_collection","dtype":"text","doc":"Notes about data collection and analysis.","quantity":"?"},{"name":"experiment_description","dtype":"text","doc":"General description of the experiment.","quantity":"?"},{"name":"experimenter","dtype":"text","doc":"Name of person(s) who performed the experiment. Can also specify roles of different people involved.","quantity":"?","dims":["num_experimenters"],"shape":[null]},{"name":"institution","dtype":"text","doc":"Institution(s) where experiment was performed.","quantity":"?"},{"name":"keywords","dtype":"text","dims":["num_keywords"],"shape":[null],"doc":"Terms to search over.","quantity":"?"},{"name":"lab","dtype":"text","doc":"Laboratory where experiment was performed.","quantity":"?"},{"name":"notes","dtype":"text","doc":"Notes about the experiment.","quantity":"?"},{"name":"pharmacology","dtype":"text","doc":"Description of drugs used, including how and when they were administered. Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.","quantity":"?"},{"name":"protocol","dtype":"text","doc":"Experimental protocol, if applicable. e.g., include IACUC protocol number.","quantity":"?"},{"name":"related_publications","dtype":"text","doc":"Publication information. PMID, DOI, URL, etc.","dims":["num_publications"],"shape":[null],"quantity":"?"},{"name":"session_id","dtype":"text","doc":"Lab-specific ID for the session.","quantity":"?"},{"name":"slices","dtype":"text","doc":"Description of slices, including information about preparation thickness, orientation, temperature, and bath solution.","quantity":"?"},{"name":"source_script","dtype":"text","doc":"Script file or link to public source code used to create this NWB file.","quantity":"?","attributes":[{"name":"file_name","dtype":"text","doc":"Name of script file."}]},{"name":"was_generated_by","dtype":"text","doc":"Name and version of software package(s) used to generate data contained in this NWB file. For each software package or library, include the name of the software as the first value and the version as the second value.","dims":["num_sources","name, version"],"shape":[null,2],"quantity":"?"},{"name":"stimulus","dtype":"text","doc":"Notes about stimuli, such as how and where they were presented.","quantity":"?"},{"name":"surgery","dtype":"text","doc":"Narrative description about surgery/surgeries, including date(s) and who performed surgery.","quantity":"?"},{"name":"virus","dtype":"text","doc":"Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.","quantity":"?"}],"groups":[{"name":"external_resources","neurodata_type_inc":"HERD","doc":"This is the HERD structure for this specific NWBFile, storing the mapped external resources.","quantity":"?"},{"neurodata_type_inc":"LabMetaData","doc":"Place-holder that can be extended so that lab-specific meta-data can be placed in /general.","quantity":"*"},{"name":"devices","doc":"Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.","quantity":"?","groups":[{"neurodata_type_inc":"Device","doc":"Data acquisition devices.","quantity":"*"},{"name":"models","doc":"Collection of data acquisition device models.","quantity":"?","groups":[{"neurodata_type_inc":"DeviceModel","doc":"Data acquisition device models.","quantity":"*"}]}]},{"name":"subject","neurodata_type_inc":"Subject","doc":"Information about the animal or person from which the data was measured.","quantity":"?"},{"name":"extracellular_ephys","doc":"Metadata related to extracellular electrophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ElectrodeGroup","doc":"Physical group of electrodes.","quantity":"*"},{"name":"electrodes","neurodata_type_inc":"ElectrodesTable","doc":"A table of all electrodes (i.e., channels) used for recording. Changed in NWB 2.9.0 to use the newly added ElectrodesTable neurodata type instead of a DynamicTable with added columns.","quantity":"?"}]},{"name":"intracellular_ephys","doc":"Metadata related to intracellular electrophysiology.","quantity":"?","datasets":[{"name":"filtering","dtype":"text","doc":"[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.","quantity":"?"}],"groups":[{"neurodata_type_inc":"IntracellularElectrode","doc":"An intracellular electrode.","quantity":"*"},{"name":"sweep_table","neurodata_type_inc":"SweepTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable and ExperimentalConditions tables provide enhanced support for experiment metadata.","quantity":"?"},{"name":"intracellular_recordings","neurodata_type_inc":"IntracellularRecordingsTable","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response are recorded as part of an experiment. In this case, both the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","quantity":"?"},{"name":"simultaneous_recordings","neurodata_type_inc":"SimultaneousRecordingsTable","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes","quantity":"?"},{"name":"sequential_recordings","neurodata_type_inc":"SequentialRecordingsTable","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","quantity":"?"},{"name":"repetitions","neurodata_type_inc":"RepetitionsTable","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","quantity":"?"},{"name":"experimental_conditions","neurodata_type_inc":"ExperimentalConditionsTable","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental conditions.","quantity":"?"}]},{"name":"optogenetics","doc":"Metadata describing optogenetic stimulation.","quantity":"?","groups":[{"neurodata_type_inc":"OptogeneticStimulusSite","doc":"An optogenetic stimulation site.","quantity":"*"}]},{"name":"optophysiology","doc":"Metadata related to optophysiology.","quantity":"?","groups":[{"neurodata_type_inc":"ImagingPlane","doc":"An imaging plane.","quantity":"*"}]}]},{"name":"intervals","doc":"Experimental intervals, whether these are logically distinct sub-experiments having a particular scientific goal, trials (see trials subgroup) during an experi)delimiter"; +constexpr std::string_view nwb_file_part1 = R"delimiter(ment, or epochs (see epochs subgroup) derived from analysis of data.","quantity":"?","groups":[{"name":"epochs","neurodata_type_inc":"TimeIntervals","doc":"Time intervals marking coarse-grained experimental phases or subdivisions of a recording session, such as baseline, task, rest, or sleep stages.","quantity":"?"},{"name":"trials","neurodata_type_inc":"TimeIntervals","doc":"Time intervals corresponding to repeated experimental units with consistent structure, such as individual stimulus-response-reward cycles.","quantity":"?"},{"name":"invalid_times","neurodata_type_inc":"TimeIntervals","doc":"Time intervals that should be removed from analysis.","quantity":"?"},{"neurodata_type_inc":"TimeIntervals","doc":"Optional additional table(s) for describing other experimental time intervals.","quantity":"*"}]},{"name":"events","doc":"Experimental event tables for the session, stored as `EventsTable` instances. This is the recommended location for all event tables in the file, regardless of whether the events were emitted directly by the acquisition system (TTL edges, hardware event channels), derived by user processing (filtering, detection algorithms), or added as post-hoc annotations. The provenance of each table can be recorded in the `source_description` attribute of the `EventsTable` and described in more detail in its `description`. Each `EventsTable` should hold events of a single type (i.e., events sharing the same per-event metadata columns). For periods with required start and stop times, see `/intervals/`. For continuous acquired signals or stimulus templates from which events may have been derived, see `/acquisition/` or `/stimulus/`.","quantity":"?","groups":[{"neurodata_type_inc":"EventsTable","doc":"An EventsTable holding a collection of event times of a particular type (e.g., licks, rewards, stimulus presentations) along with their per-event metadata.","quantity":"*"}]},{"name":"units","neurodata_type_inc":"Units","doc":"Data about sorted spike units.","quantity":"?"}]},{"neurodata_type_def":"LabMetaData","neurodata_type_inc":"NWBContainer","doc":"Lab-specific meta-data."},{"neurodata_type_def":"Subject","neurodata_type_inc":"NWBContainer","doc":"Information about the animal or person from which the data was measured.","datasets":[{"name":"age","dtype":"text","doc":"Age of subject. Can be supplied instead of 'date_of_birth'. The ISO 8601 Duration format is recommended, e.g., 'P90D' for 90 days old. If the precise age is unknown, an age range can be given by '[lower bound]/[upper bound]' e.g. 'P10D/P20D' would mean that the age is in between 10 and 20 days. If only the lower bound is known, then including only the slash after that lower bound can be used to indicate a missing bound. For instance, 'P90Y/' would indicate that the age is 90 years or older.","quantity":"?","attributes":[{"name":"reference","doc":"Age is with reference to this event. Can be 'birth' or 'gestational'. If reference is omitted, 'birth' is implied.","dtype":"text","required":false,"default_value":"birth"}]},{"name":"date_of_birth","dtype":"isodatetime","doc":"Date of birth of subject. Can be supplied instead of 'age'.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of subject and where subject came from (e.g., breeder, if animal).","quantity":"?"},{"name":"genotype","dtype":"text","doc":"Genetic strain. If absent, assume Wild Type (WT).","quantity":"?"},{"name":"sex","dtype":"text","doc":"Biological sex of subject.","quantity":"?"},{"name":"species","dtype":"text","doc":"Species of subject.","quantity":"?"},{"name":"strain","dtype":"text","doc":"Strain of subject.","quantity":"?"},{"name":"subject_id","dtype":"text","doc":"ID of animal/person used/participating in experiment (lab convention).","quantity":"?"},{"name":"weight","dtype":"text","doc":"Weight at time of experiment, at time of surgery and at other important times.","quantity":"?"}]}],"datasets":[{"neurodata_type_def":"ScratchData","neurodata_type_inc":"NWBData","doc":"Any one-off datasets","attributes":[{"name":"notes","doc":"Any notes the user has about the dataset being stored","dtype":"text"}]}]})delimiter"; constexpr std::array nwb_file_parts = {nwb_file_part0, nwb_file_part1}; constexpr std::size_t nwb_file_total_length = []() { std::size_t length = 0; @@ -59,34 +62,35 @@ constexpr auto nwb_file_concatenate = []() { constexpr std::string_view nwb_file(nwb_file_concatenate.data(), nwb_file_concatenate.size() - 1); constexpr std::string_view nwb_misc = R"delimiter( -{"groups":[{"neurodata_type_def":"AbstractFeatureSeries","neurodata_type_inc":"TimeSeries","doc":"Abstract features, such as quantitative descriptions of sensory stimuli. The TimeSeries::data field is a 2D array, storing those features (e.g., for visual grating stimulus this might be orientation, spatial frequency and contrast). Null stimuli (eg, uniform gray) can be marked as being an independent feature (eg, 1.0 for gray, 0.0 for actual stimulus) or by storing NaNs for feature values, or through use of the TimeSeries::control fields. A set of features is considered to persist until the next set of features is defined. The final set of features stored should be the null set. This is useful when storing the raw stimulus is impractical.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_features"]],"shape":[[null],[null,null]],"doc":"Values of each feature at each time.","attributes":[{"name":"unit","dtype":"text","default_value":"see 'feature_units'","doc":"Since there can be different units for different features, store the units in 'feature_units'. The default value for this attribute is \"see 'feature_units'\".","required":false}]},{"name":"feature_units","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Units of each feature.","quantity":"?"},{"name":"features","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Description of the features represented in TimeSeries::data."}]},{"neurodata_type_def":"AnnotationSeries","neurodata_type_inc":"TimeSeries","doc":"Stores user annotations made during an experiment. The data[] field stores a text array, and timestamps are stored for each annotation (ie, interval=1). This is largely an alias to a standard TimeSeries storing a text array but that is identifiable as storing annotations in a machine-readable way.","datasets":[{"name":"data","dtype":"text","dims":["num_times"],"shape":[null],"doc":"Annotations made during an experiment.","attributes":[{"name":"resolution","dtype":"float32","value":-1.0,"doc":"Smallest meaningful difference between values in data. Annotations have no units, so the value is fixed to -1.0."},{"name":"unit","dtype":"text","value":"n/a","doc":"Base unit of measurement for working with the data. Annotations have no units, so the value is fixed to 'n/a'."}]}]},{"neurodata_type_def":"IntervalSeries","neurodata_type_inc":"TimeSeries","doc":"Stores intervals of data. The timestamps field stores the beginning and end of intervals. The data field stores whether the interval just started (>0 value) or ended (<0 value). Different interval types can be represented in the same series by using multiple key values (eg, 1 for feature A, 2 for feature B, 3 for feature C, etc). The field data stores an 8-bit integer. This is largely an alias of a standard TimeSeries but that is identifiable as representing time intervals in a machine-readable way.","datasets":[{"name":"data","dtype":"int8","dims":["num_times"],"shape":[null],"doc":"Use values >0 if interval started, <0 if interval ended.","attributes":[{"name":"resolution","dtype":"float32","value":-1.0,"doc":"Smallest meaningful difference between values in data. Annotations have no units, so the value is fixed to -1.0."},{"name":"unit","dtype":"text","value":"n/a","doc":"Base unit of measurement for working with the data. Annotations have no units, so the value is fixed to 'n/a'."}]}]},{"neurodata_type_def":"FrequencyBandsTable","neurodata_type_inc":"DynamicTable","doc":"Table for describing the bands that DecompositionSeries was generated from. There should be one row in this table for each band.","datasets":[{"name":"band_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the band, e.g. theta."},{"name":"band_limits","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands","low, high"],"shape":[null,2],"doc":"Low and high limit of each band in Hz. If it is a Gaussian filter, use 2 SD on either side of the center."},{"name":"band_mean","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands"],"shape":[null],"doc":"The mean Gaussian filters, in Hz.","quantity":"?"},{"name":"band_stdev","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands"],"shape":[null],"doc":"The standard deviation of Gaussian filters, in Hz.","quantity":"?"}]},{"neurodata_type_def":"DecompositionSeries","neurodata_type_inc":"TimeSeries","doc":"Spectral analysis of a time series, e.g. of an LFP or a speech signal.","datasets":[{"name":"data","dtype":"numeric","dims":["num_times","num_channels","num_bands"],"shape":[null,null,null],"doc":"Data decomposed into frequency bands.","attributes":[{"name":"unit","dtype":"text","default_value":"no unit","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion'."}]},{"name":"metric","dtype":"text","doc":"The metric used, e.g. phase, amplitude, power."},{"name":"source_channels","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the channels that this decomposition series was generated from.","quantity":"?"}],"groups":[{"name":"bands","neurodata_type_inc":"FrequencyBandsTable","doc":"Table for describing the bands that this series was generated from.","quantity":"?"}],"links":[{"name":"source_timeseries","target_type":"TimeSeries","doc":"Link to TimeSeries object that this data was calculated from. Metadata about electrodes and their position can be read from that ElectricalSeries so it is not necessary to store that information here.","quantity":"?"}]},{"neurodata_type_def":"Units","neurodata_type_inc":"DynamicTable","default_name":"Units","doc":"Data about spiking units. Event times of observed units (e.g. cell, synapse, etc.) should be concatenated and stored in spike_times.","datasets":[{"name":"spike_times_index","neurodata_type_inc":"VectorIndex","doc":"Index into the spike_times dataset.","quantity":"?"},{"name":"spike_times","neurodata_type_inc":"VectorData","dtype":"float64","doc":"Spike times for each unit in seconds.","quantity":"?","attributes":[{"name":"resolution","dtype":"float64","doc":"The smallest possible difference between two spike times. Usually 1 divided by the acquisition sampling rate from which spike times were extracted, but could be larger if the acquisition time series was downsampled or smaller if the acquisition time series was smoothed/interpolated and it is possible for the spike time to be between samples.","required":false}]},{"name":"obs_intervals_index","neurodata_type_inc":"VectorIndex","doc":"Index into the obs_intervals dataset.","quantity":"?"},{"name":"obs_intervals","neurodata_type_inc":"VectorData","dtype":"float64","dims":["num_intervals","start|end"],"shape":[null,2],"doc":"Observation intervals for each unit.","quantity":"?"},{"name":"electrodes_index","neurodata_type_inc":"VectorIndex","doc":"Index into electrodes.","quantity":"?"},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"Electrode that each spike unit came from, specified using a DynamicTableRegion.","quantity":"?"},{"name":"electrode_group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Electrode group that each spike unit came from.","quantity":"?"},{"name":"waveform_mean","neurodata_type_inc":"VectorData","dtype":"float32","dims":[["num_units","num_samples"],["num_units","num_samples","num_electrodes"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveform mean for each spike unit.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"unit","dtype":"text","value":"volts","doc":"Unit of measurement. This value is fixed to 'volts'.","required":false}]},{"name":"waveform_sd","neurodata_type_inc":"VectorData","dtype":"float32","dims":[["num_units","num_samples"],["num_units","num_samples","num_electrodes"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveform standard deviation for each spike unit.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"unit","dtype":"text","value":"volts","doc":"Unit of measurement. This value is fixed to 'volts'.","required":false}]},{"name":"waveforms","neurodata_type_inc":"VectorData","dtype":"numeric","dims":["num_waveforms","num_samples"],"shape":[null,null],"doc":"Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be the same as the order of the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"unit","dtype":"text","value":"volts","doc":"Unit of measurement. This value is fixed to 'volts'.","required":false}]},{"name":"waveforms_index","neurodata_type_inc":"VectorIndex","doc":"Index into the 'waveforms' dataset. One value for every spike event. See 'waveforms' for more detail.","quantity":"?"},{"name":"waveforms_index_index","neurodata_type_inc":"VectorIndex","doc":"Index into the 'waveforms_index' dataset. One value for every unit (row in the table). See 'waveforms' for more detail.","quantity":"?"}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"AbstractFeatureSeries","neurodata_type_inc":"TimeSeries","doc":"Abstract features, such as quantitative descriptions of sensory stimuli. The TimeSeries::data field is a 2-D array, storing those features (e.g., for a visual grating stimulus this might be orientation, spatial frequency and contrast). Null stimuli (e.g., uniform gray) can be marked as being an independent feature (e.g., 1.0 for gray, 0.0 for actual stimulus) or by storing NaNs for feature values, or through use of the TimeSeries::control fields. A set of features is considered to persist until the next set of features is defined. The final set of features stored should be the null set. This is useful when storing the raw stimulus is impractical.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_features"]],"shape":[[null],[null,null]],"doc":"Values of each feature at each time.","attributes":[{"name":"unit","dtype":"text","default_value":"see 'feature_units'","doc":"Since there can be different units for different features, store the units in 'feature_units'. The default value for this attribute is \"see 'feature_units'\".","required":false}]},{"name":"feature_units","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Units of each feature.","quantity":"?"},{"name":"features","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Description of the features represented in TimeSeries::data."}]},{"neurodata_type_def":"AnnotationSeries","neurodata_type_inc":"TimeSeries","doc":"DEPRECATED. Use an `EventsTable` instead, placed in the top-level `/events` group of the `NWBFile`. The `timestamps` field maps to the `timestamp` column, and the `data` field (annotation strings) maps to the `annotation` column on `EventsTable`. Use the `source_description` attribute on the `EventsTable` to record where the annotations came from (e.g., \"Manual video review\"). Original definition: Stores user annotations made during an experiment. The data[] field stores a text array, and timestamps are stored for each annotation (i.e., interval=1). This is largely an alias to a standard TimeSeries storing a text array but that is identifiable as storing annotations in a machine-readable way.","datasets":[{"name":"data","dtype":"text","dims":["num_times"],"shape":[null],"doc":"Annotations made during an experiment.","attributes":[{"name":"resolution","dtype":"float32","value":-1.0,"doc":"Smallest meaningful difference between values in data. Annotations have no units, so the value is fixed to -1.0."},{"name":"unit","dtype":"text","value":"n/a","doc":"Base unit of measurement for working with the data. Annotations have no units, so the value is fixed to 'n/a'."}]}]},{"neurodata_type_def":"IntervalSeries","neurodata_type_inc":"TimeSeries","doc":"Stores intervals of data. The timestamps field stores the beginning and end of intervals. The data field stores whether the interval just started (>0 value) or ended (<0 value). Different interval types can be represented in the same series by using multiple key values (e.g., 1 for feature A, 2 for feature B, 3 for feature C, etc.). The field data stores an 8-bit integer. This is largely an alias of a standard TimeSeries but that is identifiable as representing time intervals in a machine-readable way.","datasets":[{"name":"data","dtype":"int8","dims":["num_times"],"shape":[null],"doc":"Use values >0 if interval started, <0 if interval ended.","attributes":[{"name":"resolution","dtype":"float32","value":-1.0,"doc":"Smallest meaningful difference between values in data. IntervalSeries stores integer markers, so the value is fixed to -1.0."},{"name":"unit","dtype":"text","value":"n/a","doc":"Base unit of measurement for working with the data. IntervalSeries stores integer markers with no physical unit, so the value is fixed to 'n/a'."}]}]},{"neurodata_type_def":"FrequencyBandsTable","neurodata_type_inc":"DynamicTable","doc":"Table for describing the bands that DecompositionSeries was generated from. There should be one row in this table for each band.","datasets":[{"name":"band_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the band, e.g., theta."},{"name":"band_limits","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands","low, high"],"shape":[null,2],"doc":"Low and high limit of each band in Hz. If it is a Gaussian filter, use 2 SD on either side of the center."},{"name":"band_mean","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands"],"shape":[null],"doc":"The center frequency (mean) of each Gaussian filter, in Hz.","quantity":"?"},{"name":"band_stdev","neurodata_type_inc":"VectorData","dtype":"float32","dims":["num_bands"],"shape":[null],"doc":"The standard deviation (bandwidth) of each Gaussian filter, in Hz.","quantity":"?"}]},{"neurodata_type_def":"DecompositionSeries","neurodata_type_inc":"TimeSeries","doc":"Spectral analysis of a time series, e.g., of an LFP or a speech signal.","datasets":[{"name":"data","dtype":"numeric","dims":["num_times","num_channels","num_bands"],"shape":[null,null,null],"doc":"Data decomposed into frequency bands.","attributes":[{"name":"unit","dtype":"text","default_value":"no unit","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion'."}]},{"name":"metric","dtype":"text","doc":"The metric used, e.g., phase, amplitude, power."},{"name":"source_channels","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the channels that this decomposition series was generated from.","quantity":"?"}],"groups":[{"name":"bands","neurodata_type_inc":"FrequencyBandsTable","doc":"Table for describing the bands that this series was generated from.","quantity":"?"}],"links":[{"name":"source_timeseries","target_type":"TimeSeries","doc":"Link to TimeSeries object that this data was calculated from. Metadata about electrodes and their position can be read from that ElectricalSeries so it is not necessary to store that information here.","quantity":"?"}]},{"neurodata_type_def":"Units","neurodata_type_inc":"DynamicTable","default_name":"Units","doc":"Data about spiking units. Event times of observed units (e.g., cell, synapse, etc.) should be concatenated and stored in spike_times.","datasets":[{"name":"spike_times_index","neurodata_type_inc":"VectorIndex","doc":"Index into the spike_times dataset.","quantity":"?"},{"name":"spike_times","neurodata_type_inc":"VectorData","dtype":"float64","doc":"Spike times for each unit, in seconds, relative to the session reference time (i.e., session_start_time or, if defined, timestamps_reference_time). Times should be stored in ascending order.","quantity":"?","attributes":[{"name":"resolution","dtype":"float64","doc":"The temporal resolution of the spike times, in seconds. This is typically the sampling period (1/sampling_rate) of the acquisition data from which spikes were extracted. If the acquisition data was downsampled before spike sorting, the resolution should be based on the downsampled sampling rate (lower sampling rate, larger resolution value). If interpolation was applied during spike sorting, then spike times may fall between samples of the acquisition data, so the resolution should be the interpolation step size (higher sampling rate, smaller resolution value).","required":false}]},{"name":"obs_intervals_index","neurodata_type_inc":"VectorIndex","doc":"Index into the obs_intervals dataset.","quantity":"?"},{"name":"obs_intervals","neurodata_type_inc":"VectorData","dtype":"float64","dims":["num_intervals","start|end"],"shape":[null,2],"doc":"Observation intervals for each unit, in seconds, relative to the session reference time (i.e., session_start_time or, if defined, timestamps_reference_time). Intervals should be in ascending order and non-overlapping.","quantity":"?"},{"name":"electrodes_index","neurodata_type_inc":"VectorIndex","doc":"Index into electrodes.","quantity":"?"},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"Electrode that each spike unit came from, specified using a DynamicTableRegion.","quantity":"?"},{"name":"electrode_group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Electrode group that each spike unit came from.","quantity":"?"},{"name":"waveform_mean","neurodata_type_inc":"VectorData","dtype":"float32","dims":[["num_units","num_samples"],["num_units","num_samples","num_electrodes"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveform mean for each spike unit.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"time_before_peak_in_ms","dtype":"float32","doc":"Time, in milliseconds, from the start of the waveform to the spike peak. The peak is the alignment point used during spike sorting, typically the sample of maximum absolute amplitude. The same value applies to every unit in the table. Together with 'sampling_rate' and the number of samples, this fully specifies where each waveform sample falls relative to the spike event.","required":false},{"name":"unit","dtype":"text","default_value":"volts","doc":"Unit of measurement. The default value is 'volts'.","required":false}]},{"name":"waveform_sd","neurodata_type_inc":"VectorData","dtype":"float32","dims":[["num_units","num_samples"],["num_units","num_samples","num_electrodes"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveform standard deviation for each spike unit.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"time_before_peak_in_ms","dtype":"float32","doc":"Time, in milliseconds, from the start of the waveform to the spike peak. The peak is the alignment point used during spike sorting, typically the sample of maximum absolute amplitude. The same value applies to every unit in the table. Together with 'sampling_rate' and the number of samples, this fully specifies where each waveform sample falls relative to the spike event.","required":false},{"name":"unit","dtype":"text","default_value":"volts","doc":"Unit of measurement. The default value is 'volts'.","required":false}]},{"name":"waveforms","neurodata_type_inc":"VectorData","dtype":"numeric","dims":["num_waveforms","num_samples"],"shape":[null,null],"doc":"Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be the same as the order of the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.","quantity":"?","attributes":[{"name":"sampling_rate","dtype":"float32","doc":"Sampling rate, in hertz.","required":false},{"name":"time_before_peak_in_ms","dtype":"float32","doc":"Time, in milliseconds, from the start of the waveform to the spike peak. The peak is the alignment point used during spike sorting, typically the sample of maximum absolute amplitude. The same value applies to every waveform in the table. Together with 'sampling_rate' and the number of samples, this fully specifies where each waveform sample falls relative to the spike event.","required":false},{"name":"unit","dtype":"text","default_value":"volts","doc":"Unit of measurement. The default value is 'volts'.","required":false}]},{"name":"waveforms_index","neurodata_type_inc":"VectorIndex","doc":"Index into the 'waveforms' dataset. One value for every spike event. See 'waveforms' for more detail.","quantity":"?"},{"name":"waveforms_index_index","neurodata_type_inc":"VectorIndex","doc":"Index into the 'waveforms_index' dataset. One value for every unit (row in the table). See 'waveforms' for more detail.","quantity":"?"}]}]})delimiter"; constexpr std::string_view nwb_behavior = R"delimiter( -{"groups":[{"neurodata_type_def":"SpatialSeries","neurodata_type_inc":"TimeSeries","doc":"Direction, e.g., of gaze or travel, or position. The TimeSeries::data field is a 2D array storing position or direction relative to some reference frame. Array structure: [num measurements] [num dimensions]. Each SpatialSeries has a text dataset reference_frame that indicates the zero-position, or the zero-axes for direction. For example, if representing gaze direction, 'straight-ahead' might be a specific pixel on the monitor, or some other point in space. For position data, the 0,0 point might be the top-left corner of an enclosure, as viewed from the tracking camera. The unit of data will indicate how to interpret SpatialSeries values.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","x"],["num_times","x,y"],["num_times","x,y,z"]],"shape":[[null],[null,1],[null,2],[null,3]],"doc":"1-D or 2-D array storing position or direction relative to some reference frame.","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Base unit of measurement for working with the data. The default value is 'meters'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'.","required":false}]},{"name":"reference_frame","dtype":"text","doc":"Description defining what exactly 'straight-ahead' means.","quantity":"?"}]},{"neurodata_type_def":"BehavioralEpochs","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralEpochs","doc":"TimeSeries for storing behavioral epochs. The objective of this and the other two Behavioral interfaces (e.g. BehavioralEvents and BehavioralTimeSeries) is to provide generic hooks for software tools/scripts. This allows a tool/script to take the output one specific interface (e.g., UnitTimes) and plot that data relative to another data modality (e.g., behavioral events) without having to define all possible modalities in advance. Declaring one of these interfaces means that one or more TimeSeries of the specified type is published. These TimeSeries should reside in a group having the same name as the interface. For example, if a BehavioralTimeSeries interface is declared, the module will have one or more TimeSeries defined in the module sub-group 'BehavioralTimeSeries'. BehavioralEpochs should use IntervalSeries. BehavioralEvents is used for irregular events. BehavioralTimeSeries is for continuous data.","groups":[{"neurodata_type_inc":"IntervalSeries","doc":"IntervalSeries object containing start and stop times of epochs.","quantity":"+"}]},{"neurodata_type_def":"BehavioralEvents","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralEvents","doc":"TimeSeries for storing behavioral events. See description of BehavioralEpochs for more details.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing behavioral events.","quantity":"+"}]},{"neurodata_type_def":"BehavioralTimeSeries","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralTimeSeries","doc":"TimeSeries for storing behavioral time series data. See description of BehavioralEpochs for more details.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing continuous behavioral data.","quantity":"+"}]},{"neurodata_type_def":"PupilTracking","neurodata_type_inc":"NWBDataInterface","default_name":"PupilTracking","doc":"Eye-tracking data, representing pupil size.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing time series data on pupil size.","quantity":"+"}]},{"neurodata_type_def":"EyeTracking","neurodata_type_inc":"NWBDataInterface","default_name":"EyeTracking","doc":"Eye-tracking data, representing direction of gaze.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing data measuring direction of gaze.","quantity":"+"}]},{"neurodata_type_def":"CompassDirection","neurodata_type_inc":"NWBDataInterface","default_name":"CompassDirection","doc":"With a CompassDirection interface, a module publishes a SpatialSeries object representing a floating point value for theta. The SpatialSeries::reference_frame field should indicate what direction corresponds to 0 and which is the direction of rotation (this should be clockwise). The si_unit for the SpatialSeries should be radians or degrees.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing direction of gaze travel.","quantity":"+"}]},{"neurodata_type_def":"Position","neurodata_type_inc":"NWBDataInterface","default_name":"Position","doc":"Position data, whether along the x, x/y or x/y/z axis.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing position data.","quantity":"+"}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"SpatialSeries","neurodata_type_inc":"TimeSeries","doc":"Direction, e.g., of gaze or travel, or position. The TimeSeries::data field is a 2-D array storing position or direction relative to some reference frame. Array structure: [num measurements] [num dimensions]. Each SpatialSeries has a text dataset reference_frame that indicates the zero-position, or the zero-axes for direction. For example, if representing gaze direction, 'straight-ahead' might be a specific pixel on the monitor, or some other point in space. For position data, the 0,0 point might be the top-left corner of an enclosure, as viewed from the tracking camera. The unit of data will indicate how to interpret SpatialSeries values.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","x"],["num_times","x,y"],["num_times","x,y,z"]],"shape":[[null],[null,1],[null,2],[null,3]],"doc":"1-D or 2-D array storing position or direction relative to some reference frame.","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Base unit of measurement for working with the data. The default value is 'meters'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'.","required":false}]},{"name":"reference_frame","dtype":"text","doc":"Description defining what exactly 'straight-ahead' means.","quantity":"?"}]},{"neurodata_type_def":"BehavioralEpochs","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralEpochs","doc":"TimeSeries for storing behavioral epochs. The objective of this and the other two Behavioral interfaces (e.g., BehavioralEvents and BehavioralTimeSeries) is to provide generic hooks for software tools/scripts. This allows a tool/script to take the output of one specific interface (e.g., UnitTimes) and plot that data relative to another data modality (e.g., behavioral events) without having to define all possible modalities in advance. Declaring one of these interfaces means that one or more TimeSeries of the specified type are published. These TimeSeries should reside in a group having the same name as the interface. For example, if a BehavioralTimeSeries interface is declared, the module will have one or more TimeSeries defined in the module sub-group 'BehavioralTimeSeries'. BehavioralEpochs should use IntervalSeries. BehavioralEvents is used for irregular events. BehavioralTimeSeries is for continuous data.","groups":[{"neurodata_type_inc":"IntervalSeries","doc":"IntervalSeries object containing start and stop times of epochs.","quantity":"+"}]},{"neurodata_type_def":"BehavioralEvents","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralEvents","doc":"DEPRECATED. Use an `EventsTable` instead, placed in the top-level `/events` group of the `NWBFile`. Each `TimeSeries` formerly stored under `BehavioralEvents` becomes one `EventsTable`. The `timestamps` field maps to the `timestamp` column, and the `data` field maps to an additional column named after the event marker (e.g., `reward_magnitude`, `port_number`); for multi-dimensional `data`, use one column per field. Any other per-event metadata becomes additional columns. Use the `source_description` attribute on the `EventsTable` to record where the events came from (e.g., \"Acquisition system\", \"Thresholding of analog signal ANALOG1 at 3 V\", \"Manual video review\"). Original definition: TimeSeries for storing behavioral events. See description of BehavioralEpochs for more details.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing behavioral events.","quantity":"+"}]},{"neurodata_type_def":"BehavioralTimeSeries","neurodata_type_inc":"NWBDataInterface","default_name":"BehavioralTimeSeries","doc":"TimeSeries for storing behavioral time series data. See description of BehavioralEpochs for more details.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing continuous behavioral data.","quantity":"+"}]},{"neurodata_type_def":"PupilTracking","neurodata_type_inc":"NWBDataInterface","default_name":"PupilTracking","doc":"Eye-tracking data, representing pupil size.","groups":[{"neurodata_type_inc":"TimeSeries","doc":"TimeSeries object containing time series data on pupil size.","quantity":"+"}]},{"neurodata_type_def":"EyeTracking","neurodata_type_inc":"NWBDataInterface","default_name":"EyeTracking","doc":"Eye-tracking data, representing direction of gaze.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing data measuring direction of gaze.","quantity":"+"}]},{"neurodata_type_def":"CompassDirection","neurodata_type_inc":"NWBDataInterface","default_name":"CompassDirection","doc":"With a CompassDirection interface, a module publishes a SpatialSeries object representing a floating point value for theta. The SpatialSeries::reference_frame field should indicate what direction corresponds to 0 and which is the direction of rotation (this should be clockwise). The unit for the SpatialSeries should be radians or degrees.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing direction of gaze travel.","quantity":"+"}]},{"neurodata_type_def":"Position","neurodata_type_inc":"NWBDataInterface","default_name":"Position","doc":"Position data, whether along the x, x/y, or x/y/z axes.","groups":[{"neurodata_type_inc":"SpatialSeries","doc":"SpatialSeries object containing position data.","quantity":"+"}]}]})delimiter"; constexpr std::string_view nwb_ecephys = R"delimiter( -{"groups":[{"neurodata_type_def":"ElectricalSeries","neurodata_type_inc":"TimeSeries","doc":"A time series of acquired voltage data from extracellular recordings. The data field is an int or float array storing data in volts. The first dimension should always represent time. The second dimension, if present, should represent channels.","attributes":[{"name":"filtering","dtype":"text","doc":"Filtering applied to all channels of the data. For example, if this ElectricalSeries represents high-pass-filtered data (also known as AP Band), then this value could be \"High-pass 4-pole Bessel filter at 500 Hz\". If this ElectricalSeries represents low-pass-filtered LFP data and the type of filter is unknown, then this value could be \"Low-pass filter at 300 Hz\". If a non-standard filter type is used, provide as much detail about the filter properties as possible.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_channels"],["num_times","num_channels","num_samples"]],"shape":[[null],[null,null],[null,null,null]],"doc":"Recorded voltage data.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data. This value is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion', followed by 'channel_conversion' (if present), and then add 'offset'."}]},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the electrodes that this time series was generated from."},{"name":"channel_conversion","dtype":"float32","dims":["num_channels"],"shape":[null],"doc":"Channel-specific conversion factor. Multiply the data in the 'data' dataset by these values along the channel axis (as indicated by axis attribute) AND by the global conversion factor in the 'conversion' attribute of 'data' to get the data values in Volts, i.e, data in Volts = data * data.conversion * channel_conversion. This approach allows for both global and per-channel data conversion factors needed to support the storage of electrical recordings as native values generated by data acquisition systems. If this dataset is not present, then there is no channel-specific conversion factor, i.e. it is 1 for all channels.","quantity":"?","attributes":[{"name":"axis","dtype":"int32","value":1,"doc":"The zero-indexed axis of the 'data' dataset that the channel-specific conversion factor corresponds to. This value is fixed to 1."}]}]},{"neurodata_type_def":"SpikeEventSeries","neurodata_type_inc":"ElectricalSeries","doc":"Stores snapshots/snippets of recorded spike events (i.e., threshold crossings). This may also be raw data, as reported by ephys hardware. If so, the TimeSeries::description field should describe how events were detected. All events span the same recording channels and store snapshots of equal duration. TimeSeries::data array structure: [num events] [num channels] [num samples] (or [num events] [num samples] for single electrode).","datasets":[{"name":"data","dtype":"numeric","dims":[["num_events","num_samples"],["num_events","num_channels","num_samples"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveforms.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Unit of measurement for waveforms, which is fixed to 'volts'."}]},{"name":"timestamps","dtype":"float64","dims":["num_times"],"shape":[null],"doc":"Timestamps for samples stored in data, in seconds, relative to the common experiment master-clock stored in NWBFile.timestamps_reference_time. Timestamps are required for the events. Unlike for TimeSeries, timestamps are required for SpikeEventSeries and are thus re-specified here.","quantity":1,"attributes":[{"name":"interval","dtype":"int32","value":1,"doc":"Value is '1'"},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for timestamps, which is fixed to 'seconds'."}]}]},{"neurodata_type_def":"FeatureExtraction","neurodata_type_inc":"NWBDataInterface","default_name":"FeatureExtraction","doc":"Features, such as PC1 and PC2, that are extracted from signals stored in a SpikeEventSeries or other source.","datasets":[{"name":"description","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Description of features (eg, ''PC1'') for each of the extracted features."},{"name":"features","dtype":"float32","dims":["num_events","num_channels","num_features"],"shape":[null,null,null],"doc":"Multi-dimensional array of features extracted from each event."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"Times of events that features correspond to (can be a link)."},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the electrodes that this time series was generated from."}]},{"neurodata_type_def":"EventDetection","neurodata_type_inc":"NWBDataInterface","default_name":"EventDetection","doc":"Detected spike events from voltage trace(s).","datasets":[{"name":"detection_method","dtype":"text","doc":"Description of how events were detected, such as voltage threshold, or dV/dT threshold, as well as relevant values."},{"name":"source_idx","dtype":"int32","dims":[["num_events"],["num_events","time_index, channel_index"]],"shape":[[null],[null,2]],"doc":"Indices (zero-based) into the linked source ElectricalSeries::data array corresponding to time of event or time and channel of event. ''description'' should define what is meant by time of event (e.g., .25 ms before action potential peak, zero-crossing time, etc). The index points to each event from the raw data."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"DEPRECATED. Timestamps of events, in seconds.","attributes":[{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for event times, which is fixed to 'seconds'."}],"quantity":"?"}],"links":[{"name":"source_electricalseries","target_type":"ElectricalSeries","doc":"Link to the ElectricalSeries that this data was calculated from. Metadata about electrodes and their position can be read from that ElectricalSeries so it's not necessary to include that information here."}]},{"neurodata_type_def":"EventWaveform","neurodata_type_inc":"NWBDataInterface","default_name":"EventWaveform","doc":"DEPRECATED. Represents either the waveforms of detected events, as extracted from a raw data trace in /acquisition, or the event waveforms that were stored during experiment acquisition.","groups":[{"neurodata_type_inc":"SpikeEventSeries","doc":"SpikeEventSeries object(s) containing detected spike event waveforms.","quantity":"+"}]},{"neurodata_type_def":"FilteredEphys","neurodata_type_inc":"NWBDataInterface","default_name":"FilteredEphys","doc":"Electrophysiology data from one or more channels that has been subjected to filtering. Examples of filtered data include Theta and Gamma (LFP has its own interface). FilteredEphys modules publish an ElectricalSeries for each filtered channel or set of channels. The name of each ElectricalSeries is arbitrary but should be informative. The source of the filtered data, whether this is from analysis of another time series or as acquired by hardware, should be noted in each's TimeSeries::description field. There is no assumed 1::1 correspondence between filtered ephys signals and electrodes, as a single signal can apply to many nearby electrodes, and one electrode may have different filtered (e.g., theta and/or gamma) signals represented. Filter properties should be noted in the ElectricalSeries 'filtering' attribute.","groups":[{"neurodata_type_inc":"ElectricalSeries","doc":"ElectricalSeries object(s) containing filtered electrophysiology data.","quantity":"+"}]},{"neurodata_type_def":"LFP","neurodata_type_inc":"NWBDataInterface","default_name":"LFP","doc":"LFP data from one or more channels. The electrode map in each published ElectricalSeries will identify which channels are providing LFP data. Filter properties should be noted in the ElectricalSeries 'filtering' attribute.","groups":[{"neurodata_type_inc":"ElectricalSeries","doc":"ElectricalSeries object(s) containing LFP data for one or more channels.","quantity":"+"}]},{"neurodata_type_def":"ElectrodeGroup","neurodata_type_inc":"NWBContainer","doc":"A physical grouping of electrodes, e.g. a shank of an array.","attributes":[{"name":"description","dtype":"text","doc":"Description of this electrode group."},{"name":"location","dtype":"text","doc":"Location of electrode group. Specify the area, layer, comments on estimation of area/layer, etc. Use standard atlas names for anatomical regions when possible."}],"datasets":[{"name":"position","dtype":[{"name":"x","dtype":"float32","doc":"x coordinate"},{"name":"y","dtype":"float32","doc":"y coordinate"},{"name":"z","dtype":"float32","doc":"z coordinate"}],"doc":"stereotaxic or common framework coordinates","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the device that was used to record from this electrode group."}]},{"neurodata_type_def":"ElectrodesTable","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e. channels) used for recording. Introduced in NWB 2.8.0. Replaces the \"electrodes\" table (neurodata_type_inc DynamicTable, no neurodata_type_def) that is part of NWBFile.","datasets":[{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of.","quantity":"?"},{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior).","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior).","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right).","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]},{"neurodata_type_def":"ClusterWaveforms","neurodata_type_inc":"NWBDataInterface","default_name":"ClusterWaveforms","doc":"DEPRECATED The mean waveform shape, including standard deviation, of the different clusters. Ideally, the waveform analysis should be performed on data that is only high-pass filtered. This is a separate module because it is expected to require updating. For example, IMEC probes may require different storage requirements to store/display mean waveforms, requiring a new interface or an extension of this one.","datasets":[{"name":"waveform_filtering","dtype":"text","doc":"Filtering applied to data before generating mean/sd"},{"name":"waveform_mean","dtype":"float32","dims":["num_clusters","num_samples"],"shape":[null,null],"doc":"The mean waveform for each cluster, using the same indices for each wave as cluster numbers in the associated Clustering module (i.e, cluster 3 is in array slot [3]). Waveforms corresponding to gaps in cluster sequence should be empty (e.g., zero- filled)"},{"name":"waveform_sd","dtype":"float32","dims":["num_clusters","num_samples"],"shape":[null,null],"doc":"Stdev of waveforms for each cluster, using the same indices as in mean"}],"links":[{"name":"clustering_interface","target_type":"Clustering","doc":"Link to Clustering interface that was the source of the clustered data"}]},{"neurodata_type_def":"Clustering","neurodata_type_inc":"NWBDataInterface","default_name":"Clustering","doc":"DEPRECATED Clustered spike data, whether from automatic clustering tools (e.g., klustakwik) or as a result of manual sorting.","datasets":[{"name":"description","dtype":"text","doc":"Description of clusters or clustering, (e.g. cluster 0 is noise, clusters curated using Klusters, etc)"},{"name":"num","dtype":"int32","dims":["num_events"],"shape":[null],"doc":"Cluster number of each event"},{"name":"peak_over_rms","dtype":"float32","dims":["num_clusters"],"shape":[null],"doc":"Maximum ratio of waveform peak to RMS on any channel in the cluster (provides a basic clustering metric)."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"Times of clustered events, in seconds. This may be a link to times field in associated FeatureExtraction module."}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"ElectricalSeries","neurodata_type_inc":"TimeSeries","doc":"A time series of acquired voltage data from extracellular recordings. The data field is an int or float array storing data in volts. The first dimension should always represent time. The second dimension, if present, should represent channels.","attributes":[{"name":"filtering","dtype":"text","doc":"Filtering applied to all channels of the data. For example, if this ElectricalSeries represents high-pass-filtered data (also known as AP Band), then this value could be \"High-pass 4-pole Bessel filter at 500 Hz\". If this ElectricalSeries represents low-pass-filtered LFP data and the type of filter is unknown, then this value could be \"Low-pass filter at 300 Hz\". If a non-standard filter type is used, provide as much detail about the filter properties as possible.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_channels"],["num_times","num_channels","num_samples"]],"shape":[[null],[null,null],[null,null,null]],"doc":"Recorded voltage data.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data. This value is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion', followed by 'channel_conversion' (if present), and then add 'offset'."}]},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the electrodes that this time series was generated from."},{"name":"channel_conversion","dtype":"float32","dims":["num_channels"],"shape":[null],"doc":"Channel-specific conversion factor. Multiply the data in the 'data' dataset by these values along the channel axis (as indicated by axis attribute) AND by the global conversion factor in the 'conversion' attribute of 'data' to get the data values in Volts, i.e., data in Volts = data * data.conversion * channel_conversion. This approach allows for both global and per-channel data conversion factors needed to support the storage of electrical recordings as native values generated by data acquisition systems. If this dataset is not present, then there is no channel-specific conversion factor, i.e., it is 1 for all channels.","quantity":"?","attributes":[{"name":"axis","dtype":"int32","value":1,"doc":"The zero-indexed axis of the 'data' dataset that the channel-specific conversion factor corresponds to. This value is fixed to 1."}]}]},{"neurodata_type_def":"SpikeEventSeries","neurodata_type_inc":"ElectricalSeries","doc":"Stores snapshots/snippets of recorded spike events (i.e., threshold crossings). This may also be raw data, as reported by ephys hardware. If so, the TimeSeries::description field should describe how events were detected. All events span the same recording channels and store snapshots of equal duration. TimeSeries::data array structure: [num events] [num channels] [num samples] (or [num events] [num samples] for single electrode).","datasets":[{"name":"data","dtype":"numeric","dims":[["num_events","num_samples"],["num_events","num_channels","num_samples"]],"shape":[[null,null],[null,null,null]],"doc":"Spike waveforms.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Unit of measurement for waveforms, which is fixed to 'volts'."}]},{"name":"timestamps","dtype":"float64","dims":["num_times"],"shape":[null],"doc":"Timestamps for samples stored in data, in seconds, relative to the common experiment master-clock stored in NWBFile.timestamps_reference_time. Timestamps are required for the events. Unlike for TimeSeries, timestamps are required for SpikeEventSeries and are thus re-specified here.","quantity":1,"attributes":[{"name":"interval","dtype":"int32","value":1,"doc":"Value is '1'"},{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for timestamps, which is fixed to 'seconds'."}]}]},{"neurodata_type_def":"FeatureExtraction","neurodata_type_inc":"NWBDataInterface","default_name":"FeatureExtraction","doc":"Features, such as PC1 and PC2, that are extracted from signals stored in a SpikeEventSeries or other source.","datasets":[{"name":"description","dtype":"text","dims":["num_features"],"shape":[null],"doc":"Description of features (e.g., ''PC1'') for each of the extracted features."},{"name":"features","dtype":"float32","dims":["num_events","num_channels","num_features"],"shape":[null,null,null],"doc":"Multi-dimensional array of features extracted from each event."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"Times of events that features correspond to (can be a link)."},{"name":"electrodes","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion pointer to the electrodes that this time series was generated from."}]},{"neurodata_type_def":"EventDetection","neurodata_type_inc":"NWBDataInterface","default_name":"EventDetection","doc":"Detected spike events from voltage trace(s).","datasets":[{"name":"detection_method","dtype":"text","doc":"Description of how events were detected, such as voltage threshold, or dV/dT threshold, as well as relevant values."},{"name":"source_idx","dtype":"int32","dims":[["num_events"],["num_events","time_index, channel_index"]],"shape":[[null],[null,2]],"doc":"Indices (zero-based) into the linked source ElectricalSeries::data array corresponding to the time of each event, or the time and channel of each event. ''description'' should define what is meant by time of event (e.g., .25 ms before action potential peak, zero-crossing time, etc.). The index points to each event from the raw data."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"DEPRECATED. Timestamps of events, in seconds.","attributes":[{"name":"unit","dtype":"text","value":"seconds","doc":"Unit of measurement for event times, which is fixed to 'seconds'."}],"quantity":"?"}],"links":[{"name":"source_electricalseries","target_type":"ElectricalSeries","doc":"Link to the ElectricalSeries that this data was calculated from. Metadata about electrodes and their position can be read from that ElectricalSeries so it is not necessary to include that information here."}]},{"neurodata_type_def":"EventWaveform","neurodata_type_inc":"NWBDataInterface","default_name":"EventWaveform","doc":"DEPRECATED. Represents either the waveforms of detected events, as extracted from a raw data trace in /acquisition, or the event waveforms that were stored during experiment acquisition.","groups":[{"neurodata_type_inc":"SpikeEventSeries","doc":"SpikeEventSeries object(s) containing detected spike event waveforms.","quantity":"+"}]},{"neurodata_type_def":"FilteredEphys","neurodata_type_inc":"NWBDataInterface","default_name":"FilteredEphys","doc":"Electrophysiology data from one or more channels that has been subjected to filtering. Examples of filtered data include Theta and Gamma (LFP has its own interface). FilteredEphys modules publish an ElectricalSeries for each filtered channel or set of channels. The name of each ElectricalSeries is arbitrary but should be informative. The source of the filtered data, whether this is from analysis of another time series or as acquired by hardware, should be noted in the TimeSeries::description field of each ElectricalSeries. There is no assumed 1:1 correspondence between filtered ephys signals and electrodes, as a single signal can apply to many nearby electrodes, and one electrode may have different filtered (e.g., theta and/or gamma) signals represented. Filter properties should be noted in the ElectricalSeries 'filtering' attribute.","groups":[{"neurodata_type_inc":"ElectricalSeries","doc":"ElectricalSeries object(s) containing filtered electrophysiology data.","quantity":"+"}]},{"neurodata_type_def":"LFP","neurodata_type_inc":"NWBDataInterface","default_name":"LFP","doc":"LFP data from one or more channels. The electrode map in each published ElectricalSeries will identify which channels are providing LFP data. Filter properties should be noted in the ElectricalSeries 'filtering' attribute.","groups":[{"neurodata_type_inc":"ElectricalSeries","doc":"ElectricalSeries object(s) containing LFP data for one or more channels.","quantity":"+"}]},{"neurodata_type_def":"ElectrodeGroup","neurodata_type_inc":"NWBContainer","doc":"A physical grouping of electrodes, e.g., a shank of an array. An electrode group is typically used to describe electrodes that are physically connected on a single device and are often (but not always) used together for analysis, such as for spike sorting. Note that this is descriptive metadata; electrodes from different groups can still be spike-sorted together if needed.","attributes":[{"name":"description","dtype":"text","doc":"Description of this electrode group."},{"name":"location","dtype":"text","doc":"Location of electrode group. Specify the area, layer, comments on estimation of area/layer, etc. Use standard atlas names for anatomical regions when possible."}],"datasets":[{"name":"position","dtype":[{"name":"x","dtype":"float32","doc":"x coordinate"},{"name":"y","dtype":"float32","doc":"y coordinate"},{"name":"z","dtype":"float32","doc":"z coordinate"}],"doc":"Stereotaxic or common framework coordinates of the electrode group position.","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the device that was used to record from this electrode group."}]},{"neurodata_type_def":"ElectrodesTable","neurodata_type_inc":"DynamicTable","doc":"A table of all electrodes (i.e., channels) used for recording. Introduced in NWB 2.8.0. Replaces the \"electrodes\" table (neurodata_type_inc DynamicTable, no neurodata_type_def) that is part of NWBFile.","datasets":[{"name":"location","neurodata_type_inc":"VectorData","dtype":"text","doc":"Location of the electrode (channel). Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"group","neurodata_type_inc":"VectorData","dtype":{"target_type":"ElectrodeGroup","reftype":"object"},"doc":"Reference to the ElectrodeGroup this electrode is a part of."},{"name":"group_name","neurodata_type_inc":"VectorData","dtype":"text","doc":"Name of the ElectrodeGroup this electrode is a part of.","quantity":"?"},{"name":"x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate of the channel location in the brain (+x is posterior). Units should be specified in microns.","quantity":"?"},{"name":"y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate of the channel location in the brain (+y is inferior). Units should be specified in microns.","quantity":"?"},{"name":"z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate of the channel location in the brain (+z is right). Units should be specified in microns.","quantity":"?"},{"name":"imp","neurodata_type_inc":"VectorData","dtype":"float32","doc":"Impedance of the channel, in ohms.","quantity":"?"},{"name":"filtering","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of hardware filtering, including the filter name and frequency cutoffs.","quantity":"?"},{"name":"rel_x","neurodata_type_inc":"VectorData","dtype":"float32","doc":"x coordinate in electrode group. Units should be specified in microns.","quantity":"?"},{"name":"rel_y","neurodata_type_inc":"VectorData","dtype":"float32","doc":"y coordinate in electrode group. Units should be specified in microns.","quantity":"?"},{"name":"rel_z","neurodata_type_inc":"VectorData","dtype":"float32","doc":"z coordinate in electrode group. Units should be specified in microns.","quantity":"?"},{"name":"reference","neurodata_type_inc":"VectorData","dtype":"text","doc":"Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".","quantity":"?"}]},{"neurodata_type_def":"ClusterWaveforms","neurodata_type_inc":"NWBDataInterface","default_name":"ClusterWaveforms","doc":"DEPRECATED. The mean waveform shape, including standard deviation, of the different clusters. Ideally, the waveform analysis should be performed on data that is only high-pass filtered. This is a separate module because it is expected to require updating. For example, IMEC probes may require different storage requirements to store/display mean waveforms, requiring a new interface or an extension of this one.","datasets":[{"name":"waveform_filtering","dtype":"text","doc":"Filtering applied to data before generating mean/sd"},{"name":"waveform_mean","dtype":"float32","dims":["num_clusters","num_samples"],"shape":[null,null],"doc":"The mean waveform for each cluster, using the same indices for each wave as cluster numbers in the associated Clustering module (i.e., cluster 3 is in array slot [3]). Waveforms corresponding to gaps in cluster sequence should be empty (e.g., zero-filled)."},{"name":"waveform_sd","dtype":"float32","dims":["num_clusters","num_samples"],"shape":[null,null],"doc":"Stdev of waveforms for each cluster, using the same indices as in mean."}],"links":[{"name":"clustering_interface","target_type":"Clustering","doc":"Link to Clustering interface that was the source of the clustered data."}]},{"neurodata_type_def":"Clustering","neurodata_type_inc":"NWBDataInterface","default_name":"Clustering","doc":"DEPRECATED. Clustered spike data, whether from automatic clustering tools (e.g., klustakwik) or as a result of manual sorting.","datasets":[{"name":"description","dtype":"text","doc":"Description of clusters or clustering (e.g., cluster 0 is noise, clusters curated using Klusters, etc.)."},{"name":"num","dtype":"int32","dims":["num_events"],"shape":[null],"doc":"Cluster number of each event"},{"name":"peak_over_rms","dtype":"float32","dims":["num_clusters"],"shape":[null],"doc":"Maximum ratio of waveform peak to RMS on any channel in the cluster (provides a basic clustering metric)."},{"name":"times","dtype":"float64","dims":["num_events"],"shape":[null],"doc":"Times of clustered events, in seconds. This may be a link to times field in associated FeatureExtraction module."}]}]})delimiter"; constexpr std::string_view nwb_icephys = R"delimiter( -{"groups":[{"neurodata_type_def":"PatchClampSeries","neurodata_type_inc":"TimeSeries","doc":"An abstract base class for patch-clamp data - stimulus or response, current or voltage.","attributes":[{"name":"stimulus_description","dtype":"text","doc":"Protocol/stimulus name for this patch-clamp dataset."},{"name":"sweep_number","dtype":"uint32","doc":"Sweep number, allows to group different PatchClampSeries together.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":["num_times"],"shape":[null],"doc":"Recorded voltage or current.","attributes":[{"name":"unit","dtype":"text","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"gain","dtype":"float32","doc":"Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt (c-clamp).","quantity":"?"}],"links":[{"name":"electrode","target_type":"IntracellularElectrode","doc":"Link to IntracellularElectrode object that describes the electrode that was used to apply or record this data."}]},{"neurodata_type_def":"CurrentClampSeries","neurodata_type_inc":"PatchClampSeries","doc":"Voltage data from an intracellular current-clamp recording. A corresponding CurrentClampStimulusSeries (stored separately as a stimulus) is used to store the current injected.","datasets":[{"name":"data","doc":"Recorded voltage.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data. which is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"bias_current","dtype":"float32","doc":"Bias current, in amps.","quantity":"?"},{"name":"bridge_balance","dtype":"float32","doc":"Bridge balance, in ohms.","quantity":"?"},{"name":"capacitance_compensation","dtype":"float32","doc":"Capacitance compensation, in farads.","quantity":"?"}]},{"neurodata_type_def":"IZeroClampSeries","neurodata_type_inc":"CurrentClampSeries","doc":"Voltage data from an intracellular recording when all current and amplifier settings are off (i.e., CurrentClampSeries fields will be zero). There is no CurrentClampStimulusSeries associated with an IZero series because the amplifier is disconnected and no stimulus can reach the cell.","attributes":[{"name":"stimulus_description","dtype":"text","doc":"An IZeroClampSeries has no stimulus, so this attribute is automatically set to \"N/A\"","value":"N/A"}],"datasets":[{"name":"bias_current","dtype":"float32","value":0.0,"doc":"Bias current, in amps, fixed to 0.0."},{"name":"bridge_balance","dtype":"float32","value":0.0,"doc":"Bridge balance, in ohms, fixed to 0.0."},{"name":"capacitance_compensation","dtype":"float32","value":0.0,"doc":"Capacitance compensation, in farads, fixed to 0.0."}]},{"neurodata_type_def":"CurrentClampStimulusSeries","neurodata_type_inc":"PatchClampSeries","doc":"Stimulus current applied during current clamp recording.","datasets":[{"name":"data","doc":"Stimulus current applied.","attributes":[{"name":"unit","dtype":"text","value":"amperes","doc":"Base unit of measurement for working with the data. which is fixed to 'amperes'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]}]},{"neurodata_type_def":"VoltageClampSeries","neurodata_type_inc":"PatchClampSeries","doc":"Current data from an intracellular voltage-clamp recording. A corresponding VoltageClampStimulusSeries (stored separately as a stimulus) is used to store the voltage injected.","datasets":[{"name":"data","doc":"Recorded current.","attributes":[{"name":"unit","dtype":"text","value":"amperes","doc":"Base unit of measurement for working with the data. which is fixed to 'amperes'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"capacitance_fast","dtype":"float32","doc":"Fast capacitance, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for capacitance_fast, which is fixed to 'farads'."}]},{"name":"capacitance_slow","dtype":"float32","doc":"Slow capacitance, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for capacitance_fast, which is fixed to 'farads'."}]},{"name":"resistance_comp_bandwidth","dtype":"float32","doc":"Resistance compensation bandwidth, in hertz.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"hertz","doc":"Unit of measurement for resistance_comp_bandwidth, which is fixed to 'hertz'."}]},{"name":"resistance_comp_correction","dtype":"float32","doc":"Resistance compensation correction, in percent.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"percent","doc":"Unit of measurement for resistance_comp_correction, which is fixed to 'percent'."}]},{"name":"resistance_comp_prediction","dtype":"float32","doc":"Resistance compensation prediction, in percent.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"percent","doc":"Unit of measurement for resistance_comp_prediction, which is fixed to 'percent'."}]},{"name":"whole_cell_capacitance_comp","dtype":"float32","doc":"Whole cell capacitance compensation, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for whole_cell_capacitance_comp, which is fixed to 'farads'."}]},{"name":"whole_cell_series_resistance_comp","dtype":"float32","doc":"Whole cell series resistance compensation, in ohms.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"ohms","doc":"Unit of measurement for whole_cell_series_resistance_comp, which is fixed to 'ohms'."}]}]},{"neurodata_type_def":"VoltageClampStimulusSeries","neurodata_type_inc":"PatchClampSeries","doc":"Stimulus voltage applied during a voltage clamp recording.","datasets":[{"name":"data","doc":"Stimulus voltage applied.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data. which is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]}]},{"neurodata_type_def":"IntracellularElectrode","neurodata_type_inc":"NWBContainer","doc":"An intracellular electrode and its metadata.","datasets":[{"name":"cell_id","dtype":"text","doc":"unique ID of the cell","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of electrode (e.g., whole-cell, sharp, etc.)."},{"name":"filtering","dtype":"text","doc":"Electrode specific filtering.","quantity":"?"},{"name":"initial_access_resistance","dtype":"text","doc":"Initial access resistance.","quantity":"?"},{"name":"location","dtype":"text","doc":"Location of the electrode. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible.","quantity":"?"},{"name":"resistance","dtype":"text","doc":"Electrode resistance, in ohms.","quantity":"?"},{"name":"seal","dtype":"text","doc":"Information about seal used for recording.","quantity":"?"},{"name":"slice","dtype":"text","doc":"Information about slice used for recording.","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Device that was used to record from this electrode."}]},{"neurodata_type_def":"SweepTable","neurodata_type_inc":"DynamicTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable, and ExperimentalConditions tables provide enhanced support for experiment metadata.","datasets":[{"name":"sweep_number","neurodata_type_inc":"VectorData","dtype":"uint32","doc":"Sweep number of the PatchClampSeries in that row."},{"name":"series","neurodata_type_inc":"VectorData","dtype":{"target_type":"PatchClampSeries","reftype":"object"},"doc":"The PatchClampSeries with the sweep number in that row."},{"name":"series_index","neurodata_type_inc":"VectorIndex","doc":"Index for series."}]},{"neurodata_type_def":"IntracellularElectrodesTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular electrode related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular electrode related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"electrode","neurodata_type_inc":"VectorData","dtype":{"target_type":"IntracellularElectrode","reftype":"object"},"doc":"Column for storing the reference to the intracellular electrode."}]},{"neurodata_type_def":"IntracellularStimuliTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular stimulus related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular stimulus related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"stimulus","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the recorded stimulus for the recording (rows)."},{"name":"stimulus_template","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the stimulus template for the recording (rows).","quantity":"?"}]},{"neurodata_type_def":"IntracellularResponsesTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular response related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular response related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"response","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the recorded response for the recording (rows)"}]},{"neurodata_type_def":"IntracellularRecordingsTable","neurodata_type_inc":"AlignedDynamicTable","name":"intracellular_recordings","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response is recorded as part of an experiment. In this case, both the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","attributes":[{"name":"description","dtype":"text","value":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording and for storing metadata about the intracellular recording.","doc":"Description of the contents of this table. Inherited from AlignedDynamicTable and overwritten here to fix the value of the attribute."}],"groups":[{"name":"electrodes","neurodata_type_inc":"IntracellularElectrodesTable","doc":"Table for storing intracellular electrode related metadata."},{"name":"stimuli","neurodata_type_inc":"IntracellularStimuliTable","doc":"Table for storing intracellular stimulus related metadata."},{"name":"responses","neurodata_type_inc":"IntracellularResponsesTable","doc":"Table for storing intracellular response related metadata."}]},{"neurodata_type_def":"SimultaneousRecordingsTable","neurodata_type_inc":"DynamicTable","name":"simultaneous_recordings","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes.","datasets":[{"name":"recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the IntracellularRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"IntracellularRecordingsTable","reftype":"object"},"doc":"Reference to the IntracellularRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the recordings column."}]},{"neurodata_type_def":"SequentialRecordingsTable","neurodata_type_inc":"DynamicTable","name":"sequential_recordings","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","datasets":[{"name":"simultaneous_recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the SimultaneousRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"SimultaneousRecordingsTable","reftype":"object"},"doc":"Reference to the SimultaneousRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"simultaneous_recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the simultaneous_recordings column."},{"name":"stimulus_type","neurodata_type_inc":"VectorData","dtype":"text","doc":"The type of stimulus used for the sequential recording."}]},{"neurodata_type_def":"RepetitionsTable","neurodata_type_inc":"DynamicTable","name":"repetitions","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","datasets":[{"name":"sequential_recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the SequentialRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"SequentialRecordingsTable","reftype":"object"},"doc":"Reference to the SequentialRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"sequential_recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the sequential_recordings column."}]},{"neurodata_type_def":"ExperimentalConditionsTable","neurodata_type_inc":"DynamicTable","name":"experimental_conditions","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental condition.","datasets":[{"name":"repetitions","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the RepetitionsTable table.","attributes":[{"name":"table","dtype":{"target_type":"RepetitionsTable","reftype":"object"},"doc":"Reference to the RepetitionsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"repetitions_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the repetitions column."}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"PatchClampSeries","neurodata_type_inc":"TimeSeries","doc":"An abstract base class for patch-clamp data - stimulus or response, current or voltage.","attributes":[{"name":"stimulus_description","dtype":"text","doc":"Protocol/stimulus name for this patch-clamp dataset."},{"name":"sweep_number","dtype":"uint32","doc":"Sweep number, allows users to group different PatchClampSeries together.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":["num_times"],"shape":[null],"doc":"Recorded voltage or current.","attributes":[{"name":"unit","dtype":"text","doc":"Base unit of measurement for working with the data. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"gain","dtype":"float32","doc":"Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt (c-clamp).","quantity":"?"}],"links":[{"name":"electrode","target_type":"IntracellularElectrode","doc":"Link to IntracellularElectrode object that describes the electrode that was used to apply or record this data."}]},{"neurodata_type_def":"CurrentClampSeries","neurodata_type_inc":"PatchClampSeries","doc":"Voltage data from an intracellular current-clamp recording. A corresponding CurrentClampStimulusSeries (stored separately as a stimulus) is used to store the current injected.","datasets":[{"name":"data","doc":"Recorded voltage.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data, which is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"bias_current","dtype":"float32","doc":"Bias current, in amperes.","quantity":"?"},{"name":"bridge_balance","dtype":"float32","doc":"Bridge balance, in ohms.","quantity":"?"},{"name":"capacitance_compensation","dtype":"float32","doc":"Capacitance compensation, in farads.","quantity":"?"}]},{"neurodata_type_def":"IZeroClampSeries","neurodata_type_inc":"CurrentClampSeries","doc":"Voltage data from an intracellular recording when all current and amplifier settings are off (i.e., CurrentClampSeries fields will be zero). There is no CurrentClampStimulusSeries associated with an IZero series because the amplifier is disconnected and no stimulus can reach the cell.","attributes":[{"name":"stimulus_description","dtype":"text","doc":"An IZeroClampSeries has no stimulus, so this attribute is automatically set to \"N/A\".","value":"N/A"}],"datasets":[{"name":"bias_current","dtype":"float32","value":0.0,"doc":"Bias current, in amperes, fixed to 0.0."},{"name":"bridge_balance","dtype":"float32","value":0.0,"doc":"Bridge balance, in ohms, fixed to 0.0."},{"name":"capacitance_compensation","dtype":"float32","value":0.0,"doc":"Capacitance compensation, in farads, fixed to 0.0."}]},{"neurodata_type_def":"CurrentClampStimulusSeries","neurodata_type_inc":"PatchClampSeries","doc":"Stimulus current applied during current clamp recording.","datasets":[{"name":"data","doc":"Stimulus current applied.","attributes":[{"name":"unit","dtype":"text","value":"amperes","doc":"Base unit of measurement for working with the data, which is fixed to 'amperes'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]}]},{"neurodata_type_def":"VoltageClampSeries","neurodata_type_inc":"PatchClampSeries","doc":"Current data from an intracellular voltage-clamp recording. A corresponding VoltageClampStimulusSeries (stored separately as a stimulus) is used to store the voltage injected.","datasets":[{"name":"data","doc":"Recorded current.","attributes":[{"name":"unit","dtype":"text","value":"amperes","doc":"Base unit of measurement for working with the data, which is fixed to 'amperes'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]},{"name":"capacitance_fast","dtype":"float32","doc":"Fast capacitance, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for capacitance_fast, which is fixed to 'farads'."}]},{"name":"capacitance_slow","dtype":"float32","doc":"Slow capacitance, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for capacitance_slow, which is fixed to 'farads'."}]},{"name":"resistance_comp_bandwidth","dtype":"float32","doc":"Resistance compensation bandwidth, in hertz.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"hertz","doc":"Unit of measurement for resistance_comp_bandwidth, which is fixed to 'hertz'."}]},{"name":"resistance_comp_correction","dtype":"float32","doc":"Resistance compensation correction, in percent.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"percent","doc":"Unit of measurement for resistance_comp_correction, which is fixed to 'percent'."}]},{"name":"resistance_comp_prediction","dtype":"float32","doc":"Resistance compensation prediction, in percent.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"percent","doc":"Unit of measurement for resistance_comp_prediction, which is fixed to 'percent'."}]},{"name":"whole_cell_capacitance_comp","dtype":"float32","doc":"Whole cell capacitance compensation, in farads.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"farads","doc":"Unit of measurement for whole_cell_capacitance_comp, which is fixed to 'farads'."}]},{"name":"whole_cell_series_resistance_comp","dtype":"float32","doc":"Whole cell series resistance compensation, in ohms.","quantity":"?","attributes":[{"name":"unit","dtype":"text","value":"ohms","doc":"Unit of measurement for whole_cell_series_resistance_comp, which is fixed to 'ohms'."}]}]},{"neurodata_type_def":"VoltageClampStimulusSeries","neurodata_type_inc":"PatchClampSeries","doc":"Stimulus voltage applied during a voltage clamp recording.","datasets":[{"name":"data","doc":"Stimulus voltage applied.","attributes":[{"name":"unit","dtype":"text","value":"volts","doc":"Base unit of measurement for working with the data, which is fixed to 'volts'. Actual stored values are not necessarily stored in these units. To access the data in these units, multiply 'data' by 'conversion' and add 'offset'."}]}]},{"neurodata_type_def":"IntracellularElectrode","neurodata_type_inc":"NWBContainer","doc":"An intracellular electrode and its metadata.","datasets":[{"name":"cell_id","dtype":"text","doc":"Unique identifier of the cell.","quantity":"?"},{"name":"description","dtype":"text","doc":"Description of electrode (e.g., whole-cell, sharp, etc.)."},{"name":"filtering","dtype":"text","doc":"Electrode specific filtering.","quantity":"?"},{"name":"initial_access_resistance","dtype":"text","doc":"Initial access resistance.","quantity":"?"},{"name":"location","dtype":"text","doc":"Location of the electrode. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible.","quantity":"?"},{"name":"resistance","dtype":"text","doc":"Electrode resistance, in ohms.","quantity":"?"},{"name":"seal","dtype":"text","doc":"Information about seal used for recording.","quantity":"?"},{"name":"slice","dtype":"text","doc":"Information about slice used for recording.","quantity":"?"}],"links":[{"name":"device","target_type":"Device","doc":"Device that was used to record from this electrode."}]},{"neurodata_type_def":"SweepTable","neurodata_type_inc":"DynamicTable","doc":"[DEPRECATED] Table used to group different PatchClampSeries. SweepTable is being replaced by IntracellularRecordingsTable and SimultaneousRecordingsTable tables. Additional SequentialRecordingsTable, RepetitionsTable, and ExperimentalConditions tables provide enhanced support for experiment metadata.","datasets":[{"name":"sweep_number","neurodata_type_inc":"VectorData","dtype":"uint32","doc":"Sweep number of the PatchClampSeries in that row."},{"name":"series","neurodata_type_inc":"VectorData","dtype":{"target_type":"PatchClampSeries","reftype":"object"},"doc":"The PatchClampSeries with the sweep number in that row."},{"name":"series_index","neurodata_type_inc":"VectorIndex","doc":"Index for series."}]},{"neurodata_type_def":"IntracellularElectrodesTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular electrode related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular electrode related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"electrode","neurodata_type_inc":"VectorData","dtype":{"target_type":"IntracellularElectrode","reftype":"object"},"doc":"Column for storing the reference to the intracellular electrode."}]},{"neurodata_type_def":"IntracellularStimuliTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular stimulus related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular stimulus related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"stimulus","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the recorded stimulus for the recording (rows)."},{"name":"stimulus_template","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the stimulus template for the recording (rows).","quantity":"?"}]},{"neurodata_type_def":"IntracellularResponsesTable","neurodata_type_inc":"DynamicTable","doc":"Table for storing intracellular response related metadata.","attributes":[{"name":"description","dtype":"text","value":"Table for storing intracellular response related metadata.","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"response","neurodata_type_inc":"TimeSeriesReferenceVectorData","doc":"Column storing the reference to the recorded response for the recording (rows)."}]},{"neurodata_type_def":"IntracellularRecordingsTable","neurodata_type_inc":"AlignedDynamicTable","name":"intracellular_recordings","doc":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording. Each row in the table represents a single recording consisting typically of a stimulus and a corresponding response. In some cases, however, only a stimulus or a response is recorded as part of an experiment. In this case, both the stimulus and response will point to the same TimeSeries while the idx_start and count of the invalid column will be set to -1, thus, indicating that no values have been recorded for the stimulus or response, respectively. Note, a recording MUST contain at least a stimulus or a response. Typically the stimulus and response are PatchClampSeries. However, the use of AD/DA channels that are not associated to an electrode is also common in intracellular electrophysiology, in which case other TimeSeries may be used.","attributes":[{"name":"description","dtype":"text","value":"A table to group together a stimulus and response from a single electrode and a single simultaneous recording and for storing metadata about the intracellular recording.","doc":"Description of the contents of this table. Inherited from AlignedDynamicTable and overwritten here to fix the value of the attribute."}],"groups":[{"name":"electrodes","neurodata_type_inc":"IntracellularElectrodesTable","doc":"Table for storing intracellular electrode related metadata."},{"name":"stimuli","neurodata_type_inc":"IntracellularStimuliTable","doc":"Table for storing intracellular stimulus related metadata."},{"name":"responses","neurodata_type_inc":"IntracellularResponsesTable","doc":"Table for storing intracellular response related metadata."}]},{"neurodata_type_def":"SimultaneousRecordingsTable","neurodata_type_inc":"DynamicTable","name":"simultaneous_recordings","doc":"A table for grouping different intracellular recordings from the IntracellularRecordingsTable table together that were recorded simultaneously from different electrodes.","datasets":[{"name":"recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the IntracellularRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"IntracellularRecordingsTable","reftype":"object"},"doc":"Reference to the IntracellularRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the recordings column."}]},{"neurodata_type_def":"SequentialRecordingsTable","neurodata_type_inc":"DynamicTable","name":"sequential_recordings","doc":"A table for grouping different sequential recordings from the SimultaneousRecordingsTable table together. This is typically used to group together sequential recordings where a sequence of stimuli of the same type with varying parameters have been presented in a sequence.","datasets":[{"name":"simultaneous_recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the SimultaneousRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"SimultaneousRecordingsTable","reftype":"object"},"doc":"Reference to the SimultaneousRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"simultaneous_recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the simultaneous_recordings column."},{"name":"stimulus_type","neurodata_type_inc":"VectorData","dtype":"text","doc":"The type of stimulus used for the sequential recording."}]},{"neurodata_type_def":"RepetitionsTable","neurodata_type_inc":"DynamicTable","name":"repetitions","doc":"A table for grouping different sequential intracellular recordings together. With each SequentialRecording typically representing a particular type of stimulus, the RepetitionsTable table is typically used to group sets of stimuli applied in sequence.","datasets":[{"name":"sequential_recordings","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the SequentialRecordingsTable table.","attributes":[{"name":"table","dtype":{"target_type":"SequentialRecordingsTable","reftype":"object"},"doc":"Reference to the SequentialRecordingsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"sequential_recordings_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the sequential_recordings column."}]},{"neurodata_type_def":"ExperimentalConditionsTable","neurodata_type_inc":"DynamicTable","name":"experimental_conditions","doc":"A table for grouping different intracellular recording repetitions together that belong to the same experimental condition.","datasets":[{"name":"repetitions","neurodata_type_inc":"DynamicTableRegion","doc":"A reference to one or more rows in the RepetitionsTable table.","attributes":[{"name":"table","dtype":{"target_type":"RepetitionsTable","reftype":"object"},"doc":"Reference to the RepetitionsTable table that this table region applies to. This specializes the attribute inherited from DynamicTableRegion to fix the type of table that can be referenced here."}]},{"name":"repetitions_index","neurodata_type_inc":"VectorIndex","doc":"Index dataset for the repetitions column."}]}]})delimiter"; constexpr std::string_view nwb_ogen = R"delimiter( -{"groups":[{"neurodata_type_def":"OptogeneticSeries","neurodata_type_inc":"TimeSeries","doc":"An optogenetic stimulus.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_rois"]],"shape":[[null],[null,null]],"doc":"Applied power for optogenetic stimulus, in watts. Shape can be 1D or 2D. 2D data is meant to be used in an extension of OptogeneticSeries that defines what the second dimension represents.","attributes":[{"name":"unit","dtype":"text","value":"watts","doc":"Unit of measurement for data, which is fixed to 'watts'."}]}],"links":[{"name":"site","target_type":"OptogeneticStimulusSite","doc":"Link to OptogeneticStimulusSite object that describes the site to which this stimulus was applied."}]},{"neurodata_type_def":"OptogeneticStimulusSite","neurodata_type_inc":"NWBContainer","doc":"A site of optogenetic stimulation.","datasets":[{"name":"description","dtype":"text","doc":"Description of stimulation site."},{"name":"excitation_lambda","dtype":"float32","doc":"Excitation wavelength, in nm."},{"name":"location","dtype":"text","doc":"Location of the stimulation site. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."}],"links":[{"name":"device","target_type":"Device","doc":"Device that generated the stimulus."}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"OptogeneticSeries","neurodata_type_inc":"TimeSeries","doc":"An optogenetic stimulus.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_rois"]],"shape":[[null],[null,null]],"doc":"Applied power for optogenetic stimulus, in watts. Shape can be 1-D or 2-D. 2-D data is meant to be used in an extension of OptogeneticSeries that defines what the second dimension represents.","attributes":[{"name":"unit","dtype":"text","value":"watts","doc":"Unit of measurement for data, which is fixed to 'watts'."}]}],"links":[{"name":"site","target_type":"OptogeneticStimulusSite","doc":"Link to OptogeneticStimulusSite object that describes the site to which this stimulus was applied."}]},{"neurodata_type_def":"OptogeneticStimulusSite","neurodata_type_inc":"NWBContainer","doc":"A site of optogenetic stimulation.","datasets":[{"name":"description","dtype":"text","doc":"Description of stimulation site."},{"name":"excitation_lambda","dtype":"float32","doc":"Excitation wavelength, in nm."},{"name":"location","dtype":"text","doc":"Location of the stimulation site. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."}],"links":[{"name":"device","target_type":"Device","doc":"Device that generated the stimulus."}]}]})delimiter"; constexpr std::string_view nwb_ophys = R"delimiter( -{"groups":[{"neurodata_type_def":"OnePhotonSeries","neurodata_type_inc":"ImageSeries","doc":"Image stack recorded over time from 1-photon microscope.","attributes":[{"name":"pmt_gain","dtype":"float32","doc":"Photomultiplier gain.","required":false},{"name":"scan_line_rate","dtype":"float32","doc":"Lines imaged per second. This is also stored in /general/optophysiology but is kept here as it is useful information for analysis, and so good to be stored w/ the actual data.","required":false},{"name":"exposure_time","dtype":"float32","doc":"Exposure time of the sample; often the inverse of the frequency.","required":false},{"name":"binning","dtype":"uint8","doc":"Amount of pixels combined into 'bins'; could be 1, 2, 4, 8, etc.","required":false},{"name":"power","dtype":"float32","doc":"Power of the excitation in mW, if known.","required":false},{"name":"intensity","dtype":"float32","doc":"Intensity of the excitation in mW/mm^2, if known.","required":false}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this TimeSeries data was generated."}]},{"neurodata_type_def":"TwoPhotonSeries","neurodata_type_inc":"ImageSeries","doc":"Image stack recorded over time from 2-photon microscope.","attributes":[{"name":"pmt_gain","dtype":"float32","doc":"Photomultiplier gain.","required":false},{"name":"scan_line_rate","dtype":"float32","doc":"Lines imaged per second. This is also stored in /general/optophysiology but is kept here as it is useful information for analysis, and so good to be stored w/ the actual data.","required":false}],"datasets":[{"name":"field_of_view","dtype":"float32","dims":[["width|height"],["width|height|depth"]],"shape":[[2],[3]],"doc":"Width, height and depth of image, or imaged area, in meters.","quantity":"?"}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this TimeSeries data was generated."}]},{"neurodata_type_def":"RoiResponseSeries","neurodata_type_inc":"TimeSeries","doc":"ROI responses over an imaging plane. The first dimension represents time. The second dimension, if present, represents ROIs.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_ROIs"]],"shape":[[null],[null,null]],"doc":"Signals from ROIs."},{"name":"rois","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion referencing into an ROITable containing information on the ROIs stored in this timeseries."}]},{"neurodata_type_def":"DfOverF","neurodata_type_inc":"NWBDataInterface","default_name":"DfOverF","doc":"dF/F information about a region of interest (ROI). Storage hierarchy of dF/F should be the same as for segmentation (i.e., same names for ROIs and for image planes).","groups":[{"neurodata_type_inc":"RoiResponseSeries","doc":"RoiResponseSeries object(s) containing dF/F for a ROI.","quantity":"+"}]},{"neurodata_type_def":"Fluorescence","neurodata_type_inc":"NWBDataInterface","default_name":"Fluorescence","doc":"Fluorescence information about a region of interest (ROI). Storage hierarchy of fluorescence should be the same as for segmentation (ie, same names for ROIs and for image planes).","groups":[{"neurodata_type_inc":"RoiResponseSeries","doc":"RoiResponseSeries object(s) containing fluorescence data for a ROI.","quantity":"+"}]},{"neurodata_type_def":"ImageSegmentation","neurodata_type_inc":"NWBDataInterface","default_name":"ImageSegmentation","doc":"Stores pixels in an image that represent different regions of interest (ROIs) or masks. All segmentation for a given imaging plane is stored together, with storage for multiple imaging planes (masks) supported. Each ROI is stored in its own subgroup, with the ROI group containing both a 2D mask and a list of pixels that make up this mask. Segments can also be used for masking neuropil. If segmentation is allowed to change with time, a new imaging plane (or module) is required and ROI names should remain consistent between them.","groups":[{"neurodata_type_inc":"PlaneSegmentation","doc":"Results from image segmentation of a specific imaging plane.","quantity":"+"}]},{"neurodata_type_def":"PlaneSegmentation","neurodata_type_inc":"DynamicTable","doc":"Results from image segmentation of a specific imaging plane. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","datasets":[{"name":"image_mask","neurodata_type_inc":"VectorData","dims":[["num_roi","num_x","num_y"],["num_roi","num_x","num_y","num_z"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"},{"name":"pixel_mask_index","neurodata_type_inc":"VectorIndex","doc":"Index into pixel_mask.","quantity":"?"},{"name":"pixel_mask","neurodata_type_inc":"VectorData","dtype":[{"name":"x","dtype":"uint32","doc":"Pixel x-coordinate."},{"name":"y","dtype":"uint32","doc":"Pixel y-coordinate."},{"name":"weight","dtype":"float32","doc":"Weight of the pixel."}],"doc":"Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"},{"name":"voxel_mask_index","neurodata_type_inc":"VectorIndex","doc":"Index into voxel_mask.","quantity":"?"},{"name":"voxel_mask","neurodata_type_inc":"VectorData","dtype":[{"name":"x","dtype":"uint32","doc":"Voxel x-coordinate."},{"name":"y","dtype":"uint32","doc":"Voxel y-coordinate."},{"name":"z","dtype":"uint32","doc":"Voxel z-coordinate."},{"name":"weight","dtype":"float32","doc":"Weight of the voxel."}],"doc":"Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"}],"groups":[{"name":"reference_images","doc":"Image stacks that the segmentation masks apply to.","groups":[{"neurodata_type_inc":"ImageSeries","doc":"One or more image stacks that the masks apply to (can be one-element stack).","quantity":"*"}]}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this data was generated."}]},{"neurodata_type_def":"ImagingPlane","neurodata_type_inc":"NWBContainer","doc":"An imaging plane and its metadata.","datasets":[{"name":"description","dtype":"text","doc":"Description of the imaging plane.","quantity":"?"},{"name":"excitation_lambda","dtype":"float32","doc":"Excitation wavelength, in nm."},{"name":"imaging_rate","dtype":"float32","doc":"Rate that images are acquired, in Hz. If the corresponding TimeSeries is present, the rate should be stored there instead.","quantity":"?"},{"name":"indicator","dtype":"text","doc":"Calcium indicator."},{"name":"location","dtype":"text","doc":"Location of the imaging plane. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"manifold","dtype":"float32","dims":[["height","width","x, y, z"],["height","width","depth","x, y, z"]],"shape":[[null,null,3],[null,null,null,3]],"doc":"DEPRECATED Physical position of each pixel. 'xyz' represents the position of the pixel relative to the defined coordinate space. Deprecated in favor of origin_coords and grid_spacing.","quantity":"?","attributes":[{"name":"conversion","dtype":"float32","default_value":1.0,"doc":"Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. e.g. if the data acquisition system stores values in this object as pixels from x = -500 to 499, y = -500 to 499 that correspond to a 2 m x 2 m range, then the 'conversion' multiplier to get from raw data acquisition pixel units to meters is 2/1000.","required":false},{"name":"unit","dtype":"text","default_value":"meters","doc":"Base unit of measurement for working with the data. The default value is 'meters'.","required":false}]},{"name":"origin_coords","dtype":"float32","dims":[["x, y"],["x, y, z"]],"shape":[[2],[3]],"doc":"Physical location of the first element of the imaging plane (0, 0) for 2-D data or (0, 0, 0) for 3-D data. See also reference_frame for what the physical location is relative to (e.g., bregma).","quantity":"?","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Measurement units for origin_coords. The default value is 'meters'."}]},{"name":"grid_spacing","dtype":"float32","dims":[["x, y"],["x, y, z"]],"shape":[[2],[3]],"doc":"Space between pixels in (x, y) or voxels in (x, y, z) directions, in the specified unit. Assumes imaging plane is a regular grid. See also reference_frame to interpret the grid.","quantity":"?","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Measurement units for grid_spacing. The default value is 'meters'."}]},{"name":"reference_frame","dtype":"text","doc":"Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"","quantity":"?"}],"groups":[{"neurodata_type_inc":"OpticalChannel","doc":"An optical channel used to record from an imaging plane.","quantity":"+"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the Device object that was used to record from this electrode."}]},{"neurodata_type_def":"OpticalChannel","neurodata_type_inc":"NWBContainer","doc":"An optical channel used to record from an imaging plane.","datasets":[{"name":"description","dtype":"text","doc":"Description or other notes about the channel."},{"name":"emission_lambda","dtype":"float32","doc":"Emission wavelength for channel, in nm."}]},{"neurodata_type_def":"MotionCorrection","neurodata_type_inc":"NWBDataInterface","default_name":"MotionCorrection","doc":"An image stack where all frames are shifted (registered) to a common coordinate system, to account for movement and drift between frames. Note: each frame at each point in time is assumed to be 2-D (has only x & y dimensions).","groups":[{"neurodata_type_inc":"CorrectedImageStack","doc":"Results from motion correction of an image stack.","quantity":"+"}]},{"neurodata_type_def":"CorrectedImageStack","neurodata_type_inc":"NWBDataInterface","doc":"Results from motion correction of an image stack.","groups":[{"name":"corrected","neurodata_type_inc":"ImageSeries","doc":"Image stack with frames shifted to the common coordinates."},{"name":"xy_translation","neurodata_type_inc":"TimeSeries","doc":"Stores the x,y delta necessary to align each frame to the common coordinates, for example, to align each frame to a reference image."}],"links":[{"name":"original","target_type":"ImageSeries","doc":"Link to ImageSeries object that is being registered."}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"OnePhotonSeries","neurodata_type_inc":"ImageSeries","doc":"Image stack recorded over time from 1-photon microscope.","attributes":[{"name":"pmt_gain","dtype":"float32","doc":"Photomultiplier gain.","required":false},{"name":"scan_line_rate","dtype":"float32","doc":"Lines imaged per second. This is also stored in /general/optophysiology but is kept here as it is useful information for analysis, and is therefore stored with the actual data.","required":false},{"name":"exposure_time","dtype":"float32","doc":"Exposure time of the sample; often the inverse of the frequency.","required":false},{"name":"binning","dtype":"uint8","doc":"Amount of pixels combined into 'bins'; could be 1, 2, 4, 8, etc.","required":false},{"name":"power","dtype":"float32","doc":"Power of the excitation in mW, if known.","required":false},{"name":"intensity","dtype":"float32","doc":"Intensity of the excitation in mW/mm^2, if known.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":[["time","width","height"],["time","width","height","depth"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"Time-series of microscopy data indexed as data[time, width, height].\n\nIndexing convention:\n - width: first spatial axis (horizontal direction, columns)\n - height: second spatial axis (vertical direction, rows)\n\nFor raster-scanning systems (two-photon, confocal, one-photon laser scanning):\n - width: fast scan direction (horizontal)\n - height: slow scan direction (vertical)\n\nCoordinate system diagram::\n\n height ^\n (slow) |\n | +-----------------+\n | | |\n | | Image plane |\n | | |\n | +-----------------+\n |\n 0 ----------------------> width (fast)\n\n\nFor non-raster systems (wide-field, light-sheet, random-access), width and height\nare arbitrary coordinates of the image plane.\n\nNote: This indexing convention conflicts with standard matrix[row, column] notation\nfor array access where the first index moves vertically through rows and the second moves horizontally\nthrough columns. In the schema, data[time, width, height] means that the first spatial\nindex moves horizontally (width) through columns and the second spatial index moves\nvertically (height) through rows. In summary, incrementing the first spatial index\nmoves you horizontally across the image, while in matrix notation it would move you\nvertically across the rows of the image.\n"}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this TimeSeries data was generated."}]},{"neurodata_type_def":"TwoPhotonSeries","neurodata_type_inc":"ImageSeries","doc":"Image stack recorded over time from 2-photon microscope.","attributes":[{"name":"pmt_gain","dtype":"float32","doc":"Photomultiplier gain.","required":false},{"name":"scan_line_rate","dtype":"float32","doc":"Lines imaged per second. This is also stored in /general/optophysiology but is kept here as it is useful information for analysis, and is therefore stored with the actual data.","required":false}],"datasets":[{"name":"data","dtype":"numeric","dims":[["time","width","height"],["time","width","height","depth"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"Time-series of microscopy data indexed as data[time, width, height].\n\nIndexing convention:\n - width: first spatial axis (horizontal direction, columns)\n - height: second spatial axis (vertical direction, rows)\n\nFor raster-scanning systems (two-photon, confocal, one-photon laser scanning):\n - width: fast scan direction (horizontal)\n - height: slow scan direction (vertical)\n\nCoordinate system diagram::\n\n height ^\n (slow) |\n | +-----------------+\n | | |\n | | Image plane |\n | | |\n | +-----------------+\n |\n 0 ----------------------> width (fast)\n\n\nFor non-raster systems (wide-field, light-sheet, random-access), width and height\nare arbitrary coordinates of the image plane.\n\nNote: This indexing convention conflicts with standard matrix[row, column] notation\nfor array access where the first index moves vertically through rows and the second moves horizontally\nthrough columns. In the schema, data[time, width, height] means that the first spatial\nindex moves horizontally (width) through columns and the second spatial index moves\nvertically (height) through rows.\n"},{"name":"field_of_view","dtype":"float32","dims":[["width|height"],["width|height|depth"]],"shape":[[2],[3]],"doc":"Physical dimensions of the imaging field of view in width, height, and optionally depth directions, in meters. The width dimension corresponds to the faster scanning axis, and the height dimension corresponds to the slower scanning axis.","quantity":"?"}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this TimeSeries data was generated."}]},{"neurodata_type_def":"RoiResponseSeries","neurodata_type_inc":"TimeSeries","doc":"ROI responses over an imaging plane. The first dimension represents time. The second dimension, if present, represents ROIs.","datasets":[{"name":"data","dtype":"numeric","dims":[["num_times"],["num_times","num_ROIs"]],"shape":[[null],[null,null]],"doc":"Signals from ROIs."},{"name":"rois","neurodata_type_inc":"DynamicTableRegion","doc":"DynamicTableRegion referencing into a PlaneSegmentation table containing information on the ROIs stored in this TimeSeries."}]},{"neurodata_type_def":"DfOverF","neurodata_type_inc":"NWBDataInterface","default_name":"DfOverF","doc":"dF/F information about a region of interest (ROI). Storage hierarchy of dF/F should be the same as for segmentation (i.e., same names for ROIs and for image planes).","groups":[{"neurodata_type_inc":"RoiResponseSeries","doc":"RoiResponseSeries object(s) containing dF/F for a ROI.","quantity":"+"}]},{"neurodata_type_def":"Fluorescence","neurodata_type_inc":"NWBDataInterface","default_name":"Fluorescence","doc":"Fluorescence information about a region of interest (ROI). Storage hierarchy of fluorescence should be the same as for segmentation (i.e., same names for ROIs and for image planes).","groups":[{"neurodata_type_inc":"RoiResponseSeries","doc":"RoiResponseSeries object(s) containing fluorescence data for a ROI.","quantity":"+"}]},{"neurodata_type_def":"ImageSegmentation","neurodata_type_inc":"NWBDataInterface","default_name":"ImageSegmentation","doc":"Stores pixels in an image that represent different regions of interest (ROIs) or masks. All segmentation for a given imaging plane is stored together, with storage for multiple imaging planes (masks) supported. Each ROI is stored in its own subgroup, with the ROI group containing both a 2-D mask and a list of pixels that make up this mask. Segments can also be used for masking neuropil. If segmentation is allowed to change with time, a new imaging plane (or module) is required and ROI names should remain consistent between them.","groups":[{"neurodata_type_inc":"PlaneSegmentation","doc":"Results from image segmentation of a specific imaging plane.","quantity":"+"}]},{"neurodata_type_def":"PlaneSegmentation","neurodata_type_inc":"DynamicTable","doc":"Results from image segmentation of a specific imaging plane. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","datasets":[{"name":"image_mask","neurodata_type_inc":"VectorData","dims":[["num_roi","num_x","num_y"],["num_roi","num_x","num_y","num_z"]],"shape":[[null,null,null],[null,null,null,null]],"doc":"ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"},{"name":"pixel_mask_index","neurodata_type_inc":"VectorIndex","doc":"Index into pixel_mask.","quantity":"?"},{"name":"pixel_mask","neurodata_type_inc":"VectorData","dtype":[{"name":"x","dtype":"uint32","doc":"Pixel x-coordinate."},{"name":"y","dtype":"uint32","doc":"Pixel y-coordinate."},{"name":"weight","dtype":"float32","doc":"Weight of the pixel."}],"doc":"Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"},{"name":"voxel_mask_index","neurodata_type_inc":"VectorIndex","doc":"Index into voxel_mask.","quantity":"?"},{"name":"voxel_mask","neurodata_type_inc":"VectorData","dtype":[{"name":"x","dtype":"uint32","doc":"Voxel x-coordinate."},{"name":"y","dtype":"uint32","doc":"Voxel y-coordinate."},{"name":"z","dtype":"uint32","doc":"Voxel z-coordinate."},{"name":"weight","dtype":"float32","doc":"Weight of the voxel."}],"doc":"Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.","quantity":"?"}],"groups":[{"name":"reference_images","doc":"Image stacks that the segmentation masks apply to.","groups":[{"neurodata_type_inc":"ImageSeries","doc":"One or more image stacks that the masks apply to (can be one-element stack).","quantity":"*"}]}],"links":[{"name":"imaging_plane","target_type":"ImagingPlane","doc":"Link to ImagingPlane object from which this data was generated."}]},{"neurodata_type_def":"ImagingPlane","neurodata_type_inc":"NWBContainer","doc":"An imaging plane and its metadata.","datasets":[{"name":"description","dtype":"text","doc":"Description of the imaging plane.","quantity":"?"},{"name":"excitation_lambda","dtype":"float32","doc":"Excitation wavelength, in nm."},{"name":"imaging_rate","dtype":"float32","doc":"Rate that images are acquired, in Hz. If the corresponding TimeSeries is present, the rate should be stored there instead.","quantity":"?"},{"name":"indicator","dtype":"text","doc":"Calcium indicator."},{"name":"location","dtype":"text","doc":"Location of the imaging plane. Specify the area, layer, comments on estimation of area/layer, stereotaxic coordinates if in vivo, etc. Use standard atlas names for anatomical regions when possible."},{"name":"manifold","dtype":"float32","dims":[["height","width","x, y, z"],["height","width","depth","x, y, z"]],"shape":[[null,null,3],[null,null,null,3]],"doc":"DEPRECATED. Physical position of each pixel. 'xyz' represents the position of the pixel relative to the defined coordinate space. Deprecated in favor of origin_coords and grid_spacing.","quantity":"?","attributes":[{"name":"conversion","dtype":"float32","default_value":1.0,"doc":"Scalar to multiply each element in data to convert it to the specified 'unit'. If the data are stored in acquisition system units or other units that require a conversion to be interpretable, multiply the data by 'conversion' to convert the data to the specified 'unit'. For example, if the data acquisition system stores values in this object as pixels from x = -500 to 499, y = -500 to 499 that correspond to a 2 m x 2 m range, then the 'conversion' multiplier to get from raw data acquisition pixel units to meters is 2/1000.","required":false},{"name":"unit","dtype":"text","default_value":"meters","doc":"Base unit of measurement for working with the data. The default value is 'meters'.","required":false}]},{"name":"origin_coords","dtype":"float32","dims":[["x, y"],["x, y, z"]],"shape":[[2],[3]],"doc":"Physical location of the first element of the imaging plane (0, 0) for 2-D data or (0, 0, 0) for 3-D data. See also reference_frame for what the physical location is relative to (e.g., bregma).","quantity":"?","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Measurement units for origin_coords. The default value is 'meters'."}]},{"name":"grid_spacing","dtype":"float32","dims":[["x, y"],["x, y, z"]],"shape":[[2],[3]],"doc":"Space between pixels in (x, y) or voxels in (x, y, z) directions, in the specified unit. Assumes imaging plane is a regular grid. See also reference_frame to interpret the grid.","quantity":"?","attributes":[{"name":"unit","dtype":"text","default_value":"meters","doc":"Measurement units for grid_spacing. The default value is 'meters'."}]},{"name":"reference_frame","dtype":"text","doc":"Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x mean more anterior, larger numbers in y mean more rightward, and larger numbers in z mean more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"","quantity":"?"}],"groups":[{"neurodata_type_inc":"OpticalChannel","doc":"An optical channel used to record from an imaging plane.","quantity":"+"}],"links":[{"name":"device","target_type":"Device","doc":"Link to the Device object (e.g., microscope) used to acquire images from this imaging plane."}]},{"neurodata_type_def":"OpticalChannel","neurodata_type_inc":"NWBContainer","doc":"An optical channel used to record from an imaging plane.","datasets":[{"name":"description","dtype":"text","doc":"Description or other notes about the channel."},{"name":"emission_lambda","dtype":"float32","doc":"Emission wavelength for channel, in nm."}]},{"neurodata_type_def":"MotionCorrection","neurodata_type_inc":"NWBDataInterface","default_name":"MotionCorrection","doc":"An image stack where all frames are shifted (registered) to a common coordinate system, to account for movement and drift between frames. Note: each frame at each point in time is assumed to be 2-D (has only x and y dimensions).","groups":[{"neurodata_type_inc":"CorrectedImageStack","doc":"Results from motion correction of an image stack.","quantity":"+"}]},{"neurodata_type_def":"CorrectedImageStack","neurodata_type_inc":"NWBDataInterface","doc":"Results from motion correction of an image stack.","groups":[{"name":"corrected","neurodata_type_inc":"ImageSeries","doc":"Image stack with frames shifted to the common coordinates."},{"name":"xy_translation","neurodata_type_inc":"TimeSeries","doc":"Stores the x,y delta necessary to align each frame to the common coordinates, for example, to align each frame to a reference image."}],"links":[{"name":"original","target_type":"ImageSeries","doc":"Link to ImageSeries object that is being registered."}]}]})delimiter"; constexpr std::string_view nwb_retinotopy = R"delimiter( -{"groups":[{"neurodata_type_def":"ImagingRetinotopy","neurodata_type_inc":"NWBDataInterface","default_name":"ImagingRetinotopy","doc":"DEPRECATED. Intrinsic signal optical imaging or widefield imaging for measuring retinotopy. Stores orthogonal maps (e.g., altitude/azimuth; radius/theta) of responses to specific stimuli and a combined polarity map from which to identify visual areas. This group does not store the raw responses imaged during retinotopic mapping or the stimuli presented, but rather the resulting phase and power maps after applying a Fourier transform on the averaged responses. Note: for data consistency, all images and arrays are stored in the format [row][column] and [row, col], which equates to [y][x]. Field of view and dimension arrays may appear backward (i.e., y before x).","datasets":[{"name":"axis_1_phase_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Phase response to stimulus on the first measured axis.","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_1_power_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_2_phase_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Phase response to stimulus on the second measured axis.","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_2_power_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Power response on the second measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_descriptions","dtype":"text","dims":["axis_1, axis_2"],"shape":[2],"doc":"Two-element array describing the contents of the two response axis fields. Description should be something like ['altitude', 'azimuth'] or '['radius', 'theta']."},{"name":"focal_depth_image","dtype":"uint16","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Gray-scale image taken with same settings/parameters (e.g., focal depth, wavelength) as data collection. Array format: [rows][columns].","quantity":"?","attributes":[{"name":"bits_per_pixel","dtype":"int32","doc":"Number of bits used to represent each value. This is necessary to determine maximum (white) pixel value."},{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"focal_depth","dtype":"float32","doc":"Focal depth offset, in meters."},{"name":"format","dtype":"text","doc":"Format of image. Right now only 'raw' is supported."}]},{"name":"sign_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Sine of the angle between the direction of the gradient in axis_1 and axis_2.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."}]},{"name":"vasculature_image","dtype":"uint16","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Gray-scale anatomical image of cortical surface. Array structure: [rows][columns]","attributes":[{"name":"bits_per_pixel","dtype":"int32","doc":"Number of bits used to represent each value. This is necessary to determine maximum (white) pixel value"},{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"format","dtype":"text","doc":"Format of image. Right now only 'raw' is supported."}]}]}]})delimiter"; +{"groups":[{"neurodata_type_def":"ImagingRetinotopy","neurodata_type_inc":"NWBDataInterface","default_name":"ImagingRetinotopy","doc":"DEPRECATED. Intrinsic signal optical imaging or widefield imaging for measuring retinotopy. Stores orthogonal maps (e.g., altitude/azimuth; radius/theta) of responses to specific stimuli and a combined polarity map from which to identify visual areas. This group does not store the raw responses imaged during retinotopic mapping or the stimuli presented, but rather the resulting phase and power maps after applying a Fourier transform on the averaged responses. Note: for data consistency, all images and arrays are stored in the format [row][column] and [row, col], which equates to [y][x]. Field of view and dimension arrays may appear backward (i.e., y before x).","datasets":[{"name":"axis_1_phase_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Phase response to stimulus on the first measured axis.","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_1_power_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_2_phase_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Phase response to stimulus on the second measured axis.","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_2_power_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Power response on the second measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"unit","dtype":"text","doc":"Unit that axis data is stored in (e.g., degrees)."}]},{"name":"axis_descriptions","dtype":"text","dims":["axis_1, axis_2"],"shape":[2],"doc":"Two-element array describing the contents of the two response axis fields. Description should be something like ['altitude', 'azimuth'] or ['radius', 'theta']."},{"name":"focal_depth_image","dtype":"uint16","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Gray-scale image taken with the same settings/parameters (e.g., focal depth, wavelength) as data collection. Array format: [rows][columns].","quantity":"?","attributes":[{"name":"bits_per_pixel","dtype":"int32","doc":"Number of bits used to represent each value. This is necessary to determine maximum (white) pixel value."},{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"focal_depth","dtype":"float32","doc":"Focal depth offset, in meters."},{"name":"format","dtype":"text","doc":"Format of image. Currently only 'raw' is supported."}]},{"name":"sign_map","dtype":"float32","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Sine of the angle between the direction of the gradient in axis_1 and axis_2.","quantity":"?","attributes":[{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."}]},{"name":"vasculature_image","dtype":"uint16","dims":["num_rows","num_cols"],"shape":[null,null],"doc":"Gray-scale anatomical image of cortical surface. Array structure: [rows][columns]","attributes":[{"name":"bits_per_pixel","dtype":"int32","doc":"Number of bits used to represent each value. This is necessary to determine maximum (white) pixel value."},{"name":"dimension","dtype":"int32","dims":["num_rows, num_cols"],"shape":[2],"doc":"Number of rows and columns in the image. NOTE: row, column representation is equivalent to height, width."},{"name":"field_of_view","dtype":"float32","dims":["height, width"],"shape":[2],"doc":"Size of viewing area, in meters."},{"name":"format","dtype":"text","doc":"Format of image. Currently only 'raw' is supported."}]}]}]})delimiter"; constexpr std::string_view namespaces = R"delimiter( -{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"source":"nwb.base"},{"source":"nwb.device"},{"source":"nwb.epoch"},{"source":"nwb.image"},{"source":"nwb.file"},{"source":"nwb.misc"},{"source":"nwb.behavior"},{"source":"nwb.ecephys"},{"source":"nwb.icephys"},{"source":"nwb.ogen"},{"source":"nwb.ophys"},{"source":"nwb.retinotopy"}],"version":"2.9.0"}]})delimiter"; +{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","keithg@alleninstitute.org","jteeters@berkeley.edu"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"source":"nwb.base"},{"source":"nwb.device"},{"source":"nwb.epoch"},{"source":"nwb.event"},{"source":"nwb.image"},{"source":"nwb.file"},{"source":"nwb.misc"},{"source":"nwb.behavior"},{"source":"nwb.ecephys"},{"source":"nwb.icephys"},{"source":"nwb.ogen"},{"source":"nwb.ophys"},{"source":"nwb.retinotopy"}],"version":"2.11.0"}]})delimiter"; const std::vector> specVariables {{ {"nwb.base", nwb_base}, {"nwb.device", nwb_device}, {"nwb.epoch", nwb_epoch}, + {"nwb.event", nwb_event}, {"nwb.image", nwb_image}, {"nwb.file", nwb_file}, {"nwb.misc", nwb_misc}, diff --git a/src/spec/hdmf_common.hpp b/src/spec/hdmf_common.hpp index 39c55793c..6ece227ec 100644 --- a/src/spec/hdmf_common.hpp +++ b/src/spec/hdmf_common.hpp @@ -10,7 +10,7 @@ /*! * \namespace AQNWB::SPEC::HDMF_COMMON * @brief Namespace for the hdmf-common schema specification. - * @version 1.8.0 + * @version 1.10.0 * * This file is auto-generated from the schema files. * Do not edit this file manually. @@ -20,26 +20,30 @@ namespace AQNWB::SPEC::HDMF_COMMON const std::string namespaceName = "hdmf-common"; -const std::string version = "1.8.0"; +const std::string version = "1.10.0"; // clang-format off constexpr std::string_view base = R"delimiter( {"datasets":[{"data_type_def":"Data","doc":"An abstract data type for a dataset."}],"groups":[{"data_type_def":"Container","doc":"An abstract data type for a group storing collections of data and metadata. Base type for all data and metadata containers."},{"data_type_def":"SimpleMultiContainer","data_type_inc":"Container","doc":"A simple Container for holding onto multiple containers.","datasets":[{"data_type_inc":"Data","quantity":"*","doc":"Data objects held within this SimpleMultiContainer."}],"groups":[{"data_type_inc":"Container","quantity":"*","doc":"Container objects held within this SimpleMultiContainer."}]}]})delimiter"; constexpr std::string_view table = R"delimiter( -{"datasets":[{"data_type_def":"VectorData","data_type_inc":"Data","doc":"An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on.","dims":[["dim0"],["dim0","dim1"],["dim0","dim1","dim2"],["dim0","dim1","dim2","dim3"]],"shape":[[null],[null,null],[null,null,null],[null,null,null,null]],"attributes":[{"name":"description","dtype":"text","doc":"Description of what these vectors represent."}]},{"data_type_def":"VectorIndex","data_type_inc":"VectorData","dtype":"uint8","doc":"Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\".","dims":["num_rows"],"shape":[null],"attributes":[{"name":"target","dtype":{"target_type":"VectorData","reftype":"object"},"doc":"Reference to the target dataset that this index applies to."}]},{"data_type_def":"ElementIdentifiers","data_type_inc":"Data","default_name":"element_id","dtype":"int","dims":["num_elements"],"shape":[null],"doc":"A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable."},{"data_type_def":"DynamicTableRegion","data_type_inc":"VectorData","dtype":"int","doc":"DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`.","dims":["num_rows"],"shape":[null],"attributes":[{"name":"table","dtype":{"target_type":"DynamicTable","reftype":"object"},"doc":"Reference to the DynamicTable object that this region applies to."},{"name":"description","dtype":"text","doc":"Description of what this table region points to."}]}],"groups":[{"data_type_def":"DynamicTable","data_type_inc":"Container","doc":"A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable.","attributes":[{"name":"colnames","dtype":"text","dims":["num_columns"],"shape":[null],"doc":"The names of the columns in this table. This should be used to specify an order to the columns."},{"name":"description","dtype":"text","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"id","data_type_inc":"ElementIdentifiers","dtype":"int","dims":["num_rows"],"shape":[null],"doc":"Array of unique identifiers for the rows of this dynamic table."},{"data_type_inc":"VectorData","doc":"Vector columns, including index columns, of this dynamic table.","quantity":"*"}]},{"data_type_def":"AlignedDynamicTable","data_type_inc":"DynamicTable","doc":"DynamicTable container that supports storing a collection of sub-tables. Each sub-table is a DynamicTable itself that is aligned with the main table by row index. I.e., all DynamicTables stored in this group MUST have the same number of rows. This type effectively defines a 2-level table in which the main data is stored in the main table implemented by this type and additional columns of the table are grouped into categories, with each category being represented by a separate DynamicTable stored within the group.","attributes":[{"name":"categories","dtype":"text","dims":["num_categories"],"shape":[null],"doc":"The names of the categories in this AlignedDynamicTable. Each category is represented by one DynamicTable stored in the parent group. This attribute should be used to specify an order of categories and the category names must match the names of the corresponding DynamicTable in the group."}],"groups":[{"data_type_inc":"DynamicTable","doc":"A DynamicTable representing a particular category for columns in the AlignedDynamicTable parent container. The table MUST be aligned with (i.e., have the same number of rows) as all other DynamicTables stored in the AlignedDynamicTable parent container. The name of the category is given by the name of the DynamicTable and its description by the description attribute of the DynamicTable.","quantity":"*"}]}]})delimiter"; +{"datasets":[{"data_type_def":"VectorData","data_type_inc":"Data","doc":"An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on.","dims":[["dim0"],["dim0","dim1"],["dim0","dim1","dim2"],["dim0","dim1","dim2","dim3"]],"shape":[[null],[null,null],[null,null,null],[null,null,null,null]],"attributes":[{"name":"description","dtype":"text","doc":"Description of what these vectors represent."}]},{"data_type_def":"VectorIndex","data_type_inc":"VectorData","dtype":"uint8","doc":"Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\".","dims":["num_rows"],"shape":[null],"attributes":[{"name":"target","dtype":{"target_type":"VectorData","reftype":"object"},"doc":"Reference to the target dataset that this index applies to."}]},{"data_type_def":"ElementIdentifiers","data_type_inc":"Data","default_name":"element_id","dtype":"int32","dims":["num_elements"],"shape":[null],"doc":"A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable."},{"data_type_def":"DynamicTableRegion","data_type_inc":"VectorData","dtype":"int32","doc":"DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`.","dims":["num_rows"],"shape":[null],"attributes":[{"name":"table","dtype":{"target_type":"DynamicTable","reftype":"object"},"doc":"Reference to the DynamicTable object that this region applies to."},{"name":"description","dtype":"text","doc":"Description of what this table region points to."}]}],"groups":[{"data_type_def":"DynamicTable","data_type_inc":"Container","doc":"A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement is left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable.","attributes":[{"name":"colnames","dtype":"text","dims":["num_columns"],"shape":[null],"doc":"The names of the columns in this table. This should be used to specify an order to the columns."},{"name":"description","dtype":"text","doc":"Description of what is in this dynamic table."}],"datasets":[{"name":"id","data_type_inc":"ElementIdentifiers","dims":["num_rows"],"shape":[null],"doc":"Array of unique identifiers for the rows of this dynamic table."},{"data_type_inc":"VectorData","doc":"Vector columns, including index columns, of this dynamic table.","quantity":"*"}],"groups":[{"name":"meanings_tables","doc":"Group containing MeaningsTable objects that provide meanings for values in VectorData columns within this DynamicTable.","quantity":"?","groups":[{"data_type_inc":"MeaningsTable","doc":"MeaningsTable objects that provide meanings for values in VectorData columns within this DynamicTable. Tables should be named according to the column they provide meanings for with a \"_meanings\" suffix. e.g., if a VectorData column is named \"stimulus_type\", the corresponding MeaningsTable should be named \"stimulus_type_meanings\".","quantity":"*"}]}]},{"data_type_def":"AlignedDynamicTable","data_type_inc":"DynamicTable","doc":"DynamicTable container that supports storing a collection of sub-tables. Each sub-table is a DynamicTable itself that is aligned with the main table by row index. I.e., all DynamicTables stored in this group MUST have the same number of rows. This type effectively defines a 2-level table in which the main data is stored in the main table implemented by this type and additional columns of the table are grouped into categories, with each category being represented by a separate DynamicTable stored within the group.","attributes":[{"name":"categories","dtype":"text","dims":["num_categories"],"shape":[null],"doc":"The names of the categories in this AlignedDynamicTable. Each category is represented by one DynamicTable stored in the parent group. This attribute should be used to specify an order of categories and the category names must match the names of the corresponding DynamicTable in the group."}],"groups":[{"data_type_inc":"DynamicTable","doc":"A DynamicTable representing a particular category for columns in the AlignedDynamicTable parent container. The table MUST be aligned with (i.e., have the same number of rows) as all other DynamicTables stored in the AlignedDynamicTable parent container. The name of the category is given by the name of the DynamicTable and its description by the description attribute of the DynamicTable.","quantity":"*"}]},{"data_type_def":"MeaningsTable","data_type_inc":"DynamicTable","doc":"A table to store information about the meanings of values in a referenced VectorData object. All possible values of the referenced VectorData object should be present in the 'value' column of this table, even if the value is not observed in the data. Additional columns may be added to store additional metadata about each value. The name of the MeaningsTable should correspond to the name of the referenced VectorData object with a \"_meanings\" suffix. e.g., if the referenced VectorData object is named \"stimulus_type\", the corresponding MeaningsTable should be named \"stimulus_type_meanings\".","datasets":[{"name":"value","data_type_inc":"VectorData","doc":"The value of a row in the referenced VectorData object."},{"name":"meaning","data_type_inc":"VectorData","dtype":"text","doc":"The meaning of the value in the referenced VectorData object."}],"attributes":[{"name":"target","dtype":{"target_type":"VectorData","reftype":"object"},"doc":"Reference to the VectorData object for which this table provides meanings."}]}]})delimiter"; constexpr std::string_view sparse = R"delimiter( {"groups":[{"data_type_def":"CSRMatrix","data_type_inc":"Container","doc":"A compressed sparse row matrix. Data are stored in the standard CSR format, where column indices for row i are stored in indices[indptr[i]:indptr[i+1]] and their corresponding values are stored in data[indptr[i]:indptr[i+1]].","attributes":[{"name":"shape","dtype":"uint","dims":["number of rows, number of columns"],"shape":[2],"doc":"The shape (number of rows, number of columns) of this sparse matrix."}],"datasets":[{"name":"indices","dtype":"uint","dims":["number of non-zero values"],"shape":[null],"doc":"The column indices."},{"name":"indptr","dtype":"uint","dims":["number of rows in the matrix + 1"],"shape":[null],"doc":"The row index pointer."},{"name":"data","dims":["number of non-zero values"],"shape":[null],"doc":"The non-zero values in the matrix."}]}]})delimiter"; +constexpr std::string_view resources = R"delimiter( +{"groups":[{"data_type_def":"HERD","data_type_inc":"Container","doc":"HDMF External Resources Data Structure. A set of six tables for tracking external resource references in a file or across multiple files.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the `resources` table, e.g., \"human\"."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"files","doc":"A table for storing object ids of files used in external resources.","dtype":[{"name":"file_object_id","dtype":"text","doc":"The object id (UUID) of a file that contains objects that refers to external resources."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"entity_id","dtype":"text","doc":"The compact uniform resource identifier (CURIE) of the entity, in the form [prefix]:[unique local identifier], e.g., 'NCBI_TAXON:9606'."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string. e.g., https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=9606"}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"files_idx","dtype":"uint","doc":"The row index to the file in the `files` table containing the object."},{"name":"object_id","dtype":"text","doc":"The object id (UUID) of the object."},{"name":"object_type","dtype":"text","doc":"The data type of the object."},{"name":"relative_path","dtype":"text","doc":"The relative path from the data object with the `object_id` to the dataset or attribute with the value(s) that is associated with an external resource. This can be an empty string if the object is a dataset that contains the value(s) that is associated with an external resource."},{"name":"field","dtype":"text","doc":"The field within the compound data type using an external resource. This is used only if the dataset or attribute is a compound data type; otherwise this should be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The row index to the object in the `objects` table that holds the key"},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entity_keys","doc":"A table for identifying which keys use which entity.","dtype":[{"name":"entities_idx","dtype":"uint","doc":"The row index to the entity in the `entities` table."},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]}]}]})delimiter"; + constexpr std::string_view namespaces = R"delimiter( -{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"source":"base"},{"source":"table"},{"source":"sparse"}],"version":"1.8.0"}]})delimiter"; +{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov"],"full_name":"HDMF Common","schema":[{"source":"base"},{"source":"table"},{"source":"sparse"},{"source":"resources"}],"version":"1.10.0"}]})delimiter"; const std::vector> specVariables {{ {"base", base}, {"table", table}, {"sparse", sparse}, + {"resources", resources}, {"namespace", namespaces} }}; diff --git a/src/spec/hdmf_experimental.hpp b/src/spec/hdmf_experimental.hpp index 6b14ef00d..8a467082c 100644 --- a/src/spec/hdmf_experimental.hpp +++ b/src/spec/hdmf_experimental.hpp @@ -10,7 +10,7 @@ /*! * \namespace AQNWB::SPEC::HDMF_EXPERIMENTAL * @brief Namespace for the hdmf-experimental schema specification. - * @version 0.5.0 + * @version 0.6.0 * * This file is auto-generated from the schema files. * Do not edit this file manually. @@ -20,22 +20,18 @@ namespace AQNWB::SPEC::HDMF_EXPERIMENTAL const std::string namespaceName = "hdmf-experimental"; -const std::string version = "0.5.0"; +const std::string version = "0.6.0"; // clang-format off constexpr std::string_view experimental = R"delimiter( -{"groups":[],"datasets":[{"data_type_def":"EnumData","data_type_inc":"VectorData","dtype":"uint8","doc":"Data that come from a fixed set of values. A data value of i corresponds to the i-th value in the VectorData referenced by the 'elements' attribute.","attributes":[{"name":"elements","dtype":{"target_type":"VectorData","reftype":"object"},"doc":"Reference to the VectorData object that contains the enumerable elements"}]}]})delimiter"; - -constexpr std::string_view resources = R"delimiter( -{"groups":[{"data_type_def":"HERD","data_type_inc":"Container","doc":"HDMF External Resources Data Structure. A set of six tables for tracking external resource references in a file or across multiple files.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the `resources` table, e.g., \"human\"."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"files","doc":"A table for storing object ids of files used in external resources.","dtype":[{"name":"file_object_id","dtype":"text","doc":"The object id (UUID) of a file that contains objects that refers to external resources."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"entity_id","dtype":"text","doc":"The compact uniform resource identifier (CURIE) of the entity, in the form [prefix]:[unique local identifier], e.g., 'NCBI_TAXON:9606'."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string. e.g., https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=9606"}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"files_idx","dtype":"uint","doc":"The row index to the file in the `files` table containing the object."},{"name":"object_id","dtype":"text","doc":"The object id (UUID) of the object."},{"name":"object_type","dtype":"text","doc":"The data type of the object."},{"name":"relative_path","dtype":"text","doc":"The relative path from the data object with the `object_id` to the dataset or attribute with the value(s) that is associated with an external resource. This can be an empty string if the object is a dataset that contains the value(s) that is associated with an external resource."},{"name":"field","dtype":"text","doc":"The field within the compound data type using an external resource. This is used only if the dataset or attribute is a compound data type; otherwise this should be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The row index to the object in the `objects` table that holds the key"},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entity_keys","doc":"A table for identifying which keys use which entity.","dtype":[{"name":"entities_idx","dtype":"uint","doc":"The row index to the entity in the `entities` table."},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]}]}]})delimiter"; +{"groups":[],"datasets":[{"data_type_def":"EnumData","data_type_inc":"VectorData","dtype":"uint8","doc":"Data that come from a fixed set of values. A data value of i corresponds to the i-th value in the VectorData referenced by the 'elements' attribute.","attributes":[{"name":"elements","dtype":{"target_type":"VectorData","reftype":"object"},"doc":"Reference to the VectorData object that contains the enumerable elements."}]}]})delimiter"; constexpr std::string_view namespaces = R"delimiter( -{"namespaces":[{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future.","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Matthew Avaylon"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","mavaylon@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"source":"experimental"},{"source":"resources"}],"version":"0.5.0"}]})delimiter"; +{"namespaces":[{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future.","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Matthew Avaylon"],"contact":["ajtritt@lbl.gov","oruebel@lbl.gov","rly@lbl.gov","bdichter@lbl.gov","mavaylon@lbl.gov"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"source":"experimental"}],"version":"0.6.0"}]})delimiter"; const std::vector> specVariables {{ {"experimental", experimental}, - {"resources", resources}, {"namespace", namespaces} }}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f790e1cf5..ceb57932f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,14 +15,18 @@ add_executable(aqnwb_test testBaseIO.cpp testChannel.cpp testData.cpp + testDurationVectorData.cpp testDevice.cpp testDynamicTable.cpp + testDynamicTableForwardDeclaredMeanings.cpp testEcephys.cpp testElementIdentifiers.cpp + testEventsTable.cpp testFile.cpp testHDF5IO.cpp testHDF5ArrayDataSetConfig.cpp testHDF5RecordingData.cpp + testMeaningsTable.cpp testMisc.cpp testNamespaceRegistry.cpp testNWBFile.cpp @@ -31,19 +35,25 @@ add_executable(aqnwb_test testRecordingWorkflow.cpp testRecordingObjects.cpp testRegisteredType.cpp + testSubject.cpp testTimeSeries.cpp + testTimestampVectorData.cpp testTypes.cpp testUtilsFunctions.cpp testVectorData.cpp + testVectorIndex.cpp + testTimeIntervals.cpp examples/test_HDF5IO_examples.cpp examples/test_example.cpp examples/testWorkflowExamples.cpp + examples/testEventsWorkflowExamples.cpp examples/test_ecephys_data_read.cpp examples/test_link_timeseries_example.cpp examples/test_RegisteredType_example.cpp examples/test_remfile_read_example.cpp examples/test_ros3_read_example.cpp examples/test_templated_RegisteredType_example.cpp + examples/testTimeIntervalsWorkflowExamples.cpp ) # Ensure the aqnwb_test target can include headers from the current directory diff --git a/tests/examples/testEventsWorkflowExamples.cpp b/tests/examples/testEventsWorkflowExamples.cpp new file mode 100644 index 000000000..361d25621 --- /dev/null +++ b/tests/examples/testEventsWorkflowExamples.cpp @@ -0,0 +1,359 @@ + +#include + +#include "Types.hpp" +#include "Utils.hpp" +#include "io/BaseIO.hpp" +#include "io/hdf5/HDF5ArrayDataSetConfig.hpp" +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/event/EventsTable.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" +#include "nwb/hdmf/table/MeaningsTable.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("eventsWorkflowExamples") +{ + // ------------------------------------------------------------------------- + // Variant 1: Row-based acquisition + // ------------------------------------------------------------------------- + SECTION("row-based event acquisition workflow") + { + std::string path = getTestFilePath("exampleEventsRowBased.nwb"); + + // [example_events_rowbased_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + io->open(); + // [example_events_rowbased_io_snippet] + REQUIRE(io->isOpen()); + + // [example_events_rowbased_nwbfile_snippet] + auto nwbfile = NWB::NWBFile::create(io); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); + AQNWB::checkStatus(initStatus, "NWBFile initialization"); + // [example_events_rowbased_nwbfile_snippet] + REQUIRE(initStatus == Status::Success); + + // [example_events_rowbased_create_table_snippet] + // Create an EventsTable for lick events. The table will live at + // /events/licks inside the NWB file. We request a timestamp column + // (resolution 1/30000 s) and an optional annotation column; no duration + // column is needed for this event type. + float timestampResolution = 1.0f / 30000.0f; + auto columnSpecs = NWB::EventsTable::createDefaultDataSpecs( + timestampResolution, + false, // omit duration column + std::nullopt, // duration resolution is ignored since no duration + // column + true, // create annotation column + 100); // number of rows in a chunk (chunked storage is required to + // support append when the total number of rows is not known in + // advance) + + // Optionally add a custom column to the spec list before creating the + // table. Here we add a float32 "confidence" column with a chunk size of 100 + // rows. + IO::ArrayDataSetConfig confidenceConfig( + BaseDataType::F32, // data type + SizeArray {0}, // initial size (0 = extensible) + SizeArray {100}); // chunk size + columnSpecs.push_back(NWB::VectorData::createDataSpec( + "confidence", confidenceConfig, "Detection confidence score [0, 1].")); + + auto eventsTable = nwbfile->createEventsTable( + "licks", // name of the table + "Lick events detected from the lickometer signal.", // description + "Thresholding of lickometer analog signal at 1.5 V", // source + // description + columnSpecs); // pre-built column spec list + // [example_events_rowbased_create_table_snippet] + REQUIRE(eventsTable != nullptr); + + // [example_events_rowbased_start_snippet] + Status startStatus = io->startRecording(); + // [example_events_rowbased_start_snippet] + REQUIRE(startStatus == Status::Success); + + // [example_events_rowbased_write_snippet] + // During acquisition, append individual rows, each + // representing a detected event using addRow(). + // Each row is a map from column name to value. The row ID is + // auto-generated (0, 1, 2, …) when not supplied. + // All columns — including custom ones — must be provided for every row. + AQNWB::Types::RowData row0 = {{"timestamp", 0.123f}, + {"annotation", std::string("lick")}, + {"confidence", 0.95f}}; + Status s0 = eventsTable->addRow(row0); + + // We can also append multiple rows at once using addRows(). + std::vector moreRows = { + {{"timestamp", 0.456f}, + {"annotation", std::string("lick")}, + {"confidence", 0.87f}}, + {{"timestamp", 0.789f}, + {"annotation", std::string("lick")}, + {"confidence", 0.91f}}}; + Status s1 = eventsTable->addRows(moreRows); + + io->flush(); // optional, flush data to disk + // [example_events_rowbased_write_snippet] + REQUIRE(s0 == Status::Success); + REQUIRE(s1 == Status::Success); + + // [example_events_rowbased_stop_snippet] + io->stopRecording(); + io->close(); + // [example_events_rowbased_stop_snippet] + } + + // ------------------------------------------------------------------------- + // Variant 2: Column-based (bulk) acquisition + // ------------------------------------------------------------------------- + SECTION("column-based event acquisition workflow") + { + std::string path = getTestFilePath("exampleEventsColumnBased.nwb"); + + // [example_events_colbased_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + io->open(); + // [example_events_colbased_io_snippet] + REQUIRE(io->isOpen()); + + // [example_events_colbased_nwbfile_snippet] + auto nwbfile = NWB::NWBFile::create(io); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); + AQNWB::checkStatus(initStatus, "NWBFile initialization"); + // [example_events_colbased_nwbfile_snippet] + REQUIRE(initStatus == Status::Success); + + // [example_events_colbased_create_table_snippet] + // Create an EventsTable for stimulus onset events. Here we include both + // a timestamp column and a duration column (resolution 1/30000 s). + float timestampResolution = 1.0f / 30000.0f; + float durationResolution = 1.0f / 30000.0f; + auto columnSpecs = + NWB::EventsTable::createDefaultDataSpecs(timestampResolution, + true, + durationResolution, + false, // no annotation column + 100); // row chunk size + + auto eventsTable = + nwbfile->createEventsTable("stimulus_onsets", + "Stimulus onset events for visual gratings.", + "Hardware TTL pulse on channel 1", + columnSpecs); + // [example_events_colbased_create_table_snippet] + REQUIRE(eventsTable != nullptr); + + // [example_events_colbased_addcolumn_snippet] + // Custom columns can also be added after table creation but before + // startRecording() using addColumn(DataSpecPtr). This is the recommended + // approach and is consistent with the DataSpec-based approach used by + // initialize(). The alternative approach would be to create VectorData + // columns directly and then adding them via addColumn(). + auto conditionSpec = NWB::VectorData::createDataSpec( + "condition", + IO::ArrayDataSetConfig(BaseDataType::V_STR, // variable-length string + SizeArray {0}, // initial size (0 = extensible) + SizeArray {100}), // chunk size + "Stimulus condition label."); + Status addColStatus = eventsTable->addColumn(conditionSpec); + REQUIRE(addColStatus == Status::Success); + // [example_events_colbased_addcolumn_snippet] + + // [example_events_colbased_start_snippet] + Status startStatus = io->startRecording(); + // [example_events_colbased_start_snippet] + REQUIRE(startStatus == Status::Success); + + // [example_events_colbased_write_snippet] + // After the recording session, write all event data as full columns in a + // single call per column. This is efficient when the complete dataset is + // available in memory at write time. + std::vector timestamps = {0.100f, 0.600f, 1.100f, 1.600f}; + std::vector durations = {0.250f, 0.250f, 0.250f, 0.250f}; + std::vector conditions = { + "grating_0", "grating_90", "grating_0", "grating_90"}; + std::vector rowIds = {0, 1, 2, 3}; + + SizeArray dataShape = {timestamps.size()}; + SizeArray positionOffset = {0}; + + // Write the timestamp column + auto timestampColumn = eventsTable->readTimestampColumn(); + Status tsStatus = timestampColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + REQUIRE(tsStatus == Status::Success); + + // Write the duration column + auto durationColumn = eventsTable->readDurationColumn(); + Status durStatus = durationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, durations.data()); + REQUIRE(durStatus == Status::Success); + + // Write the custom "condition" column + auto conditionCol = eventsTable->readColumn("condition"); + Status condStatus = conditionCol->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::V_STR, conditions); + REQUIRE(condStatus == Status::Success); + + // Write the row IDs + Status idStatus = eventsTable->setRowIDs(rowIds); + REQUIRE(idStatus == Status::Success); + + io->flush(); // optional, flush data to disk + // [example_events_colbased_write_snippet] + REQUIRE(tsStatus == Status::Success); + REQUIRE(durStatus == Status::Success); + REQUIRE(condStatus == Status::Success); + REQUIRE(idStatus == Status::Success); + + // [example_events_colbased_stop_snippet] + io->stopRecording(); + io->close(); + // [example_events_colbased_stop_snippet] + } + + // ------------------------------------------------------------------------- + // Variant 3: Adding a MeaningsTable for an event_type column + // ------------------------------------------------------------------------- + SECTION("event acquisition with MeaningsTable for event_type column") + { + std::string path = getTestFilePath("exampleEventsMeaningsTable.nwb"); + + // [example_events_meanings_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + io->open(); + // [example_events_meanings_io_snippet] + REQUIRE(io->isOpen()); + + // [example_events_meanings_nwbfile_snippet] + auto nwbfile = NWB::NWBFile::create(io); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); + AQNWB::checkStatus(initStatus, "NWBFile initialization"); + // [example_events_meanings_nwbfile_snippet] + REQUIRE(initStatus == Status::Success); + + // [example_events_meanings_create_table_snippet] + // Create an EventsTable for behavioral events that have an "event_type" + // column storing integer codes. We will attach a MeaningsTable to that + // column so that readers can look up what each integer code means. + float timestampResolution = 1.0f / 30000.0f; + auto columnSpecs = NWB::EventsTable::createDefaultDataSpecs( + timestampResolution, + false, + std::nullopt, // omit duration column + false, // no annotation column + 100); // row chunk size + + // Add an integer "event_type" column that stores event-type codes. + IO::ArrayDataSetConfig eventTypeConfig( + BaseDataType::I32, // integer event-type codes + SizeArray {0}, // initial size (0 = extensible / chunked) + SizeArray {100}); // chunk size + columnSpecs.push_back(NWB::VectorData::createDataSpec( + "event_type", + eventTypeConfig, + "Integer code identifying the type of behavioral event.")); + + auto eventsTable = + nwbfile->createEventsTable("behavioral_events", + "Behavioral events with typed event codes.", + "Behavioral monitoring system", + columnSpecs); + // [example_events_meanings_create_table_snippet] + REQUIRE(eventsTable != nullptr); + + // [example_events_meanings_create_meanings_table_snippet] + // Before startRecording(), create a MeaningsTable for the "event_type" + // column. The MeaningsTable is stored at + // "meanings_tables/event_type_meanings" inside the EventsTable group and + // maps each integer code to a human-readable label. + auto meaningsTable = eventsTable->createMeaningsTable("event_type"); + // [example_events_meanings_create_meanings_table_snippet] + REQUIRE(meaningsTable != nullptr); + + // [example_events_meanings_start_snippet] + Status startStatus = io->startRecording(); + // [example_events_meanings_start_snippet] + REQUIRE(startStatus == Status::Success); + + // [example_events_meanings_write_events_snippet] + // Write event rows using addRows(). Each row supplies a timestamp and the + // integer event_type code. + std::vector eventRows = { + {{"timestamp", 0.100f}, {"event_type", 1}}, + {{"timestamp", 0.350f}, {"event_type", 2}}, + {{"timestamp", 0.700f}, {"event_type", 1}}, + {{"timestamp", 1.200f}, {"event_type", 3}}, + }; + Status eventsStatus = eventsTable->addRows(eventRows); + // [example_events_meanings_write_events_snippet] + REQUIRE(eventsStatus == Status::Success); + + // [example_events_meanings_write_meanings_snippet] + // Populate the MeaningsTable: list every possible event_type code together + // with its human-readable meaning. All possible values should be present + // even if they do not appear in the recorded data. + std::vector meaningsRows = { + {{"value", 1}, {"meaning", "lick"}}, + {{"value", 2}, {"meaning", "reward_delivery"}}, + {{"value", 3}, {"meaning", "air_puff"}}, + }; + Status meaningsStatus = meaningsTable->addRows(meaningsRows); + io->flush(); // optional, flush data to disk + // [example_events_meanings_write_meanings_snippet] + REQUIRE(meaningsStatus == Status::Success); + + // [example_events_meanings_stop_snippet] + io->stopRecording(); + io->close(); + // [example_events_meanings_stop_snippet] + } +} diff --git a/tests/examples/testTimeIntervalsWorkflowExamples.cpp b/tests/examples/testTimeIntervalsWorkflowExamples.cpp new file mode 100644 index 000000000..67b8d7ab5 --- /dev/null +++ b/tests/examples/testTimeIntervalsWorkflowExamples.cpp @@ -0,0 +1,234 @@ +#include + +#include "Types.hpp" +#include "Utils.hpp" +#include "io/BaseIO.hpp" +#include "io/hdf5/HDF5ArrayDataSetConfig.hpp" +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/epoch/TimeIntervals.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("timeIntervalsWorkflowExamples") +{ + // ------------------------------------------------------------------------- + // Variant 1: Row-based acquisition + // ------------------------------------------------------------------------- + SECTION("row-based time intervals acquisition workflow") + { + std::string path = getTestFilePath("exampleTimeIntervalsRowBased.nwb"); + + // [example_timeintervals_rowbased_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + io->open(); + // [example_timeintervals_rowbased_io_snippet] + REQUIRE(io->isOpen()); + + // [example_timeintervals_rowbased_nwbfile_snippet] + auto nwbfile = NWB::NWBFile::create(io); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); + AQNWB::checkStatus(initStatus, "NWBFile initialization"); + // [example_timeintervals_rowbased_nwbfile_snippet] + REQUIRE(initStatus == Status::Success); + + // [example_timeintervals_rowbased_create_table_snippet] + // Create a TimeIntervals table for trials. The table will live at + // /intervals/trials inside the NWB file. We request the optional tags + // column. + auto columnSpecs = NWB::TimeIntervals::createDefaultDataSpecs( + NWB::NWBFile::INTERVALS_PATH + "/trials", + 100, // number of rows in a chunk (chunked storage is required to + // support append when the total number of rows is not known in + // advance) + true); // create tags column + + // Optionally add a custom column to the spec list before creating the + // table. Here we add a string "condition" column with a chunk size of 100 + // rows. + IO::ArrayDataSetConfig conditionConfig( + BaseDataType::V_STR, // data type + SizeArray {0}, // initial size (0 = extensible) + SizeArray {100}); // chunk size + columnSpecs.push_back(NWB::VectorData::createDataSpec( + "condition", conditionConfig, "Trial condition label.")); + + auto trialsTable = nwbfile->createTimeIntervals( + "trials", // name of the table + "Experimental trials.", // description + columnSpecs); // pre-built column spec list + // [example_timeintervals_rowbased_create_table_snippet] + REQUIRE(trialsTable != nullptr); + + // [example_timeintervals_rowbased_start_snippet] + Status startStatus = io->startRecording(); + // [example_timeintervals_rowbased_start_snippet] + REQUIRE(startStatus == Status::Success); + + // [example_timeintervals_rowbased_write_snippet] + // During acquisition, append individual rows, each + // representing a detected interval using addRow(). + // Each row is a map from column name to value. The row ID is + // auto-generated (0, 1, 2, …) when not supplied. + // All columns — including custom ones — must be provided for every row. + AQNWB::Types::RowData row0 = { + {"start_time", 0.123f}, + {"stop_time", 1.456f}, + {"tags", std::vector {"correct", "fast"}}, + {"condition", std::string("visual_stimulus")}}; + Status s0 = trialsTable->addRow(row0); + + // We can also append multiple rows at once using addRows(). + std::vector moreRows = { + {{"start_time", 2.123f}, + {"stop_time", 3.456f}, + {"tags", std::vector {"incorrect"}}, + {"condition", std::string("auditory_stimulus")}}, + {{"start_time", 4.123f}, + {"stop_time", 5.456f}, + {"tags", std::vector {"correct", "slow"}}, + {"condition", std::string("visual_stimulus")}}}; + Status s1 = trialsTable->addRows(moreRows); + + io->flush(); // optional, flush data to disk + // [example_timeintervals_rowbased_write_snippet] + REQUIRE(s0 == Status::Success); + REQUIRE(s1 == Status::Success); + + // [example_timeintervals_rowbased_stop_snippet] + io->stopRecording(); + io->close(); + // [example_timeintervals_rowbased_stop_snippet] + } + + // ------------------------------------------------------------------------- + // Variant 2: Column-based (bulk) acquisition + // ------------------------------------------------------------------------- + SECTION("column-based time intervals acquisition workflow") + { + std::string path = getTestFilePath("exampleTimeIntervalsColumnBased.nwb"); + + // [example_timeintervals_colbased_io_snippet] + std::shared_ptr io = createIO("HDF5", path); + io->open(); + // [example_timeintervals_colbased_io_snippet] + REQUIRE(io->isOpen()); + + // [example_timeintervals_colbased_nwbfile_snippet] + auto nwbfile = NWB::NWBFile::create(io); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); + AQNWB::checkStatus(initStatus, "NWBFile initialization"); + // [example_timeintervals_colbased_nwbfile_snippet] + REQUIRE(initStatus == Status::Success); + + // [example_timeintervals_colbased_create_table_snippet] + // Create a TimeIntervals table for epochs. + auto columnSpecs = NWB::TimeIntervals::createDefaultDataSpecs( + NWB::NWBFile::INTERVALS_PATH + "/epochs", + 100, // row chunk size + false); // no tags column + + auto epochsTable = nwbfile->createTimeIntervals( + "epochs", "Experimental epochs.", columnSpecs); + // [example_timeintervals_colbased_create_table_snippet] + REQUIRE(epochsTable != nullptr); + + // [example_timeintervals_colbased_addcolumn_snippet] + // Custom columns can also be added after table creation but before + // startRecording() using addColumn(DataSpecPtr). This is the recommended + // approach and is consistent with the DataSpec-based approach used by + // initialize(). The alternative approach would be to create VectorData + // columns directly and then adding them via addColumn(). + auto labelSpec = NWB::VectorData::createDataSpec( + "label", + IO::ArrayDataSetConfig(BaseDataType::V_STR, // variable-length string + SizeArray {0}, // initial size (0 = extensible) + SizeArray {100}), // chunk size + "Epoch label."); + Status addColStatus = epochsTable->addColumn(labelSpec); + REQUIRE(addColStatus == Status::Success); + // [example_timeintervals_colbased_addcolumn_snippet] + + // [example_timeintervals_colbased_start_snippet] + Status startStatus = io->startRecording(); + // [example_timeintervals_colbased_start_snippet] + REQUIRE(startStatus == Status::Success); + + // [example_timeintervals_colbased_write_snippet] + // After the recording session, write all interval data as full columns in a + // single call per column. This is efficient when the complete dataset is + // available in memory at write time. + std::vector startTimes = {0.100f, 10.600f, 20.100f}; + std::vector stopTimes = {10.000f, 20.000f, 30.000f}; + std::vector labels = {"baseline", "stimulation", "recovery"}; + std::vector rowIds = {0, 1, 2}; + + SizeArray dataShape = {startTimes.size()}; + SizeArray positionOffset = {0}; + + // Write the start_time column + auto startTimeColumn = epochsTable->readStartTime(); + Status startStatusWrite = startTimeColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, startTimes.data()); + REQUIRE(startStatusWrite == Status::Success); + + // Write the stop_time column + auto stopTimeColumn = epochsTable->readStopTime(); + Status stopStatusWrite = stopTimeColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, stopTimes.data()); + REQUIRE(stopStatusWrite == Status::Success); + + // Write the custom "label" column + auto labelCol = epochsTable->readColumn("label"); + Status labelStatusWrite = labelCol->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::V_STR, labels); + REQUIRE(labelStatusWrite == Status::Success); + + // Write the row IDs + Status idStatus = epochsTable->setRowIDs(rowIds); + REQUIRE(idStatus == Status::Success); + + io->flush(); // optional, flush data to disk + // [example_timeintervals_colbased_write_snippet] + REQUIRE(startStatusWrite == Status::Success); + REQUIRE(stopStatusWrite == Status::Success); + REQUIRE(labelStatusWrite == Status::Success); + REQUIRE(idStatus == Status::Success); + + // [example_timeintervals_colbased_stop_snippet] + io->stopRecording(); + io->close(); + // [example_timeintervals_colbased_stop_snippet] + } +} \ No newline at end of file diff --git a/tests/examples/testWorkflowExamples.cpp b/tests/examples/testWorkflowExamples.cpp index ec7135443..a1bb343d9 100644 --- a/tests/examples/testWorkflowExamples.cpp +++ b/tests/examples/testWorkflowExamples.cpp @@ -13,6 +13,7 @@ #include "nwb/NWBFile.hpp" #include "nwb/ecephys/ElectricalSeries.hpp" #include "nwb/file/ElectrodesTable.hpp" +#include "nwb/file/Subject.hpp" #include "testUtils.hpp" using namespace AQNWB; @@ -46,7 +47,21 @@ TEST_CASE("workflowExamples") // [example_workflow_nwbfile_snippet] auto nwbfile = NWB::NWBFile::create(io); - Status initStatus = nwbfile->initialize(generateUuid()); + auto subjectSpec = + AQNWB::NWB::Subject::SubjectSpec() + .withSubjectId("mouse001") + .withSpecies("Mus musculus") + .withSex("M") + .withAge("P90D") + .withDescription( + "Wild type mouse used for electrophysiology study"); + std::string currentTime = getCurrentTime(); + Status initStatus = nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); AQNWB::checkStatus(initStatus, "NWBFile initialization"); // [example_workflow_nwbfile_snippet] REQUIRE(initStatus == Status::Success); diff --git a/tests/examples/test_ecephys_data_read.cpp b/tests/examples/test_ecephys_data_read.cpp index f81166b41..5517f05ef 100644 --- a/tests/examples/test_ecephys_data_read.cpp +++ b/tests/examples/test_ecephys_data_read.cpp @@ -9,6 +9,8 @@ # define HAVE_MDSPAN 0 #endif +#include + #include #include #include @@ -115,6 +117,23 @@ TEST_CASE("ElectricalSeriesReadExample", "[ecephys]") mockArrays, mockChannelNames, dataType, containerIndexes); REQUIRE(resultCreate == Status::Success); + // create InvalidTimes table for testing DynamicTable::readRows + auto invalidTimesTable = nwbfile->createInvalidTimes(true); + REQUIRE(invalidTimesTable != nullptr); + std::vector invalidTimesRows = { + {{"start_time", 1.0f}, + {"stop_time", 2.0f}, + {"tags", std::vector {"device_error", "user_error"}}}, + {{"start_time", 3.0f}, + {"stop_time", 4.0f}, + {"tags", std::vector {"external_interference"}}}, + {{"start_time", 5.0f}, + {"stop_time", 6.0f}, + {"tags", + std::vector { + "lost_connection", "external_interference", "user_error"}}}}; + REQUIRE(invalidTimesTable->addRows(invalidTimesRows) == Status::Success); + // get the new ElectricalSeries auto recordingObjects = io->getRecordingObjects(); auto registeredTypePtr = @@ -394,6 +413,71 @@ TEST_CASE("ElectricalSeriesReadExample", "[ecephys]") REQUIRE(meanFromVariant == Catch::Approx(meanFromTypedVector)); // [example_use_std_variant_to_compute_on_data] + // [example_read_dynamic_table_rows_setup_snippet] + // Read the InvalidTimes table + auto readInvalidTimesTable = readNWBFile->readInvalidTimes(); + REQUIRE(readInvalidTimesTable != nullptr); + + // Standard approach: Read a single column's data + // This is efficient when you only need data from one or a few columns + auto startTimeColumn = readInvalidTimesTable->readStartTime(); + auto startTimeData = startTimeColumn->readData()->values(); + REQUIRE(startTimeData.data.size() == 3); + // [example_read_dynamic_table_rows_setup_snippet] + + // [example_read_dynamic_table_rows_all_cols_snippet] + // readRows approach: Read a slice of rows (e.g., start=0, count=2 -> rows 0 + // and 1) This is convenient when you need to process data row-by-row across + // multiple columns + auto rows = readInvalidTimesTable->readRows(0, 2); + REQUIRE(rows.size() == 2); + + // Access data from the rows + for (const auto& row : rows) { + // Implicit conversion to float + float startTime = row.at("start_time"); + // Explicit conversion to float + float stopTime = row.at("stop_time").get(); + // Access ragged array data (tags) with checks for type safety + std::vector tags; + if (row.at("tags").holds_alternative>()) { + tags = row.at("tags").get>(); + } else { + throw std::runtime_error("Expected 'tags' to be a vector of strings"); + } + // Process the data... + std::cout << "Row start_time: " << startTime + << ", stop_time: " << stopTime << ", tags: "; + for (const auto& tag : tags) { + std::cout << tag << " "; + } + std::cout << std::endl; + } + // [example_read_dynamic_table_rows_all_cols_snippet] + + // [example_read_dynamic_table_rows_specific_cols_snippet] + // Read specific columns for a slice of rows + auto specificRows = + readInvalidTimesTable->readRows(0, 2, {"start_time", "tags"}); + REQUIRE(specificRows.size() == 2); + REQUIRE(specificRows[0].find("stop_time") == specificRows[0].end()); + REQUIRE(specificRows[0].find("tags") != specificRows[0].end()); + // [example_read_dynamic_table_rows_specific_cols_snippet] + + // [example_read_dynamic_table_to_string_snippet] + // Print the contents of the table + std::string tableString = readInvalidTimesTable->toString(); + std::cout << "InvalidTimes table contents:\n" << tableString << std::endl; + // [example_read_dynamic_table_to_string_snippet] + + std::string expectedTableString = + "id,start_time,stop_time,tags\n" + "0,1.000000,2.000000,\"[device_error, user_error]\"\n" + "1,3.000000,4.000000,\"[external_interference]\"\n" + "2,5.000000,6.000000,\"[lost_connection, external_interference, user_" + "error]\"\n"; + REQUIRE(tableString == expectedTableString); + // Close the io readio->close(); } diff --git a/tests/examples/test_link_timeseries_example.cpp b/tests/examples/test_link_timeseries_example.cpp index c494e5a4d..ae7d87415 100644 --- a/tests/examples/test_link_timeseries_example.cpp +++ b/tests/examples/test_link_timeseries_example.cpp @@ -21,7 +21,12 @@ TEST_CASE("LinkTimeSeriesExamples", "[timeseries][link]") io->open(); auto nwbfile = NWB::NWBFile::create(io); - auto status = nwbfile->initialize(generateUuid()); + auto status = nwbfile->initialize(generateUuid(), + "Test linked time series", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); REQUIRE(status == Status::Success); // [example_link_timeseries_setup] diff --git a/tests/testBaseIO.cpp b/tests/testBaseIO.cpp index 041bdf710..3e84795ca 100644 --- a/tests/testBaseIO.cpp +++ b/tests/testBaseIO.cpp @@ -81,6 +81,42 @@ TEST_CASE("BaseDataType fromTypeId", "[BaseIO]") } } +TEST_CASE("BaseDataType value compatibility", "[BaseIO]") +{ + SECTION("Numeric values require an exact type match") + { + const BaseDataType dataType = BaseDataType::I32; + + REQUIRE( + dataType.isCompatibleCellValue(AQNWB::Types::CellValue(int32_t {1}))); + REQUIRE(dataType.isCompatibleCellValue( + AQNWB::Types::CellValue(std::vector {1, 2}))); + REQUIRE_FALSE( + dataType.isCompatibleCellValue(AQNWB::Types::CellValue(float {1.0f}))); + REQUIRE_FALSE(dataType.isCompatibleCellValue( + AQNWB::Types::CellValue(std::vector {1.0f, 2.0f}))); + + REQUIRE(dataType.isCompatibleVector(std::vector {1, 2})); + REQUIRE_FALSE(dataType.isCompatibleVector(std::vector {1.0f, 2.0f})); + } + + SECTION("String values support both string storage types") + { + const AQNWB::Types::CellValue scalarValue(std::string("value")); + const AQNWB::Types::CellValue vectorValue( + std::vector {"value1", "value2"}); + + REQUIRE(BaseDataType::DSTR.isCompatibleCellValue(scalarValue)); + REQUIRE(BaseDataType::DSTR.isCompatibleCellValue(vectorValue)); + REQUIRE(BaseDataType::STR(8).isCompatibleCellValue(scalarValue)); + REQUIRE(BaseDataType::STR(8).isCompatibleCellValue(vectorValue)); + REQUIRE(BaseDataType::DSTR.isCompatibleVector( + std::vector {"value"})); + REQUIRE(BaseDataType::STR(8).isCompatibleVector( + std::vector {"value"})); + } +} + TEST_CASE("Test findTypes and getFullTypeName", "[BaseIO]") { std::string filename = getTestFilePath("test_findTypes.h5"); diff --git a/tests/testData.cpp b/tests/testData.cpp index 600c02742..045661740 100644 --- a/tests/testData.cpp +++ b/tests/testData.cpp @@ -4,8 +4,10 @@ #include "Types.hpp" #include "Utils.hpp" #include "io/BaseIO.hpp" +#include "io/RecordingObjects.hpp" #include "io/hdf5/HDF5RecordingData.hpp" #include "nwb/RegisteredType.hpp" +#include "nwb/base/NWBData.hpp" #include "nwb/hdmf/base/Data.hpp" #include "testUtils.hpp" @@ -101,6 +103,22 @@ TEST_CASE("Data", "[base]") TEST_CASE("DataTyped", "[base]") { + SECTION("typed data facades are not registered") + { + auto io = createIO("HDF5", getTestFilePath("testDataTypedFacade.h5")); + const std::string dataPath = "/typed_data"; + + auto dataTyped = NWB::DataTyped::create(dataPath, io); + REQUIRE(dataTyped != nullptr); + REQUIRE(io->getRecordingObjects()->getRecordingObject(dataPath) == nullptr); + + auto nwbData = NWB::NWBData::create("/nwb_data", io); + auto nwbDataTyped = NWB::NWBDataTyped::fromNWBData(*nwbData); + REQUIRE(nwbDataTyped != nullptr); + REQUIRE(io->getRecordingObjects()->getRecordingObject("/nwb_data") + == nwbData); + } + SECTION("test DataTyped write/read") { std::string path = getTestFilePath("testDataTypedInt.h5"); diff --git a/tests/testDurationVectorData.cpp b/tests/testDurationVectorData.cpp new file mode 100644 index 000000000..de744d6ac --- /dev/null +++ b/tests/testDurationVectorData.cpp @@ -0,0 +1,179 @@ +#include +#include + +#include "Types.hpp" +#include "io/BaseIO.hpp" +#include "nwb/RegisteredType.hpp" +#include "nwb/event/DurationVectorData.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("DurationVectorData", "[event]") +{ + SECTION( + "test DurationVectorData is registered as a subclass of RegisteredType") + { + auto registry = AQNWB::NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("core::DurationVectorData") != registry.end()); + } + + SECTION("test DurationVectorData initialize, write, and read") + { + std::string path = getTestFilePath("testDurationVectorData.h5"); + std::string dataPath = "/durations"; + std::string description = "Test duration data"; + float resolution = 1.0f / 30000.0f; + std::vector durations = {0.0f, 0.05f, 0.25f, 0.5f, 1.25f}; + SizeArray dataShape = {durations.size()}; + SizeArray chunking = {durations.size()}; + SizeArray positionOffset = {0}; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto durationVectorData = + AQNWB::NWB::DurationVectorData::create(dataPath, io); + REQUIRE(durationVectorData != nullptr); + + Status initStatus = + durationVectorData->initialize(config, description, resolution); + REQUIRE(initStatus == Status::Success); + + auto dataRecorder = durationVectorData->recordData(); + REQUIRE(dataRecorder != nullptr); + + Status writeStatus = dataRecorder->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, durations.data()); + REQUIRE(writeStatus == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + REQUIRE(readDataUntyped != nullptr); + + auto readDurationVectorData = + std::dynamic_pointer_cast( + readDataUntyped); + REQUIRE(readDurationVectorData != nullptr); + + REQUIRE(readDurationVectorData->getNamespace() == "core"); + REQUIRE(readDurationVectorData->getTypeName() == "DurationVectorData"); + + auto namespaceData = readDurationVectorData->readNamespace(); + REQUIRE(namespaceData->values().data[0] == "core"); + + auto neurodataTypeData = readDurationVectorData->readNeurodataType(); + REQUIRE(neurodataTypeData->values().data[0] == "DurationVectorData"); + + auto descriptionData = readDurationVectorData->readDescription(); + REQUIRE(descriptionData->values().data[0] == description); + + auto unitData = readDurationVectorData->readUnit(); + REQUIRE(unitData->values().data[0] == "seconds"); + + auto resolutionData = readDurationVectorData->readResolution(); + REQUIRE(resolutionData->values().data[0] == Catch::Approx(resolution)); + + auto durationData = readDurationVectorData->readData(); + auto writtenValues = durationData->values().data; + REQUIRE(writtenValues.size() == durations.size()); + for (size_t i = 0; i < writtenValues.size(); ++i) { + REQUIRE(writtenValues[i] == Catch::Approx(durations[i])); + } + + io->close(); + } + } + + SECTION("test DurationVectorData findOwnedTypes") + { + std::string path = getTestFilePath("testDurationVectorDataOwnedTypes.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + SizeArray dataShape = {4}; + SizeArray chunking = {4}; + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto durationVectorData = + AQNWB::NWB::DurationVectorData::create("/durations", io); + REQUIRE(durationVectorData != nullptr); + + Status initStatus = + durationVectorData->initialize(config, "Owned types test", 0.001f); + REQUIRE(initStatus == Status::Success); + + auto ownedTypes = durationVectorData->findOwnedTypes(); + REQUIRE(ownedTypes.empty()); + + io->close(); + } + + SECTION("test DurationVectorData initialize with optional resolution") + { + std::string path = getTestFilePath("testDurationVectorDataOptional.h5"); + std::string dataPath = "/durations_optional"; + std::vector durations = {0.0f, 0.05f, 0.25f}; + SizeArray dataShape = {durations.size()}; + SizeArray chunking = {durations.size()}; + + { + std::string description = "Test duration data optional"; + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto durationVectorData = + AQNWB::NWB::DurationVectorData::create(dataPath, io); + REQUIRE(durationVectorData != nullptr); + + Status initStatus = + durationVectorData->initialize(config, description, std::nullopt); + REQUIRE(initStatus == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + REQUIRE(readDataUntyped != nullptr); + + auto readDurationVectorData = + std::dynamic_pointer_cast( + readDataUntyped); + REQUIRE(readDurationVectorData != nullptr); + + auto resolutionData = readDurationVectorData->readResolution(); + REQUIRE(resolutionData->exists() == false); + + io->close(); + } + } + + SECTION("test DurationVectorData initialize fails after IO deletion") + { + SizeArray dataShape = {3}; + SizeArray chunking = {3}; + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + + auto io = + createIO("HDF5", getTestFilePath("testDurationVectorDataNoIO.h5")); + auto durationVectorData = + AQNWB::NWB::DurationVectorData::create("/durations", io); + REQUIRE(durationVectorData != nullptr); + + io.reset(); + + Status initStatus = + durationVectorData->initialize(config, "Missing IO", 0.01f); + REQUIRE(initStatus == Status::Failure); + } +} diff --git a/tests/testDynamicTable.cpp b/tests/testDynamicTable.cpp index b3114c2e4..e7d036400 100644 --- a/tests/testDynamicTable.cpp +++ b/tests/testDynamicTable.cpp @@ -2,7 +2,12 @@ #include #include "io/hdf5/HDF5IO.hpp" +#include "nwb/event/DurationVectorData.hpp" +#include "nwb/event/TimestampVectorData.hpp" #include "nwb/hdmf/table/DynamicTable.hpp" +#include "nwb/hdmf/table/ElementIdentifiers.hpp" +#include "nwb/hdmf/table/MeaningsTable.hpp" +#include "nwb/hdmf/table/VectorIndex.hpp" #include "testUtils.hpp" using namespace AQNWB; @@ -17,6 +22,19 @@ TEST_CASE("DynamicTable", "[table]") REQUIRE(registry.find("hdmf-common::DynamicTable") != registry.end()); } + SECTION("test createDefaultDataSpecs") + { + auto specs = NWB::DynamicTable::createDefaultDataSpecs(50); + REQUIRE(specs.size() == 1); + REQUIRE(specs[0]->name == "id"); + REQUIRE(specs[0]->getType() == IO::BaseDataType::I32); + REQUIRE(specs[0]->getChunking() == SizeArray {50}); + auto idSpec = + std::dynamic_pointer_cast>( + specs[0]); + REQUIRE(idSpec != nullptr); + } + SECTION("test initialization and column names") { std::string path = getTestFilePath("testDynamicTable.h5"); @@ -31,16 +49,87 @@ TEST_CASE("DynamicTable", "[table]") auto readDesc = table->readDescription()->values().data; REQUIRE(readDesc[0] == "A test dynamic table"); - // Test setting and reading column names - std::vector colNames = {"col1", "col2", "col3"}; + // Add columns so the table tracks them before reordering colnames + IO::ArrayDataSetConfig emptyConfig(BaseDataType::V_STR, {0}, {3}); + auto col1 = NWB::VectorData::create(mergePaths(tablePath, "col1"), io); + auto col2 = NWB::VectorData::create(mergePaths(tablePath, "col2"), io); + auto col3 = NWB::VectorData::create(mergePaths(tablePath, "col3"), io); + REQUIRE(col1->initialize(emptyConfig, "Column 1") == Status::Success); + REQUIRE(col2->initialize(emptyConfig, "Column 2") == Status::Success); + REQUIRE(col3->initialize(emptyConfig, "Column 3") == Status::Success); + REQUIRE(table->addColumn(col1) == Status::Success); + REQUIRE(table->addColumn(col2) == Status::Success); + REQUIRE(table->addColumn(col3) == Status::Success); + + // Adding columns should flush colnames immediately. + auto readInitialColNames = table->readColNames()->values().data; + REQUIRE(readInitialColNames + == std::vector({"col1", "col2", "col3"})); + + // Test reordering and reading column names + std::vector colNames = {"col3", "col2", "col1"}; table->setColNames(colNames); - status = table->finalize(); - REQUIRE(status == Status::Success); auto readColNames = table->readColNames()->values().data; REQUIRE(readColNames == colNames); io->close(); + + // Verify colnames persist without relying on finalize() + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + auto reopenedColNames = readTable->readColNames()->values().data; + REQUIRE(reopenedColNames == colNames); + io->close(); + } + + SECTION("test addColumn with DataSpecPtr") + { + std::string path = getTestFilePath("testDynamicTableAddColumnDataSpec.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table with DataSpec column"); + REQUIRE(status == Status::Success); + + // Create a DataSpec for a new column + auto colSpec = NWB::VectorData::createDataSpec( + "col_spec", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "Column added via DataSpec"); + + // Add the column using the DataSpecPtr overload + status = table->addColumn(colSpec); + REQUIRE(status == Status::Success); + + // Verify the column name was registered + auto colNames = table->readColNames()->values().data; + REQUIRE(colNames == std::vector({"col_spec"})); + + // Add a row to verify the column is configured and writable + AQNWB::Types::RowData row = {{"col_spec", std::string("test_value")}}; + status = table->addRow(row); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + + auto readCol = readTable->readColumn("col_spec"); + REQUIRE(readCol != nullptr); + auto readData = readCol->readData()->valuesGeneric(); + auto readTyped = DataBlock::fromGeneric(readData); + REQUIRE(readTyped.data == std::vector({"test_value"})); + + io->close(); } SECTION("test adding columns and row IDs") @@ -65,12 +154,7 @@ TEST_CASE("DynamicTable", "[table]") // Set row IDs std::vector ids = {1, 2, 3}; - SizeArray idShape = {ids.size()}; - SizeArray idChunking = {ids.size()}; - IO::ArrayDataSetConfig idConfig(BaseDataType::I32, idShape, idChunking); - auto elementIDs = NWB::ElementIdentifiers::create(tablePath + "/id", io); - elementIDs->initialize(idConfig); - status = table->setRowIDs(elementIDs, ids); + status = table->setRowIDs(ids); REQUIRE(status == Status::Success); // Finalize table @@ -167,6 +251,988 @@ TEST_CASE("DynamicTable", "[table]") } } + SECTION("test setColNames rejects non-permutations") + { + std::string path = getTestFilePath("testDynamicTableInvalidColNames.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table with invalid colname reorder"); + REQUIRE(status == Status::Success); + + IO::ArrayDataSetConfig emptyConfig(BaseDataType::V_STR, {0}, {2}); + auto col1 = NWB::VectorData::create(mergePaths(tablePath, "col1"), io); + auto col2 = NWB::VectorData::create(mergePaths(tablePath, "col2"), io); + REQUIRE(col1->initialize(emptyConfig, "Column 1") == Status::Success); + REQUIRE(col2->initialize(emptyConfig, "Column 2") == Status::Success); + REQUIRE(table->addColumn(col1) == Status::Success); + REQUIRE(table->addColumn(col2) == Status::Success); + + auto initialColNames = table->readColNames()->values().data; + REQUIRE(initialColNames == std::vector({"col1", "col2"})); + + REQUIRE_THROWS_AS(table->setColNames({"col1"}), std::invalid_argument); + REQUIRE_THROWS_AS(table->setColNames({"col1", "col3"}), + std::invalid_argument); + + auto unchangedColNames = table->readColNames()->values().data; + REQUIRE(unchangedColNames == initialColNames); + + io->close(); + } + + SECTION("test DynamicTable validation") + { + std::string path = getTestFilePath("testDynamicTableValidation.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // 1. Valid id spec + std::vector validSpecs = { + NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))}; + REQUIRE(table->validateDataSpecs(validSpecs) == Status::Success); + + // 2. Invalid id specs: wrong DataSpec type, shape, and data type. + std::vector missingIdSpecs = { + NWB::VectorData::createDataSpec( + "col1", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}), + "column 1")}; + std::vector wrongIdSpecType = { + NWB::VectorData::createDataSpec( + "id", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "id")}; + std::vector wrongIdShape = { + NWB::ElementIdentifiers::createDataSpec( + "id", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0, 1}, {1, 1}))}; + std::vector wrongIdDataType = { + NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}))}; + + REQUIRE(table->validateDataSpecs(missingIdSpecs) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongIdSpecType) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongIdShape) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongIdDataType) == Status::Failure); + + REQUIRE_THROWS_AS(table->initialize("Test Table", wrongIdSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize("Test Table", wrongIdShape), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize("Test Table", wrongIdDataType), + std::invalid_argument); + + io->close(); + } + + // This test section tests support for derived types of VectorData as columns + // in DynamicTable, specifically TimestampVectorData and DurationVectorData. + // It creates a DynamicTable with these columns, writes data to them, and then + // reads back the data via DynamicTable.readColumn to verify correctness. + SECTION("test adding TimestampVectorData and DurationVectorData columns") + { + std::string path = getTestFilePath("testDynamicTableEventColumns.h5"); + std::vector timestamps = {0.1f, 0.4f, 0.9f}; + std::vector durations = {0.05f, 0.2f, 0.15f}; + std::vector ids = {1, 2, 3}; + SizeArray dataShape = {timestamps.size()}; + SizeArray chunking = {timestamps.size()}; + SizeArray positionOffset = {0}; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table with event columns"); + REQUIRE(status == Status::Success); + + IO::ArrayDataSetConfig eventConfig( + BaseDataType::F32, dataShape, chunking); + auto timestampColumn = AQNWB::NWB::TimestampVectorData::create( + mergePaths(tablePath, "timestamp"), io); + REQUIRE(timestampColumn != nullptr); + status = timestampColumn->initialize( + eventConfig, "Timestamp column for event table", 0.001f); + REQUIRE(status == Status::Success); + status = timestampColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + REQUIRE(status == Status::Success); + + auto durationColumn = AQNWB::NWB::DurationVectorData::create( + mergePaths(tablePath, "duration"), io); + REQUIRE(durationColumn != nullptr); + status = durationColumn->initialize( + eventConfig, "Duration column for event table", 0.001f); + REQUIRE(status == Status::Success); + status = durationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, durations.data()); + REQUIRE(status == Status::Success); + + status = table->setRowIDs(ids); + REQUIRE(status == Status::Success); + + status = table->addColumn(timestampColumn); + REQUIRE(status == Status::Success); + + status = table->addColumn(durationColumn); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + auto colNames = table->readColNames()->values().data; + REQUIRE(colNames == std::vector({"timestamp", "duration"})); + + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readTable = NWB::DynamicTable::create(tablePath, io); + auto readColNames = readTable->readColNames()->values().data; + REQUIRE(readColNames + == std::vector({"timestamp", "duration"})); + + auto readTimestampColumn = + readTable->readColumn("timestamp"); + REQUIRE(readTimestampColumn != nullptr); + REQUIRE(readTimestampColumn->readUnit()->values().data[0] == "seconds"); + REQUIRE(readTimestampColumn->readResolution()->values().data[0] + == Catch::Approx(0.001f)); + auto readTimestampValues = readTimestampColumn->readData()->values().data; + REQUIRE(readTimestampValues.size() == timestamps.size()); + for (size_t i = 0; i < readTimestampValues.size(); ++i) { + REQUIRE(readTimestampValues[i] == Catch::Approx(timestamps[i])); + } + + auto readDurationColumn = + readTable->readColumn("duration"); + REQUIRE(readDurationColumn != nullptr); + REQUIRE(readDurationColumn->readUnit()->values().data[0] == "seconds"); + REQUIRE(readDurationColumn->readResolution()->values().data[0] + == Catch::Approx(0.001f)); + auto readDurationValues = readDurationColumn->readData()->values().data; + REQUIRE(readDurationValues.size() == durations.size()); + for (size_t i = 0; i < readDurationValues.size(); ++i) { + REQUIRE(readDurationValues[i] == Catch::Approx(durations[i])); + } + + auto readIds = readTable->readIdColumn()->readData()->values().data; + REQUIRE(readIds == ids); + + io->close(); + } + } + + SECTION("test createMeaningsTable and readMeaningsTable") + { + std::string path = getTestFilePath("testDynamicTableMeanings.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table with meanings"); + REQUIRE(status == Status::Success); + + // Add string vector data column + std::vector values = {"value1", "value2", "value3"}; + SizeArray dataShape = {values.size()}; + SizeArray chunking = {values.size()}; + IO::ArrayDataSetConfig strConfig(BaseDataType::V_STR, dataShape, chunking); + std::string columnPath = mergePaths(tablePath, "col1"); + auto vectorData = NWB::VectorData::create(columnPath, io); + vectorData->initialize(strConfig, "Column 1"); + status = table->addColumn(vectorData, values); + REQUIRE(status == Status::Success); + + // Create MeaningsTable + auto meaningsTable = table->createMeaningsTable("col1"); + REQUIRE(meaningsTable != nullptr); + + // Add values to MeaningsTable + std::vector meaningValues = {"value1", "value2", "value3"}; + std::vector meanings = {"meaning1", "meaning2", "meaning3"}; + + auto valueCol = meaningsTable->readValueColumn(); + REQUIRE(valueCol != nullptr); + status = + valueCol->recordData()->writeDataBlock(SizeArray {meaningValues.size()}, + SizeArray {0}, + BaseDataType::V_STR, + meaningValues); + REQUIRE(status == Status::Success); + + auto meaningCol = meaningsTable->readMeaningColumn(); + REQUIRE(meaningCol != nullptr); + status = + meaningCol->recordData()->writeDataBlock(SizeArray {meanings.size()}, + SizeArray {0}, + BaseDataType::V_STR, + meanings); + REQUIRE(status == Status::Success); + + std::vector meaningIds = {1, 2, 3}; + status = meaningsTable->setRowIDs(meaningIds); + REQUIRE(status == Status::Success); + + // Set row IDs for main table + std::vector ids = {1, 2, 3}; + status = table->setRowIDs(ids); + REQUIRE(status == Status::Success); + + // Finalize main table + io->stopRecording(); + io->close(); + + // Reopen and read + auto readio = createIO("HDF5", path); + readio->open(); + + auto readTable = NWB::DynamicTable::create(tablePath, readio); + + // Read MeaningsTable + auto readMeaningsTable = readTable->readMeaningsTable("col1_meanings"); + REQUIRE(readMeaningsTable != nullptr); + REQUIRE(readMeaningsTable->getPath() + == mergePaths(tablePath, "meanings_tables/col1_meanings")); + + // Test createMeaningsTableInstance + auto meaningsTableInstance = + readTable->createMeaningsTableInstance("col1_meanings"); + REQUIRE(meaningsTableInstance != nullptr); + REQUIRE(meaningsTableInstance->getPath() + == mergePaths(tablePath, "meanings_tables/col1_meanings")); + + // Read value column + auto readValueCol = readMeaningsTable->readValueColumn(); + REQUIRE(readValueCol != nullptr); + auto readValueColTyped = + AQNWB::NWB::VectorDataTyped::fromVectorData(readValueCol); + REQUIRE(readValueColTyped != nullptr); + auto readValueData = readValueColTyped->readData()->values().data; + REQUIRE(readValueData.size() == meaningValues.size()); + for (size_t i = 0; i < readValueData.size(); ++i) { + REQUIRE(readValueData[i] == values[i]); + } + + // Read meaning column + auto readMeaningCol = readMeaningsTable->readMeaningColumn(); + REQUIRE(readMeaningCol != nullptr); + auto readMeaningData = readMeaningCol->readData()->values().data; + REQUIRE(readMeaningData.size() == meanings.size()); + for (size_t i = 0; i < readMeaningData.size(); ++i) { + REQUIRE(readMeaningData[i] == meanings[i]); + } + + // Read target VectorData link + auto readTarget = readMeaningsTable->readTarget(); + REQUIRE(readTarget != nullptr); + auto readTargetTyped = + AQNWB::NWB::VectorDataTyped::fromVectorData(readTarget); + REQUIRE(readTargetTyped != nullptr); + auto readTargetValues = readTargetTyped->readData()->values().data; + REQUIRE(readTargetValues.size() == values.size()); + for (size_t i = 0; i < readTargetValues.size(); ++i) { + // Check that the target values from the link match the original values + REQUIRE(readTargetValues[i] == values[i]); + } + + // close + readio->close(); + } + + SECTION("test getNumberOfRows") + { + std::string path = getTestFilePath("testDynamicTableGetNumberOfRows.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Missing id column + REQUIRE(table->getNumberOfRows() == 0); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + + Status status = table->initialize("Table with rows", specs); + REQUIRE(status == Status::Success); + + // Initially 0 rows + REQUIRE(table->getNumberOfRows() == 0); + + // Vector-valued cells are valid only for VectorIndex target columns. + AQNWB::Types::RowData vectorValuedRow = { + {"col_str", std::vector {"row1", "row2"}}}; + REQUIRE(table->addRow(vectorValuedRow) == Status::Failure); + REQUIRE(table->getNumberOfRows() == 0); + + // Add single row + AQNWB::Types::RowData row1 = {{"col_str", std::string("row1")}}; + status = table->addRow(row1); + REQUIRE(status == Status::Success); + REQUIRE(table->getNumberOfRows() == 1); + + // Add multiple rows + std::vector rows = { + {{"col_str", std::string("row2")}}, {{"col_str", std::string("row3")}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + REQUIRE(table->getNumberOfRows() == 3); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + REQUIRE(readTable->getNumberOfRows() == 3); + + io->close(); + } + + SECTION("test row-wise append with explicit IDs") + { + std::string path = getTestFilePath("testDynamicTableExplicitIDs.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + + Status status = table->initialize("Table with explicit IDs", specs); + REQUIRE(status == Status::Success); + + // Add single row with explicit ID + AQNWB::Types::RowData row1 = {{"col_str", std::string("row1")}}; + status = table->addRow(row1, 100); + REQUIRE(status == Status::Success); + + // Add multiple rows with explicit IDs + std::vector rows = { + {{"col_str", std::string("row2")}}, {{"col_str", std::string("row3")}}}; + std::vector rowIds = {101, 102}; + status = table->addRows(rows, rowIds); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + + auto readIds = readTable->readIdColumn()->readData()->values().data; + REQUIRE(readIds == std::vector({100, 101, 102})); + + io->close(); + } + + SECTION("test row-wise append") + { + std::string path = getTestFilePath("testDynamicTableRows.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + specs.push_back(NWB::VectorData::createDataSpec( + "col_f32", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}), + "Float column")); + + Status status = table->initialize("Table with rows", specs); + REQUIRE(status == Status::Success); + + // Add single row + AQNWB::Types::RowData row1 = {{"col_str", std::string("row1")}, + {"col_f32", 1.5f}}; + status = table->addRow(row1); + REQUIRE(status == Status::Success); + + // Add multiple rows + std::vector rows = { + {{"col_str", std::string("row2")}, {"col_f32", 2.5f}}, + {{"col_str", std::string("row3")}, {"col_f32", 3.5f}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + + auto readColNames = readTable->readColNames()->values().data; + REQUIRE(readColNames == std::vector({"col_str", "col_f32"})); + + auto readIds = readTable->readIdColumn()->readData()->values().data; + REQUIRE(readIds == std::vector({0, 1, 2})); + + auto colStr = readTable->readColumn("col_str"); + auto colStrDataGeneric = colStr->readData()->valuesGeneric(); + auto colStrDataTyped = + DataBlock::fromGeneric(colStrDataGeneric); + auto colStrData = colStrDataTyped.data; + REQUIRE(colStrData.size() == 3); + REQUIRE(colStrData == std::vector({"row1", "row2", "row3"})); + + auto colF32 = readTable->readColumn("col_f32"); + auto colF32DataGeneric = colF32->readData()->valuesGeneric(); + auto colF32DataTyped = DataBlock::fromGeneric(colF32DataGeneric); + auto colF32Data = colF32DataTyped.data; + REQUIRE(colF32Data.size() == 3); + REQUIRE(colF32Data == std::vector({1.5f, 2.5f, 3.5f})); + + // Append a row after reopening + NWB::DynamicTable::RowData row4 = {{"col_str", std::string("row4")}, + {"col_f32", 4.5f}}; + status = readTable->addRow(row4); + REQUIRE(status == Status::Success); + + // Read back again to check appended row + auto readIdsNew = readTable->readIdColumn()->readData()->values().data; + REQUIRE(readIdsNew == std::vector({0, 1, 2, 3})); + + auto colStrNew = readTable->readColumn("col_str"); + auto colStrDataTypedNew = DataBlock::fromGeneric( + colStrNew->readData()->valuesGeneric()); + REQUIRE(colStrDataTypedNew.data + == std::vector({"row1", "row2", "row3", "row4"})); + + io->close(); + } + + SECTION("test addColumn with values configures column for addRow") + { + // Verify that addColumn(vectorData, values) registers the column in + // m_configuredColumns so that addRow/addRows can be used afterward + // without pre-configuring via specs. + std::string path = + getTestFilePath("testDynamicTableAddColumnWithValues.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table built with addColumn"); + REQUIRE(status == Status::Success); + + // Add a string column using addColumn(vectorData, values) + std::vector initialValues = {"a", "b", "c"}; + SizeArray dataShape = {initialValues.size()}; + SizeArray chunking = {10}; // chunked to allow append + IO::ArrayDataSetConfig config(BaseDataType::V_STR, SizeArray {0}, chunking); + auto col1 = NWB::VectorData::create(mergePaths(tablePath, "col1"), io); + col1->initialize(config, "Column 1"); + status = table->addColumn(col1, initialValues); + REQUIRE(status == Status::Success); + + // Now use addRow to append another row — this requires col1 to be in + // m_configuredColumns, which addColumn should have registered it into. + AQNWB::Types::RowData newRow = {{"col1", std::string("d")}}; + status = table->addRow(newRow); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify all 4 values are present + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + auto readCol = readTable->readColumn("col1"); + REQUIRE(readCol != nullptr); + auto readData = readCol->readData()->valuesGeneric(); + auto readTyped = DataBlock::fromGeneric(readData); + REQUIRE(readTyped.data == std::vector({"a", "b", "c", "d"})); + io->close(); + } + + SECTION("test addColumn without values configures column for addRow") + { + // Verify that addColumn(vectorData) (no values) registers the column in + // m_configuredColumns so that addRow/addRows can be used afterward. + std::string path = + getTestFilePath("testDynamicTableAddColumnConfigures.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table built with addColumn no values"); + REQUIRE(status == Status::Success); + + // Initialize and add a float column without writing data yet + SizeArray chunking = {10}; + IO::ArrayDataSetConfig config(BaseDataType::F32, SizeArray {0}, chunking); + auto col1 = NWB::VectorData::create(mergePaths(tablePath, "col1"), io); + col1->initialize(config, "Float column"); + status = table->addColumn(col1); + REQUIRE(status == Status::Success); + + // Use addRows to write data — requires col1 to be in m_configuredColumns + std::vector rows = { + {{"col1", 1.0f}}, {{"col1", 2.0f}}, {{"col1", 3.0f}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + auto readCol = readTable->readColumn("col1"); + REQUIRE(readCol != nullptr); + auto readData = readCol->readData()->valuesGeneric(); + auto readTyped = DataBlock::fromGeneric(readData); + REQUIRE(readTyped.data == std::vector({1.0f, 2.0f, 3.0f})); + io->close(); + } + + // TODO : Add row for reference columns is not yet working. + // This will require support for chunked reference columns + // and support for column configuration with reference columns. + /* + SECTION("test addReferenceColumn configures column for addRow") + { + // Verify that addReferenceColumn registers the column in + // m_configuredColumns so that addRow/addRows can be used afterward. + std::string path = + getTestFilePath("testDynamicTableAddReferenceColumn.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + Status status = table->initialize("Table with reference column"); + REQUIRE(status == Status::Success); + + // Add a reference column + std::vector refs = {"/path/to/obj1", "/path/to/obj2"}; + status = table->addReferenceColumn("group", "electrode group", refs); + REQUIRE(status == Status::Success); + + // Verify the column name was registered + auto colNames = table->readColNames()->values().data; + REQUIRE(colNames == std::vector({"group"})); + + io->close(); + } + */ + + SECTION("test ragged array columns with VectorIndex") + { + std::string path = getTestFilePath("testDynamicTableRaggedArray.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + + Status status = table->initialize("Table with ragged array", specs); + REQUIRE(status == Status::Success); + + // Add target column for ragged array + auto targetCol = + NWB::VectorData::create(mergePaths(tablePath, "ragged_data"), io); + targetCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "Target data for ragged array"); + status = table->addColumn(targetCol); + REQUIRE(status == Status::Success); + REQUIRE(table->readColNames()->values().data + == std::vector({"col_str", "ragged_data"})); + + // Add VectorIndex column + auto indexCol = NWB::VectorIndex::create( + mergePaths(tablePath, "ragged_data_index"), io); + indexCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::U32, {0}, {10}), + "Index for ragged array", + targetCol->getPath()); + status = table->addColumn(indexCol); + REQUIRE(status == Status::Success); + REQUIRE(table->readColNames()->values().data + == std::vector({"col_str", "ragged_data"})); + + // Add rows + std::vector rows = { + {{"col_str", std::string("row1")}, + {"ragged_data", std::vector {1, 2, 3}}}, + {{"col_str", std::string("row2")}, + {"ragged_data", std::vector {4, 5}}}, + {{"col_str", std::string("row3")}, + {"ragged_data", std::vector {6, 7, 8, 9}}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + + auto readColNames = readTable->readColNames()->values().data; + REQUIRE(readColNames + == std::vector({"col_str", "ragged_data"})); + + auto readIds = readTable->readIdColumn()->readData()->values().data; + REQUIRE(readIds == std::vector({0, 1, 2})); + + auto colStr = readTable->readColumn("col_str"); + auto colStrDataGeneric = colStr->readData()->valuesGeneric(); + auto colStrDataTyped = + DataBlock::fromGeneric(colStrDataGeneric); + auto colStrData = colStrDataTyped.data; + REQUIRE(colStrData.size() == 3); + REQUIRE(colStrData == std::vector({"row1", "row2", "row3"})); + + auto raggedDataCol = readTable->readColumn("ragged_data"); + auto raggedDataGeneric = raggedDataCol->readData()->valuesGeneric(); + auto raggedDataTyped = DataBlock::fromGeneric(raggedDataGeneric); + auto raggedData = raggedDataTyped.data; + REQUIRE(raggedData.size() == 9); + REQUIRE(raggedData == std::vector({1, 2, 3, 4, 5, 6, 7, 8, 9})); + + auto raggedIndexCol = + readTable->readColumn("ragged_data_index"); + auto raggedIndexGeneric = raggedIndexCol->readData()->valuesGeneric(); + auto raggedIndexTyped = + DataBlock::fromGeneric(raggedIndexGeneric); + auto raggedIndexData = raggedIndexTyped.data; + REQUIRE(raggedIndexData.size() == 3); + REQUIRE(raggedIndexData == std::vector({3, 5, 9})); + + io->close(); + } + + SECTION("test readRows") + { + std::string path = getTestFilePath("testDynamicTableReadRows.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + specs.push_back(NWB::VectorData::createDataSpec( + "col_f32", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}), + "Float column")); + + Status status = table->initialize("Table with rows", specs); + REQUIRE(status == Status::Success); + + // Add target column for ragged array + auto targetCol = + NWB::VectorData::create(mergePaths(tablePath, "ragged_data"), io); + targetCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "Target data for ragged array"); + status = table->addColumn(targetCol); + REQUIRE(status == Status::Success); + + // Add VectorIndex column + auto indexCol = NWB::VectorIndex::create( + mergePaths(tablePath, "ragged_data_index"), io); + indexCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::U32, {0}, {10}), + "Index for ragged array", + targetCol->getPath()); + status = table->addColumn(indexCol); + REQUIRE(status == Status::Success); + + // Add rows + std::vector rows = { + {{"col_str", std::string("row1")}, + {"col_f32", 1.5f}, + {"ragged_data", std::vector {1, 2, 3}}}, + {{"col_str", std::string("row2")}, + {"col_f32", 2.5f}, + {"ragged_data", std::vector {4, 5}}}, + {{"col_str", std::string("row3")}, + {"col_f32", 3.5f}, + {"ragged_data", std::vector {6, 7, 8, 9}}}, + {{"col_str", std::string("row4")}, + {"col_f32", 4.5f}, + {"ragged_data", std::vector {10}}}, + {{"col_str", std::string("row5")}, + {"col_f32", 5.5f}, + {"ragged_data", std::vector {11, 12}}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + io->close(); + + // Reopen and verify + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::DynamicTable::create(tablePath, io); + + // Read all rows + auto readAllRows = readTable->readRows(0, 5); + REQUIRE(readAllRows.size() == 5); + + // Read rows without ID + auto readRowsNoId = readTable->readRows(0, 1, {}, false); + REQUIRE(readRowsNoId.size() == 1); + REQUIRE(readRowsNoId[0].find("id") == readRowsNoId[0].end()); + REQUIRE(readRowsNoId[0].find("col_str") != readRowsNoId[0].end()); + + // Read out of bounds + REQUIRE_THROWS_AS(readTable->readRows(10, 5), std::invalid_argument); + + // Read with count exceeding available rows + auto readExceedingRows = readTable->readRows(3, 10); + REQUIRE(readExceedingRows.size() == 2); + + // Check row 1 + std::string r1_str = readAllRows[0].at("col_str"); + float r1_f32 = readAllRows[0].at("col_f32"); + std::vector r1_ragged = readAllRows[0].at("ragged_data"); + REQUIRE(r1_str == "row1"); + REQUIRE(r1_f32 == Catch::Approx(1.5f)); + REQUIRE(r1_ragged == std::vector({1, 2, 3})); + + // Check row 3 + std::string r3_str = readAllRows[2].at("col_str"); + float r3_f32 = readAllRows[2].at("col_f32"); + std::vector r3_ragged = readAllRows[2].at("ragged_data"); + REQUIRE(r3_str == "row3"); + REQUIRE(r3_f32 == Catch::Approx(3.5f)); + REQUIRE(r3_ragged == std::vector({6, 7, 8, 9})); + + // Read a slice of rows + auto readSliceRows = readTable->readRows(1, 3); + REQUIRE(readSliceRows.size() == 3); + + // Check row 2 (index 0 in slice) + std::string s0_str = readSliceRows[0].at("col_str"); + float s0_f32 = readSliceRows[0].at("col_f32"); + std::vector s0_ragged = readSliceRows[0].at("ragged_data"); + REQUIRE(s0_str == "row2"); + REQUIRE(s0_f32 == Catch::Approx(2.5f)); + REQUIRE(s0_ragged == std::vector({4, 5})); + + // Check row 4 (index 2 in slice) + std::string s2_str = readSliceRows[2].at("col_str"); + float s2_f32 = readSliceRows[2].at("col_f32"); + + // Read a slice of rows with specific columns + auto readSliceRowsCols = readTable->readRows(1, 3, {"col_str", "col_f32"}); + REQUIRE(readSliceRowsCols.size() == 3); + + // Check row 2 (index 0 in slice) + std::string s0_str_cols = readSliceRowsCols[0].at("col_str"); + float s0_f32_cols = readSliceRowsCols[0].at("col_f32"); + REQUIRE(s0_str_cols == "row2"); + REQUIRE(s0_f32_cols == Catch::Approx(2.5f)); + REQUIRE(readSliceRowsCols[0].find("ragged_data") + == readSliceRowsCols[0].end()); + + // Check row 4 (index 2 in slice) + std::string s2_str_cols = readSliceRowsCols[2].at("col_str"); + float s2_f32_cols = readSliceRowsCols[2].at("col_f32"); + REQUIRE(s2_str_cols == "row4"); + REQUIRE(s2_f32_cols == Catch::Approx(4.5f)); + REQUIRE(readSliceRowsCols[2].find("ragged_data") + == readSliceRowsCols[2].end()); + std::vector s2_ragged = readSliceRows[2].at("ragged_data"); + REQUIRE(s2_str == "row4"); + REQUIRE(s2_f32 == Catch::Approx(4.5f)); + REQUIRE(s2_ragged == std::vector({10})); + + io->close(); + } + + SECTION("test DynamicTable.toString") + { + std::string path = getTestFilePath("testDynamicTableToString.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + // Configure schema + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + specs.push_back(NWB::VectorData::createDataSpec( + "col_f32", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}), + "Float column")); + + Status status = table->initialize("Table for toString test", specs); + REQUIRE(status == Status::Success); + + // Add VectorData column for ragged array target + auto targetCol = + NWB::VectorData::create(mergePaths(tablePath, "ragged_data"), io); + targetCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "Target data for ragged array"); + status = table->addColumn(targetCol); + REQUIRE(status == Status::Success); + + // Add VectorIndex column + auto indexCol = NWB::VectorIndex::create( + mergePaths(tablePath, "ragged_data_index"), io); + indexCol->initialize( + IO::ArrayDataSetConfig(IO::BaseDataType::U32, {0}, {10}), + "Index for ragged array", + targetCol->getPath()); + status = table->addColumn(indexCol); + REQUIRE(status == Status::Success); + + // Add rows + std::vector rows = { + {{"col_str", std::string("row1")}, + {"col_f32", 1.5f}, + {"ragged_data", std::vector {1, 2, 3}}}, + {{"col_str", std::string("row2")}, + {"col_f32", 2.5f}, + {"ragged_data", std::vector {4, 5}}}}; + status = table->addRows(rows); + REQUIRE(status == Status::Success); + + status = table->finalize(); + REQUIRE(status == Status::Success); + + // Test toString + std::string tableStr = table->toString(); + std::string expectedTableStr = + "id,col_str,col_f32,ragged_data\n" + "0,row1,1.500000,\"[1, 2, 3]\"\n" + "1,row2,2.500000,\"[4, 5]\"\n"; + REQUIRE(tableStr == expectedTableStr); + + // Test toString with specific rows + std::string rowStr = table->toString(1, 1); + std::string expectedRowStr = + "id,col_str,col_f32,ragged_data\n" + "1,row2,2.500000,\"[4, 5]\"\n"; + REQUIRE(rowStr == expectedRowStr); + + // Test toString with specific columns + std::string colStr = + table->toString(0, Types::SizeTypeNotSet, {"col_str"}, false); + std::string expectedColStr = + "col_str\n" + "row1\n" + "row2\n"; + REQUIRE(colStr == expectedColStr); + + io->close(); + } + + SECTION("test edge cases and error handling") + { + std::string path = getTestFilePath("testDynamicTableEdgeCases.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + + std::vector specs; + specs.push_back(NWB::ElementIdentifiers::createDataSpec( + "id", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}))); + specs.push_back(NWB::VectorData::createDataSpec( + "col_str", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10}), + "String column")); + + Status status = table->initialize("Table for edge cases", specs); + REQUIRE(status == Status::Success); + + // Test readColumn with non-existent column + auto nonExistentCol = table->readColumn("non_existent"); + REQUIRE(nonExistentCol == nullptr); + + // Test addRow with missing required column + AQNWB::Types::RowData missingColRow = {{"wrong_col", std::string("val")}}; + REQUIRE(table->addRow(missingColRow) == Status::Failure); + + // Add a valid row so we can test readRows + AQNWB::Types::RowData validRow = {{"col_str", std::string("val")}}; + REQUIRE(table->addRow(validRow) == Status::Success); + + // Test readRows with non-existent column name + // It should just ignore the non-existent column and return the others + auto rowsWithInvalidCol = + table->readRows(0, 1, {"col_str", "non_existent"}); + REQUIRE(rowsWithInvalidCol.size() == 1); + REQUIRE(rowsWithInvalidCol[0].find("col_str") + != rowsWithInvalidCol[0].end()); + REQUIRE(rowsWithInvalidCol[0].find("non_existent") + == rowsWithInvalidCol[0].end()); + + io->close(); + } + SECTION("test DynamicTable.findOwnedTypes") { std::string path = getTestFilePath("testDynamicTableFindOwned.h5"); @@ -190,14 +1256,7 @@ TEST_CASE("DynamicTable", "[table]") // Set row IDs std::vector ids = {1, 2, 3}; - SizeArray idShape = {ids.size()}; - SizeArray idChunking = {ids.size()}; - - std::string idPath = mergePaths(tablePath, "id"); - IO::ArrayDataSetConfig i32Config(BaseDataType::I32, idShape, idChunking); - auto elementIDs = NWB::ElementIdentifiers::create(idPath, io); - elementIDs->initialize(i32Config); - status = table->setRowIDs(elementIDs, ids); + status = table->setRowIDs(ids); REQUIRE(status == Status::Success); // Final diff --git a/tests/testDynamicTableForwardDeclaredMeanings.cpp b/tests/testDynamicTableForwardDeclaredMeanings.cpp new file mode 100644 index 000000000..4435505e7 --- /dev/null +++ b/tests/testDynamicTableForwardDeclaredMeanings.cpp @@ -0,0 +1,56 @@ +#include + +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE( + "DynamicTable MeaningsTable accessors compile with forward declaration", + "[table]") +{ + const std::string path = + getTestFilePath("testDynamicTableForwardDeclaredMeanings.h5"); + const std::string tablePath = "/test_table"; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + REQUIRE(table != nullptr); + REQUIRE(table->initialize("Table with meanings") == Status::Success); + + const std::vector values = {"value1", "value2", "value3"}; + const SizeArray dataShape = {values.size()}; + const SizeArray chunking = {values.size()}; + IO::ArrayDataSetConfig strConfig(BaseDataType::V_STR, dataShape, chunking); + auto vectorData = + NWB::VectorData::create(mergePaths(tablePath, "col1"), io); + REQUIRE(vectorData != nullptr); + REQUIRE(vectorData->initialize(strConfig, "Column 1") == Status::Success); + REQUIRE(table->addColumn(vectorData, values) == Status::Success); + + auto meaningsTable = table->createMeaningsTable("col1"); + REQUIRE(meaningsTable != nullptr); + REQUIRE(io->objectExists( + mergePaths(tablePath, "meanings_tables/col1_meanings"))); + + io->stopRecording(); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::DynamicTable::create(tablePath, io); + REQUIRE(table != nullptr); + + auto meaningsTable = table->readMeaningsTable("col1_meanings"); + REQUIRE(meaningsTable != nullptr); + + io->close(); + } +} diff --git a/tests/testEcephys.cpp b/tests/testEcephys.cpp index b68f1b6fb..6504cbc30 100644 --- a/tests/testEcephys.cpp +++ b/tests/testEcephys.cpp @@ -68,7 +68,7 @@ TEST_CASE("ElectricalSeries", "[ecephys]") elecGroup->initialize("description", "unknown", device); auto elecTable = NWB::ElectrodesTable::create(io); - Status elecTableStatus = elecTable->initialize(); + Status elecTableStatus = elecTable->initialize("description"); REQUIRE(elecTableStatus == Status::Success); elecTable->addElectrodes(mockArrays[0]); elecTableStatus = elecTable->finalize(); @@ -90,7 +90,7 @@ TEST_CASE("ElectricalSeries", "[ecephys]") // Confirm that the electrode table is created correctly auto readColNames = elecTable->readColNames()->values().data; std::vector expectedColNames = { - "location", "group", "group_name"}; + "location", "group_name", "group"}; REQUIRE(readColNames == expectedColNames); // write channel data @@ -133,7 +133,7 @@ TEST_CASE("ElectricalSeries", "[ecephys]") // Confirm that the electrode table is created correctly auto readColNames = elecTable->readColNames()->values().data; std::vector expectedColNames = { - "location", "group", "group_name"}; + "location", "group_name", "group"}; REQUIRE(readColNames == expectedColNames); // write channel data in segments @@ -341,7 +341,7 @@ TEST_CASE("ElectricalSeries", "[ecephys]") // setup electrode table auto elecTable = NWB::ElectrodesTable::create(io); - elecTable->initialize(); + elecTable->initialize("description"); elecTable->addElectrodes(mockArraysElectrodes[0]); elecTable->finalize(); @@ -402,7 +402,7 @@ TEST_CASE("ElectricalSeries", "[ecephys]") // setup electrode table auto elecTable = NWB::ElectrodesTable::create(io); - elecTable->initialize(); + elecTable->initialize("description"); elecTable->addElectrodes(mockArraysElectrodes[0]); elecTable->finalize(); @@ -507,7 +507,7 @@ TEST_CASE("SpikeEventSeries", "[ecephys]") // setup electrode table, device, and electrode group auto elecTable = NWB::ElectrodesTable::create(io); - Status elecTableStatus = elecTable->initialize(); + Status elecTableStatus = elecTable->initialize("description"); REQUIRE(elecTableStatus == Status::Success); elecTable->addElectrodes(mockArrays[0]); elecTableStatus = elecTable->finalize(); @@ -574,7 +574,7 @@ TEST_CASE("SpikeEventSeries", "[ecephys]") // setup electrode table, device, and electrode group auto elecTable = NWB::ElectrodesTable::create(io); - Status elecTableStatus = elecTable->initialize(); + Status elecTableStatus = elecTable->initialize("description"); REQUIRE(elecTableStatus == Status::Success); elecTable->addElectrodes(mockArrays[0]); elecTableStatus = elecTable->finalize(); diff --git a/tests/testEventsTable.cpp b/tests/testEventsTable.cpp new file mode 100644 index 000000000..0bb308f5e --- /dev/null +++ b/tests/testEventsTable.cpp @@ -0,0 +1,305 @@ +#include +#include + +#include "Types.hpp" +#include "io/BaseIO.hpp" +#include "nwb/RegisteredType.hpp" +#include "nwb/event/DurationVectorData.hpp" +#include "nwb/event/EventsTable.hpp" +#include "nwb/event/TimestampVectorData.hpp" +#include "nwb/hdmf/table/ElementIdentifiers.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("EventsTable", "[event]") +{ + SECTION("test EventsTable is registered as a subclass of RegisteredType") + { + auto registry = AQNWB::NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("core::EventsTable") != registry.end()); + } + + SECTION("test EventsTable initialize, write, and read") + { + std::string path = getTestFilePath("testEventsTable.h5"); + std::string tablePath = "/events/test_events"; + std::string description = "Test events table"; + std::string sourceDescription = "Test source description"; + float timestampResolution = 1.0f / 30000.0f; + float durationResolution = 1.0f / 30000.0f; + + std::vector timestamps = {0.0f, 0.1f, 0.25f, 0.5f, 0.75f}; + std::vector durations = {0.01f, 0.02f, 0.03f, 0.04f, 0.05f}; + std::vector annotations = {"a", "b", "c", "d", "e"}; + std::vector ids = {1, 2, 3, 4, 5}; + + SizeArray dataShape = {timestamps.size()}; + SizeArray positionOffset = {0}; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + // Verify events group does not exist initially + REQUIRE(io->objectExists("/events") == false); + + auto eventsTable = AQNWB::NWB::EventsTable::create(tablePath, io); + REQUIRE(eventsTable != nullptr); + + auto specs = NWB::EventsTable::createDefaultDataSpecs( + timestampResolution, true, durationResolution, true, 100); + Status initStatus = + eventsTable->initialize(description, sourceDescription, specs); + REQUIRE(initStatus == Status::Success); + + // Verify events group was created + REQUIRE(io->objectExists("/events") == true); + + // Write timestamps + auto timestampColumn = eventsTable->readTimestampColumn(); + REQUIRE(timestampColumn != nullptr); + Status writeTimestampStatus = + timestampColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + REQUIRE(writeTimestampStatus == Status::Success); + + // Write durations + auto durationColumn = eventsTable->readDurationColumn(); + REQUIRE(durationColumn != nullptr); + Status writeDurationStatus = durationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, durations.data()); + REQUIRE(writeDurationStatus == Status::Success); + + // Write annotations + auto annotationColumn = eventsTable->readAnnotationColumn(); + REQUIRE(annotationColumn != nullptr); + Status writeAnnotationStatus = + annotationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::V_STR, annotations); + REQUIRE(writeAnnotationStatus == Status::Success); + + // Set row IDs + Status setRowIDsStatus = eventsTable->setRowIDs(ids); + REQUIRE(setRowIDsStatus == Status::Success); + + Status finalizeStatus = eventsTable->finalize(); + REQUIRE(finalizeStatus == Status::Success); + + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readDataUntyped = NWB::RegisteredType::create(tablePath, io); + REQUIRE(readDataUntyped != nullptr); + + auto readEventsTable = + std::dynamic_pointer_cast(readDataUntyped); + REQUIRE(readEventsTable != nullptr); + + REQUIRE(readEventsTable->getNamespace() == "core"); + REQUIRE(readEventsTable->getTypeName() == "EventsTable"); + + auto descriptionData = readEventsTable->readDescription(); + REQUIRE(descriptionData->values().data[0] == description); + + auto sourceDescriptionData = readEventsTable->readSourceDescription(); + REQUIRE(sourceDescriptionData->values().data[0] == sourceDescription); + + // readRows resolves and caches columns before the typed accessors below. + const auto rows = readEventsTable->readRows(); + REQUIRE(rows.size() == timestamps.size()); + + // Read timestamps + auto readTimestampColumn = readEventsTable->readTimestampColumn(); + REQUIRE(readTimestampColumn != nullptr); + auto readTimestampValues = readTimestampColumn->readData()->values().data; + REQUIRE(readTimestampValues.size() == timestamps.size()); + for (size_t i = 0; i < readTimestampValues.size(); ++i) { + REQUIRE(readTimestampValues[i] == Catch::Approx(timestamps[i])); + } + + // Read durations + auto readDurationColumn = readEventsTable->readDurationColumn(); + REQUIRE(readDurationColumn != nullptr); + auto readDurationValues = readDurationColumn->readData()->values().data; + REQUIRE(readDurationValues.size() == durations.size()); + for (size_t i = 0; i < readDurationValues.size(); ++i) { + REQUIRE(readDurationValues[i] == Catch::Approx(durations[i])); + } + + // Read annotations + auto readAnnotationColumn = readEventsTable->readAnnotationColumn(); + REQUIRE(readAnnotationColumn != nullptr); + auto readAnnotationColumnTyped = + AQNWB::NWB::VectorDataTyped::fromVectorData( + readAnnotationColumn); + REQUIRE(readAnnotationColumnTyped != nullptr); + auto readAnnotationValues = + readAnnotationColumnTyped->readData()->values().data; + REQUIRE(readAnnotationValues.size() == annotations.size()); + for (size_t i = 0; i < readAnnotationValues.size(); ++i) { + REQUIRE(readAnnotationValues[i] == annotations[i]); + } + + // Read row IDs + auto readIdsData = + readEventsTable->readIdColumn()->readData()->values().data; + REQUIRE(readIdsData == ids); + + io->close(); + } + } + + SECTION("test EventsTable initialize with optional parameters") + { + std::string path = getTestFilePath("testEventsTableOptional.h5"); + std::string tablePath = "/events/test_events_optional"; + std::string description = "Test events table optional"; + + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto eventsTable = AQNWB::NWB::EventsTable::create(tablePath, io); + REQUIRE(eventsTable != nullptr); + + // Use std::nullopt for resolutions and sourceDescription + auto specs = NWB::EventsTable::createDefaultDataSpecs( + std::nullopt, false, std::nullopt, false, 100); + Status initStatus = + eventsTable->initialize(description, std::nullopt, specs); + REQUIRE(initStatus == Status::Success); + + // Verify timestamp column exists but has no resolution attribute + auto timestampColumn = eventsTable->readTimestampColumn(); + REQUIRE(timestampColumn != nullptr); + auto timestampRes = timestampColumn->readResolution(); + REQUIRE(timestampRes->exists() == false); // Should not exist + + // Verify duration column does not exist + auto durationColumn = eventsTable->readDurationColumn(); + REQUIRE(durationColumn == nullptr); + + // Verify source_description attribute does not exist + auto sourceDesc = eventsTable->readSourceDescription(); + REQUIRE(sourceDesc->exists() == false); + + io->close(); + } + + SECTION("test EventsTable initialize at non-events path") + { + std::string path = getTestFilePath("testEventsTableNonEventsPath.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + // Verify events group does not exist initially + REQUIRE(io->objectExists("/events") == false); + REQUIRE(io->createGroup("/events2") == Status::Success); + + auto eventsTable = + AQNWB::NWB::EventsTable::create("/events2/test_events", io); + REQUIRE(eventsTable != nullptr); + + auto specs = + NWB::EventsTable::createDefaultDataSpecs(0.01f, true, 0.01f, true, 100); + Status initStatus = eventsTable->initialize( + "Test events table", "Test source description", specs); + REQUIRE(initStatus == Status::Success); + + // Verify events group was NOT created + REQUIRE(io->objectExists("/events") == false); + + io->close(); + } + + SECTION("test EventsTable initialize fails after IO deletion") + { + auto io = createIO("HDF5", getTestFilePath("testEventsTableNoIO.h5")); + auto eventsTable = + AQNWB::NWB::EventsTable::create("/events/test_events", io); + REQUIRE(eventsTable != nullptr); + + io.reset(); + + auto specs = + NWB::EventsTable::createDefaultDataSpecs(0.01f, true, 0.01f, true, 100); + Status initStatus = eventsTable->initialize("Missing IO", "", specs); + REQUIRE(initStatus == Status::Failure); + } + + SECTION("test EventsTable validation") + { + std::string path = getTestFilePath("testEventsTableValidation.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto eventsTable = + AQNWB::NWB::EventsTable::create("/events/test_events", io); + + // 1. Valid default specs contain a TimestampVectorData timestamp column. + std::vector validSpecs = + NWB::EventsTable::createDefaultDataSpecs(); + REQUIRE(eventsTable->validateDataSpecs(validSpecs) == Status::Success); + + // 2. Timestamp must have the expected DataSpec type, shape, and dtype. + auto wrongTimestampSpecType = validSpecs; + wrongTimestampSpecType[1] = NWB::VectorData::createDataSpec( + "timestamp", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0}, {10}), + "timestamp"); + auto wrongTimestampShape = validSpecs; + wrongTimestampShape[1] = NWB::TimestampVectorData::createDataSpec( + "timestamp", + IO::ArrayDataSetConfig(IO::BaseDataType::F32, {0, 1}, {1, 1}), + "timestamp"); + auto wrongTimestampDataType = validSpecs; + wrongTimestampDataType[1] = NWB::TimestampVectorData::createDataSpec( + "timestamp", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "timestamp"); + + REQUIRE(eventsTable->validateDataSpecs(wrongTimestampSpecType) + == Status::Failure); + REQUIRE(eventsTable->validateDataSpecs(wrongTimestampShape) + == Status::Failure); + REQUIRE(eventsTable->validateDataSpecs(wrongTimestampDataType) + == Status::Failure); + + REQUIRE_THROWS_AS( + eventsTable->initialize("Test Events", "", wrongTimestampSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS( + eventsTable->initialize("Test Events", "", wrongTimestampShape), + std::invalid_argument); + REQUIRE_THROWS_AS( + eventsTable->initialize("Test Events", "", wrongTimestampDataType), + std::invalid_argument); + + io->close(); + } + + SECTION("test EventsTable initialize with defaulted columnSpecs") + { + std::string path = getTestFilePath("testEventsTableDefaultedSpecs.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto eventsTable = + AQNWB::NWB::EventsTable::create("/events/test_events", io); + REQUIRE(eventsTable != nullptr); + + // Call initialize with only description, allowing columnSpecs to default + Status initStatus = eventsTable->initialize("Test events"); + REQUIRE(initStatus == Status::Success); + + // Verify timestamp column was created + auto timestampColumn = eventsTable->readTimestampColumn(); + REQUIRE(timestampColumn != nullptr); + + io->close(); + } +} diff --git a/tests/testFile.cpp b/tests/testFile.cpp index 451a268c0..958a0e0d9 100644 --- a/tests/testFile.cpp +++ b/tests/testFile.cpp @@ -44,7 +44,7 @@ TEST_CASE("ElectrodesTable", "[ecephys]") // Confirm that the column names are created correctly auto readColNames = electrodeTable->readColNames()->values().data; std::vector expectedColNames = { - "location", "group", "group_name"}; + "location", "group_name", "group"}; REQUIRE(readColNames == expectedColNames); // Check if id datasets are created correctly @@ -120,6 +120,84 @@ TEST_CASE("ElectrodesTable", "[ecephys]") io->close(); } + SECTION("test ElectrodesTable validation") + { + std::string filename = getTestFilePath("electrodeTableValidation.h5"); + std::shared_ptr io = std::make_unique(filename); + io->open(); + + auto electrodeTable = NWB::ElectrodesTable::create(io); + + // 1. Valid default specs have required VectorData string columns. + std::vector validSpecs = + NWB::ElectrodesTable::createDefaultDataSpecs(); + REQUIRE(electrodeTable->validateDataSpecs(validSpecs) == Status::Success); + + // 2. Each required string column rejects the wrong DataSpec type, shape, + // and dtype. + auto wrongLocationSpecType = validSpecs; + wrongLocationSpecType[1] = NWB::ElementIdentifiers::createDataSpec( + "location", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10})); + auto wrongLocationShape = validSpecs; + wrongLocationShape[1] = NWB::VectorData::createDataSpec( + "location", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0, 1}, {1, 1}), + "location"); + auto wrongLocationDataType = validSpecs; + wrongLocationDataType[1] = NWB::VectorData::createDataSpec( + "location", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "location"); + + auto wrongGroupNameSpecType = validSpecs; + wrongGroupNameSpecType[2] = NWB::ElementIdentifiers::createDataSpec( + "group_name", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10})); + auto wrongGroupNameShape = validSpecs; + wrongGroupNameShape[2] = NWB::VectorData::createDataSpec( + "group_name", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0, 1}, {1, 1}), + "group_name"); + auto wrongGroupNameDataType = validSpecs; + wrongGroupNameDataType[2] = NWB::VectorData::createDataSpec( + "group_name", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "group_name"); + + REQUIRE(electrodeTable->validateDataSpecs(wrongLocationSpecType) + == Status::Failure); + REQUIRE(electrodeTable->validateDataSpecs(wrongLocationShape) + == Status::Failure); + REQUIRE(electrodeTable->validateDataSpecs(wrongLocationDataType) + == Status::Failure); + REQUIRE(electrodeTable->validateDataSpecs(wrongGroupNameSpecType) + == Status::Failure); + REQUIRE(electrodeTable->validateDataSpecs(wrongGroupNameShape) + == Status::Failure); + REQUIRE(electrodeTable->validateDataSpecs(wrongGroupNameDataType) + == Status::Failure); + + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongLocationSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongLocationShape), + std::invalid_argument); + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongLocationDataType), + std::invalid_argument); + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongGroupNameSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongGroupNameShape), + std::invalid_argument); + REQUIRE_THROWS_AS( + electrodeTable->initialize("Test Table", wrongGroupNameDataType), + std::invalid_argument); + + io->close(); + } + SECTION("test table creation with multiple arrays") { // TODO diff --git a/tests/testMeaningsTable.cpp b/tests/testMeaningsTable.cpp new file mode 100644 index 000000000..3db44a76d --- /dev/null +++ b/tests/testMeaningsTable.cpp @@ -0,0 +1,226 @@ +#include +#include + +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/hdmf/table/MeaningsTable.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("MeaningsTable", "[table]") +{ + std::string tablePath = "/test_meanings_table"; + std::string targetPath = "/test_target_vector_data"; + + SECTION("test MeaningsTable is registered as a subclass of RegisteredType") + { + auto registry = AQNWB::NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("hdmf-common::MeaningsTable") != registry.end()); + } + + SECTION("test initialization and column creation") + { + std::string path = getTestFilePath("testMeaningsTable.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + // Create target VectorData + auto targetVectorData = NWB::VectorData::create(targetPath, io); + IO::ArrayDataSetConfig targetConfig( + BaseDataType::I32, SizeArray {0}, SizeArray {100}); + Status targetStatus = + targetVectorData->initialize(targetConfig, "Target vector data"); + REQUIRE(targetStatus == Status::Success); + targetVectorData->finalize(); + + // Create MeaningsTable + auto table = NWB::MeaningsTable::create(tablePath, io); + auto specs = + NWB::MeaningsTable::createDefaultDataSpecs(BaseDataType::I32, 50); + Status status = table->initialize( + *targetVectorData, BaseDataType::I32, "A test meanings table", specs); + REQUIRE(status == Status::Success); + + // Finalize table to write attributes + Status finalizeStatus = table->finalize(); + REQUIRE(finalizeStatus == Status::Success); + + // Test reading description + auto readDesc = table->readDescription()->values().data; + REQUIRE(readDesc[0] == "A test meanings table"); + + // Test column names + auto readColNames = table->readColNames()->values().data; + std::vector expectedColNames = {"value", "meaning"}; + REQUIRE(readColNames == expectedColNames); + + // Test value column chunk size + auto chunking = io->getStorageObjectChunking(tablePath + "/value"); + REQUIRE(chunking.size() == 1); + REQUIRE(chunking[0] == 50); + + // Test meaning column chunk size + auto meaningChunking = io->getStorageObjectChunking(tablePath + "/meaning"); + REQUIRE(meaningChunking.size() == 1); + REQUIRE(meaningChunking[0] == 50); + + // Test target reference attribute + REQUIRE(io->attributeExists(tablePath + "/target")); + REQUIRE(io->getStorageObjectType(tablePath + "/target") + == StorageObjectType::Attribute); + + io->close(); + } + + SECTION("test adding and reading data") + { + std::string path = getTestFilePath("testMeaningsTableData.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + // Create target VectorData + auto targetVectorData = NWB::VectorData::create(targetPath, io); + IO::ArrayDataSetConfig targetConfig( + BaseDataType::I32, SizeArray {0}, SizeArray {100}); + targetVectorData->initialize(targetConfig, "Target vector data"); + targetVectorData->finalize(); + + // Create MeaningsTable + auto table = NWB::MeaningsTable::create(tablePath, io); + auto specs = NWB::MeaningsTable::createDefaultDataSpecs(BaseDataType::I32); + table->initialize(*targetVectorData, BaseDataType::I32, "", specs); + + // Add data to the table + std::vector values = {1, 2, 3}; + std::vector meanings = {"meaning 1", "meaning 2", "meaning 3"}; + + // Get the columns + auto valueCol = std::dynamic_pointer_cast>( + table->readColumn>("value")); + auto meaningCol = + table->readColumn>("meaning"); + + REQUIRE(valueCol != nullptr); + REQUIRE(meaningCol != nullptr); + + // Start recording to write data + io->startRecording(); + + // Write data + valueCol->recordData()->writeDataBlock(SizeArray {values.size()}, + SizeArray {0}, + BaseDataType::I32, + values.data()); + meaningCol->recordData()->writeDataBlock(SizeArray {meanings.size()}, + SizeArray {0}, + BaseDataType::V_STR, + meanings); + + // Stop the recording and finalize the table + io->stopRecording(); + io->close(); + + // Reopen file and verify data + io = createIO("HDF5", path); + io->open(); + auto readTable = NWB::MeaningsTable::create(tablePath, io); + + // Read values + auto readValueCol = readTable->readValueColumn(); + auto readValueColTyped = + NWB::VectorDataTyped::fromVectorData(readValueCol); + REQUIRE(readValueCol != nullptr); + REQUIRE(readValueColTyped != nullptr); + auto readValues = readValueColTyped->readData()->values().data; + REQUIRE(readValues == values); + + // Read meanings + auto readMeaningCol = readTable->readMeaningColumn(); + REQUIRE(readMeaningCol != nullptr); + auto readMeanings = readMeaningCol->readData()->values().data; + REQUIRE(readMeanings == meanings); + + io->close(); + } + + SECTION("test MeaningsTable validation") + { + std::string path = getTestFilePath("testMeaningsTableValidation.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto table = NWB::MeaningsTable::create(tablePath, io); + + // 1. Value preserves its configurable data type; meaning is VLEN string. + std::vector validSpecs = + NWB::MeaningsTable::createDefaultDataSpecs(IO::BaseDataType::I32); + REQUIRE(table->validateDataSpecs(validSpecs) == Status::Success); + REQUIRE(table->validateDataSpecs(NWB::MeaningsTable::createDefaultDataSpecs( + IO::BaseDataType::F32)) + == Status::Success); + + // 2. Value and meaning must use VectorData and be one-dimensional; meaning + // must additionally use VLEN string storage. + auto wrongValueSpecType = validSpecs; + wrongValueSpecType[1] = NWB::ElementIdentifiers::createDataSpec( + "value", IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10})); + auto wrongValueShape = validSpecs; + wrongValueShape[1] = NWB::VectorData::createDataSpec( + "value", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0, 1}, {1, 1}), + "value"); + auto wrongMeaningSpecType = validSpecs; + wrongMeaningSpecType[2] = NWB::ElementIdentifiers::createDataSpec( + "meaning", IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0}, {10})); + auto wrongMeaningShape = validSpecs; + wrongMeaningShape[2] = NWB::VectorData::createDataSpec( + "meaning", + IO::ArrayDataSetConfig(IO::BaseDataType::V_STR, {0, 1}, {1, 1}), + "meaning"); + auto wrongMeaningDataType = validSpecs; + wrongMeaningDataType[2] = NWB::VectorData::createDataSpec( + "meaning", + IO::ArrayDataSetConfig(IO::BaseDataType::I32, {0}, {10}), + "meaning"); + + REQUIRE(table->validateDataSpecs(wrongValueSpecType) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongValueShape) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongMeaningSpecType) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongMeaningShape) == Status::Failure); + REQUIRE(table->validateDataSpecs(wrongMeaningDataType) == Status::Failure); + + // 3. Invalid specifications cause initialization to reject the table. + auto targetVectorData = NWB::VectorData::create(targetPath, io); + targetVectorData->initialize( + IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), "target"); + + REQUIRE_THROWS_AS(table->initialize(*targetVectorData, + BaseDataType::I32, + "Test Meanings", + wrongValueSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize(*targetVectorData, + BaseDataType::I32, + "Test Meanings", + wrongValueShape), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize(*targetVectorData, + BaseDataType::I32, + "Test Meanings", + wrongMeaningSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize(*targetVectorData, + BaseDataType::I32, + "Test Meanings", + wrongMeaningShape), + std::invalid_argument); + REQUIRE_THROWS_AS(table->initialize(*targetVectorData, + BaseDataType::I32, + "Test Meanings", + wrongMeaningDataType), + std::invalid_argument); + + io->close(); + } +} diff --git a/tests/testNWBFile.cpp b/tests/testNWBFile.cpp index afeb61f39..e66d816c1 100644 --- a/tests/testNWBFile.cpp +++ b/tests/testNWBFile.cpp @@ -13,6 +13,8 @@ #include "nwb/NWBFile.hpp" #include "nwb/base/TimeSeries.hpp" #include "nwb/ecephys/SpikeEventSeries.hpp" +#include "nwb/file/Subject.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" #include "nwb/misc/AnnotationSeries.hpp" #include "spec/core.hpp" #include "testUtils.hpp" @@ -64,20 +66,343 @@ TEST_CASE("initialize", "[nwb]") REQUIRE(initStatus == Status::Failure); // check that regular init with current times works - initStatus = nwbfile->initialize(generateUuid()); + initStatus = nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); REQUIRE(initStatus == Status::Success); REQUIRE(nwbfile->isInitialized()); - // Since we didn't create any typed objects within the NWBFile, we should - // have no owned types + // Verify that events and intervals groups are NOT created by default + REQUIRE(io->objectExists("/events") == false); + REQUIRE(io->objectExists("/intervals") == false); + + // We created a Subject so we should have 1 owned type auto result = nwbfile->findOwnedTypes(); - REQUIRE(result.size() == 0); + REQUIRE(result.size() == 1); nwbfile->finalize(); // Good practice since we don't call stop recording, but // not essential io->close(); // close the io } +TEST_CASE("initialize preserves existing subject metadata", "[nwb]") +{ + const std::string filename = + getTestFilePath("testInitializeExistingSubject.nwb"); + auto io = std::make_shared(filename); + io->open(); + + auto nwbfile = NWB::NWBFile::create(io); + REQUIRE(nwbfile->initialize(generateUuid()) == Status::Success); + + auto existingSubjectSpec = getTestSubjectSpec(); + existingSubjectSpec.subjectId = "existing-subject"; + auto subject = NWB::Subject::create("/general/subject", io); + REQUIRE(subject->initialize(existingSubjectSpec) == Status::Success); + + REQUIRE(nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()) + == Status::Success); + REQUIRE(subject->readSubjectId()->values().data + == std::vector {"existing-subject"}); +} + +TEST_CASE("initialize fails when Subject has an incompatible cached type", + "[nwb]") +{ + const std::string filename = + getTestFilePath("testInitializeIncompatibleCachedSubject.h5"); + auto io = std::make_shared(filename); + io->open(); + + auto nwbfile = NWB::NWBFile::create(io); + REQUIRE(nwbfile->initialize(generateUuid()) == Status::Success); + + auto incompatibleSubject = NWB::DynamicTable::create("/general/subject", io); + REQUIRE(incompatibleSubject != nullptr); + REQUIRE_FALSE(io->objectExists("/general/subject")); + + REQUIRE(nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()) + == Status::Failure); + + io->close(); +} + +TEST_CASE("createTimeIntervalsTables", "[nwb]") +{ + std::string filename = getTestFilePath("createTimeIntervalsTables.nwb"); + + // initialize nwbfile object and create base structure + std::shared_ptr io = + std::make_shared(filename); + io->open(); + auto nwbfile = NWB::NWBFile::create(io); + REQUIRE(nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()) + == Status::Success); + + // Verify intervals group does not exist initially + REQUIRE(io->objectExists("/intervals") == false); + + // create the Epochs Table + auto epochsTable = nwbfile->createEpochs(false, 50); + REQUIRE(epochsTable != nullptr); + REQUIRE(epochsTable->getName() == "epochs"); + + // Verify intervals group was created + REQUIRE(io->objectExists("/intervals") == true); + REQUIRE(epochsTable->readDescription()->values().data[0] == "Time intervals marking coarse-grained experimental phases or subdivisions of a recording session, such as baseline, task, rest, or sleep stage"); + + // create the Trials Table + auto trialsTable = nwbfile->createTrials(false, 50); + REQUIRE(trialsTable != nullptr); + REQUIRE(trialsTable->getName() == "trials"); + REQUIRE(trialsTable->readDescription()->values().data[0] == "Time intervals corresponding to repeated experimental units with consistent structure, such as individual stimulus-response-reward cycles."); + + // create the Invalid Times Table + auto invalidTimesTable = nwbfile->createInvalidTimes(false, 50); + REQUIRE(invalidTimesTable != nullptr); + REQUIRE(invalidTimesTable->getName() == "invalid_times"); + REQUIRE(invalidTimesTable->readDescription()->values().data[0] + == "Time intervals that should be removed from analysis"); + + // create a custom TimeIntervals Table + auto customTable = nwbfile->createTimeIntervals( + "custom_intervals", "test custom", false, 50); + REQUIRE(customTable != nullptr); + REQUIRE(customTable->getName() == "custom_intervals"); + REQUIRE(customTable->readDescription()->values().data[0] == "test custom"); + + // test readTimeIntervals + auto readCustomTable = nwbfile->readTimeIntervals("custom_intervals"); + REQUIRE(readCustomTable != nullptr); + REQUIRE(readCustomTable->getName() == "custom_intervals"); + REQUIRE(readCustomTable->readDescription()->values().data[0] + == "test custom"); + + // create a custom TimeIntervals Table with post initialization + auto customTimeIntervals = + nwbfile->createTimeIntervalsInstance("custom_intervals2"); + REQUIRE(customTimeIntervals != nullptr); + Status customTimeIntervalsInitStatus = + customTimeIntervals->initialize("no description"); + REQUIRE(customTimeIntervalsInitStatus == Status::Success); + REQUIRE(customTimeIntervals->getName() == "custom_intervals2"); + + // test readTimeIntervals for the table created with specs + auto readCustomTimeIntervals = + nwbfile->readTimeIntervals("custom_intervals2"); + REQUIRE(readCustomTimeIntervals != nullptr); + REQUIRE(readCustomTimeIntervals->getName() == "custom_intervals2"); + REQUIRE(readCustomTimeIntervals->readDescription()->values().data[0] + == "no description"); + + // Write some data to the epochs table + io->startRecording(); + + AQNWB::Types::RowData row1 = { + {"start_time", 1.0f}, {"stop_time", 2.0f}, {"tags", std::string("tag1")}}; + AQNWB::Types::RowData row2 = { + {"start_time", 2.5f}, {"stop_time", 3.5f}, {"tags", std::string("tag2")}}; + + REQUIRE(epochsTable->addRow(row1) == Status::Success); + REQUIRE(epochsTable->addRow(row2) == Status::Success); + + // Write some data to the trials table + AQNWB::Types::RowData trialRow = {{"start_time", 1.0f}, + {"stop_time", 2.0f}, + {"tags", std::string("trial1")}}; + REQUIRE(trialsTable->addRow(trialRow) == Status::Success); + + // Write some data to the invalid times table + AQNWB::Types::RowData invalidTimeRow = {{"start_time", 1.0f}, + {"stop_time", 2.0f}, + {"tags", std::string("invalid1")}}; + REQUIRE(invalidTimesTable->addRow(invalidTimeRow) == Status::Success); + + // Write some data to the custom intervals table + AQNWB::Types::RowData customRow = {{"start_time", 1.0f}, + {"stop_time", 2.0f}, + {"tags", std::string("custom1")}}; + REQUIRE(customTable->addRow(customRow) == Status::Success); + + // Write some data to the custom intervals table with specs + AQNWB::Types::RowData customRow2 = {{"start_time", 1.0f}, + {"stop_time", 2.0f}}; + REQUIRE(customTimeIntervals->addRow(customRow2) == Status::Success); + + io->stopRecording(); + io->close(); +} + +TEST_CASE("createEventsTable with full initialization", "[nwb]") +{ + std::string filename = getTestFilePath("createEventsTable.nwb"); + + // initialize nwbfile object and create base structure + std::shared_ptr io = + std::make_shared(filename); + io->open(); + auto nwbfile = NWB::NWBFile::create(io); + Status fileStatus = nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); + REQUIRE(fileStatus == Status::Success); + + // Verify events group does not exist initially + REQUIRE(io->objectExists("/events") == false); + + // create the Events Table + auto eventsTable = nwbfile->createEventsTable("test_events", + "test description", + "test source", + 0.001f, + false, + std::nullopt, + true, + 50); + REQUIRE(eventsTable != nullptr); + REQUIRE(eventsTable->getName() == "test_events"); + + // Verify events group was created + REQUIRE(io->objectExists("/events") == true); + REQUIRE(eventsTable->readDescription()->values().data[0] + == "test description"); + REQUIRE(eventsTable->readSourceDescription()->values().data[0] + == "test source"); + REQUIRE(eventsTable->readTimestampColumn()->readResolution()->values().data[0] + == Catch::Approx(0.001f)); + REQUIRE(eventsTable->readDurationColumn() + == nullptr); // duration column should not exist + REQUIRE(eventsTable->readAnnotationColumn() + != nullptr); // annotation column should exist + + // Write some data to the table + io->startRecording(); + + std::vector timestamps = {1.0f, 2.0f, 3.0f}; + std::vector annotations = {"event1", "event2", "event3"}; + std::vector ids = {1, 2, 3}; + + SizeArray dataShape = {timestamps.size()}; + SizeArray positionOffset = {0}; + + auto timestampColumn = eventsTable->readTimestampColumn(); + timestampColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + + auto annotationColumn = eventsTable->readAnnotationColumn(); + annotationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::V_STR, annotations); + + eventsTable->setRowIDs(ids); + + io->stopRecording(); + io->close(); +} + +TEST_CASE("createEventsTableInstance with post initialization", "[nwb]") +{ + std::string filename = getTestFilePath("createEventsTablePostInit.nwb"); + + // initialize nwbfile object and create base structure + std::shared_ptr io = + std::make_shared(filename); + io->open(); + auto nwbfile = NWB::NWBFile::create(io); + Status fileStatus = nwbfile->initialize(generateUuid(), + "Test initialized NWB file", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); + REQUIRE(fileStatus == Status::Success); + + // create the Events Table + auto eventsTable = nwbfile->createEventsTableInstance("test_events"); + REQUIRE(eventsTable != nullptr); + std::string description = "Test events table"; + std::string sourceDescription = "Test source description"; + float timestampResolution = 1.0f / 30000.0f; + std::optional durationResolution = std::nullopt; // no duration column + auto specs = NWB::EventsTable::createDefaultDataSpecs( + timestampResolution, false, durationResolution, true, 100); + Status initStatus = + eventsTable->initialize(description, sourceDescription, specs); + REQUIRE(initStatus == Status::Success); + REQUIRE(eventsTable->getName() == "test_events"); + REQUIRE(eventsTable->readDescription()->values().data[0] == description); + REQUIRE(eventsTable->readSourceDescription()->values().data[0] + == sourceDescription); + REQUIRE(eventsTable->readTimestampColumn()->readResolution()->values().data[0] + == Catch::Approx(timestampResolution)); + REQUIRE(eventsTable->readDurationColumn() + == nullptr); // duration column should not exist + REQUIRE(eventsTable->readAnnotationColumn() + != nullptr); // annotation column should exist + + // Write some data to the table + io->startRecording(); + + std::vector timestamps = {1.0f, 2.0f, 3.0f}; + std::vector annotations = {"event1", "event2", "event3"}; + std::vector ids = {1, 2, 3}; + + SizeArray dataShape = {timestamps.size()}; + SizeArray positionOffset = {0}; + + auto timestampColumn = eventsTable->readTimestampColumn(); + timestampColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + + auto annotationColumn = eventsTable->readAnnotationColumn(); + annotationColumn->recordData()->writeDataBlock( + dataShape, positionOffset, BaseDataType::V_STR, annotations); + + eventsTable->setRowIDs(ids); + + io->stopRecording(); + io->close(); + + // Reopen the file and verify that the data was written correctly + std::shared_ptr readio = + std::make_shared(filename); + readio->open(); + auto readnwbfile = NWB::NWBFile::create(readio); + REQUIRE(readnwbfile != nullptr); + auto readEventsTable = readnwbfile->readEventsTable("test_events"); + REQUIRE(readEventsTable != nullptr); + REQUIRE(readEventsTable->readDescription()->values().data[0] == description); + REQUIRE(readEventsTable->readSourceDescription()->values().data[0] + == sourceDescription); + REQUIRE( + readEventsTable->readTimestampColumn()->readResolution()->values().data[0] + == Catch::Approx(timestampResolution)); + REQUIRE(readEventsTable->readDurationColumn() + == nullptr); // duration column should not exist + REQUIRE(readEventsTable->readAnnotationColumn() + != nullptr); // annotation column should exist +} + TEST_CASE("createElectrodesTable", "[nwb]") { std::string filename = getTestFilePath("createElectrodesTable.nwb"); @@ -87,12 +412,23 @@ TEST_CASE("createElectrodesTable", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test electrodes table", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // create the Electrodes Table std::vector mockArrays = getMockChannelArrays(1, 2); - auto electrodesTable = nwbfile->createElectrodesTable(mockArrays); + auto electrodesTable = nwbfile->createElectrodesTable(mockArrays, true, 50); REQUIRE(electrodesTable != nullptr); + + // Verify chunk size + auto chunking = io->getStorageObjectChunking( + "/general/extracellular_ephys/electrodes/id"); + REQUIRE(chunking.size() == 1); + REQUIRE(chunking[0] == 50); } TEST_CASE("createElectricalSeriesWithSubsetOfElectrodes", "[nwb]") @@ -105,7 +441,12 @@ TEST_CASE("createElectricalSeriesWithSubsetOfElectrodes", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test electrical series subset", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // Create electrode table with full set of electrodes (4 channels) std::vector allElectrodes = getMockChannelArrays(4, 1); @@ -155,7 +496,12 @@ TEST_CASE("createElectricalSeriesFailsWithoutElectrodesTable", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test multiple ecephys datasets", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // Attempt to create electrical series without creating electrodes table first std::vector recordingElectrodes = @@ -181,7 +527,12 @@ TEST_CASE("createElectricalSeriesFailsWithOutOfRangeIndices", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test annotation series", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // Create electrode table with 2 channels std::vector tableElectrodes = @@ -211,7 +562,19 @@ TEST_CASE("createElectricalSeries", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + AQNWB::NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.subjectId = "mouse001"; + subjectSpec.species = "Mus musculus"; + subjectSpec.sex = "M"; + subjectSpec.age = "P90D"; + subjectSpec.description = "Wild type mouse used for electrophysiology study"; + std::string currentTime = getCurrentTime(); + nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); // create the Electrodes Table std::vector mockArrays = getMockChannelArrays(); @@ -268,6 +631,7 @@ TEST_CASE("createElectricalSeries", "[nwb]") } // Check that we can find all the types that we created + // - /general/subject : core::Subject (created by default in initialize) // - /general/extracellular_ephys/array0 : core::ElectrodeGroup // - /general/devices/array1 : core::Device // - /general/extracellular_ephys/electrodes : core::DynamicTable @@ -276,7 +640,7 @@ TEST_CASE("createElectricalSeries", "[nwb]") // - /general/extracellular_ephys/array1 : core::ElectrodeGroup // - /acquisition/esdata0 : core::ElectricalSeries auto result = nwbfile->findOwnedTypes(); - REQUIRE(result.size() == 7); + REQUIRE(result.size() == 8); // finalize the nwb file io->stopRecording(); @@ -290,7 +654,13 @@ TEST_CASE("createMultipleEcephysDatasets", "[nwb]") std::shared_ptr io = std::make_shared(filename); io->open(); auto nwbfile = AQNWB::NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + auto currentTime = getCurrentTime(); + nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + getTestSubjectSpec()); // create ElectrodesTable std::vector mockArrays = getMockChannelArrays(2, 2); @@ -360,7 +730,14 @@ TEST_CASE("createAnnotationSeries", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + auto currentTime = getCurrentTime(); + auto subjectSpec = getTestSubjectSpec(); + nwbfile->initialize(generateUuid(), + "a recording session", + "data collection info", + currentTime, + currentTime, + subjectSpec); // create Annotation Series std::vector mockAnnotationNames = {"annotations1", @@ -419,7 +796,12 @@ TEST_CASE("setCanModifyObjectsMode", "[nwb]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - Status initStatus = nwbfile->initialize(generateUuid()); + Status initStatus = nwbfile->initialize(generateUuid(), + "Test recording mode", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); REQUIRE(initStatus == Status::Success); // start recording @@ -436,13 +818,13 @@ TEST_CASE("setCanModifyObjectsMode", "[nwb]") std::vector mockArrays = getMockChannelArrays(1, 2); std::vector mockChannelNames = getMockChannelArrayNames("esdata"); - auto electrodesTable = nwbfile->createElectrodesTable( - mockArrays); // create the Electrodes Table - REQUIRE(electrodesTable != nullptr); - std::vector containerIndices = {}; - Status resultCreatePostStart = nwbfile->createElectricalSeries( - mockArrays, mockChannelNames, BaseDataType::F32, containerIndices); - REQUIRE(resultCreatePostStart == Status::Failure); + // create the Electrodes Table + // This will fail because we cannot modify objects after starting the + // recording + auto electrodesTable = nwbfile->createElectrodesTable(mockArrays); + REQUIRE(electrodesTable == nullptr); + // Because we done have and ElectrodesTable, we cannot create an + // ElectricalSeries either and that would fail even earlier. // stop recording io->stopRecording(); @@ -471,7 +853,8 @@ TEST_CASE("testAttributeAndDatasetFields", "[nwb]") description, dataCollection, sessionStartTime, - timestampsReferenceTime); + timestampsReferenceTime, + getTestSubjectSpec()); REQUIRE(initStatus == Status::Success); REQUIRE(nwbfile->isInitialized()); diff --git a/tests/testProcessingModule.cpp b/tests/testProcessingModule.cpp index 28e469e5d..c6766f529 100644 --- a/tests/testProcessingModule.cpp +++ b/tests/testProcessingModule.cpp @@ -30,7 +30,12 @@ TEST_CASE("createProcessingModule", "[processingmodule]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test processing module", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // create and initialize a ProcessingModule auto processingModule = nwbfile->createProcessingModule("test_module"); @@ -64,7 +69,12 @@ TEST_CASE("createMultipleProcessingModules", "[processingmodule]") std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test processing modules", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // create and initialize two ProcessingModules auto module1 = nwbfile->createProcessingModule("module1"); @@ -104,7 +114,12 @@ TEST_CASE("ProcessingModule createNWBDataInterface and readNWBDataInterface", std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test processing module time series", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // create processing module auto processingModule = nwbfile->createProcessingModule("ecephys"); @@ -191,7 +206,12 @@ TEST_CASE("ProcessingModule createDynamicTable and readDynamicTable", std::make_shared(filename); io->open(); auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test processing module dynamic table", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // create processing module auto processingModule = nwbfile->createProcessingModule("analysis_module"); @@ -219,11 +239,7 @@ TEST_CASE("ProcessingModule createDynamicTable and readDynamicTable", // set row IDs std::vector ids = {0, 1, 2}; - SizeArray idShape = {ids.size()}; - IO::ArrayDataSetConfig idConfig(BaseDataType::I32, idShape, idShape); - auto elementIDs = NWB::ElementIdentifiers::create(tablePath + "/id", io); - elementIDs->initialize(idConfig); - REQUIRE(table->setRowIDs(elementIDs, ids) == Status::Success); + REQUIRE(table->setRowIDs(ids) == Status::Success); // read back the table within the same session via readDynamicTable auto readTable = processingModule->readDynamicTable("summary_table"); diff --git a/tests/testRecordingObjects.cpp b/tests/testRecordingObjects.cpp index 30cfdf5e0..d5826a89a 100644 --- a/tests/testRecordingObjects.cpp +++ b/tests/testRecordingObjects.cpp @@ -44,27 +44,6 @@ class FaultyTimeSeries : public NWB::TimeSeries // make finalize always fail Status finalize() override { return Status::Failure; } - - // a no-op clear (the default implementation would be fine) - void clearRecordingDataCache() override {} -}; - -class ExceptionThrowingSeries : public NWB::TimeSeries -{ -public: - ExceptionThrowingSeries(const std::string& p, - const std::shared_ptr& io) - : TimeSeries(p, io) - { - } - - Status finalize() override { return Status::Success; } - - // throw when the RecordingObjects wrapper asks us to clear the cache - void clearRecordingDataCache() override - { - throw std::runtime_error("simulated clearRecordingDataCache failure"); - } }; } // anonymous namespace @@ -101,27 +80,38 @@ TEST_CASE("RecordingObjects recording workflow tests", "[recording]") // ---- electrodes ---------------------------------------------------- auto mockRecordingArrays = getMockChannelArrays(); // default 4 channels - auto electrodesTable = nwbFile->createElectrodesTable(mockRecordingArrays); + auto electrodesTable = + nwbFile->createElectrodesTable(mockRecordingArrays, true, 50); REQUIRE(electrodesTable != nullptr); - // recordingObjects should hold: - // Index = 0; Type = core::NWBFile; Path = /; -- cache size: 0 - // Index = 1; Type = hdmf-common::ElementIdentifiers; Path = - // /general/extracellular_ephys/electrodes/id; -- cache size: 1 Index = 2; - // Type = hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/group_name; -- cache size: 1 + // Verify chunk size + auto chunking = io->getStorageObjectChunking( + "/general/extracellular_ephys/electrodes/id"); + REQUIRE(chunking.size() == 1); + REQUIRE(chunking[0] == 50); + + // After createElectrodesTable() (with finalizeTable=true) recordingObjects + // should hold the following 10 objects. Note that the ElectrodesTable + // registers itself first and then configures its columns during + // initialize(); the "group" reference column is added last during + // finalize(). Each object appears exactly once (no duplicates): + // Index = 0; Type = core::NWBFile; Path = /; + // Index = 1; Type = core::ElectrodesTable; Path = + // /general/extracellular_ephys/electrodes; + // Index = 2; Type = hdmf-common::ElementIdentifiers; Path = + // /general/extracellular_ephys/electrodes/id; // Index = 3; Type = hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/location; -- cache size: 1 Index - // = 4; Type = core::DynamicTable; Path = - // /general/extracellular_ephys/electrodes; -- cache size: 0 Index = 5; - // Type = core::Device; Path = /general/devices/array0; -- cache size: 0 + // /general/extracellular_ephys/electrodes/location; + // Index = 4; Type = hdmf-common::VectorData; Path = + // /general/extracellular_ephys/electrodes/group_name; + // Index = 5; Type = core::Device; Path = /general/devices/array0; // Index = 6; Type = core::ElectrodeGroup; Path = - // /general/extracellular_ephys/array0; -- cache size: 0 Index = 7; Type = - // core::Device; Path = /general/devices/array1; -- cache size: 0 Index = - // 8; Type = core::ElectrodeGroup; Path = - // /general/extracellular_ephys/array1; -- cache size: 0 Index = 9; Type = - // hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/group; -- cache size: 0 + // /general/extracellular_ephys/array0; + // Index = 7; Type = core::Device; Path = /general/devices/array1; + // Index = 8; Type = core::ElectrodeGroup; Path = + // /general/extracellular_ephys/array1; + // Index = 9; Type = hdmf-common::VectorData; Path = + // /general/extracellular_ephys/electrodes/group; (added in finalize()) SizeType expectedNumRecordingObjects = 10; REQUIRE(recordingObjects->size() == expectedNumRecordingObjects); @@ -194,39 +184,42 @@ TEST_CASE("RecordingObjects recording workflow tests", "[recording]") // registered object auto existingSeries = NWB::ElectricalSeries::create("/acquisition/esdata0", io); - expectedNumRecordingObjects += 1; // we created a new object + // expectedNumRecordingObjects remains the same because we reuse the + // existing object REQUIRE(existingSeries != nullptr); REQUIRE(existingSeries->isRegisteredRecordingObject() == true); REQUIRE(recordingObjects->size() == expectedNumRecordingObjects); - // NOTE: since we created a new ElectricalSeries object (instead of reusing - // the existing one), we now have 2 objects for the same path + // NOTE: since we reuse the existing ElectricalSeries object, we get the + // same index SizeType existingIdx = recordingObjects->getRecordingIndex(existingSeries); - REQUIRE(existingIdx == 12); // should be the same as before + REQUIRE(existingIdx == 10); // should be the same as before // Since existingSeries exists in the file, requesting recordData() should // yield the same object that is already in the cache auto existingRecordData = existingSeries->recordData(); REQUIRE(existingRecordData != nullptr); REQUIRE(existingSeries->isRegisteredRecordingObject() == true); - // recordingObjects should now hold: - // RecordingObjects contents: + // recordingObjects should now hold (12 entries): // Index = 0; Type = core::NWBFile; Path = /; - // Index = 1; Type = hdmf-common::ElementIdentifiers; Path = - // /general/extracellular_ephys/electrodes/id; Index = 2; Type = - // hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/group_name; Index = 3; Type = - // hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/location; Index = 4; Type = - // core::DynamicTable; Path = /general/extracellular_ephys/electrodes; Index - // = 5; Type = core::Device; Path = /general/devices/array0; Index = 6; Type - // = core::ElectrodeGroup; Path = /general/extracellular_ephys/array0; Index - // = 7; Type = core::Device; Path = /general/devices/array1; Index = 8; Type - // = core::ElectrodeGroup; Path = /general/extracellular_ephys/array1; Index - // = 9; Type = hdmf-common::VectorData; Path = - // /general/extracellular_ephys/electrodes/group; Index = 10; Type = - // core::ElectricalSeries; Path = /acquisition/esdata0; Index = 11; Type = - // core::ElectricalSeries; Path = /acquisition/esdata1; Index = 12; Type = - // core::ElectricalSeries; Path = /acquisition/esdata0; + // Index = 1; Type = core::ElectrodesTable; Path = + // /general/extracellular_ephys/electrodes; + // Index = 2; Type = hdmf-common::ElementIdentifiers; Path = + // /general/extracellular_ephys/electrodes/id; + // Index = 3; Type = hdmf-common::VectorData; Path = + // /general/extracellular_ephys/electrodes/location; + // Index = 4; Type = hdmf-common::VectorData; Path = + // /general/extracellular_ephys/electrodes/group_name; + // Index = 5; Type = core::Device; Path = /general/devices/array0; + // Index = 6; Type = core::ElectrodeGroup; Path = + // /general/extracellular_ephys/array0; + // Index = 7; Type = core::Device; Path = /general/devices/array1; + // Index = 8; Type = core::ElectrodeGroup; Path = + // /general/extracellular_ephys/array1; + // Index = 9; Type = hdmf-common::VectorData; Path = + // /general/extracellular_ephys/electrodes/group; + // Index = 10; Type = core::ElectricalSeries; Path = /acquisition/esdata0; + // Index = 11; Type = core::ElectricalSeries; Path = /acquisition/esdata1; + // Index = 12; Type = core::ElectricalSeries; Path = /acquisition/esdata0; // cleanup io->close(); @@ -257,7 +250,9 @@ TEST_CASE("RecordingObjects recording workflow tests", "[recording]") nwbFile->createElectricalSeries( mockArrays, mockNames, BaseDataType::I16, contIdx); REQUIRE(contIdx.size() == mockNames.size()); - REQUIRE(recordingObjects->size() == 12); // see previous section + // 10 from the finalized electrodes table (see SECTION 1) + 2 + // ElectricalSeries + REQUIRE(recordingObjects->size() == 12); // Repeated calls to finalize should succeed without error for (SizeType i = 0; i < 4; i++) { @@ -274,65 +269,4 @@ TEST_CASE("RecordingObjects recording workflow tests", "[recording]") io->close(); } - - // -------------------------------------------------------------- - // SECTION 3 – clearRecordingDataCache aggregates & tolerates exceptions - // -------------------------------------------------------------- - SECTION("clearRecordingDataCache aggregates and tolerates exceptions") - { - const std::string path = getTestFilePath("recObjectsWorkflow_3.h5"); - std::shared_ptr io = createIO("HDF5", path); - io->open(); - REQUIRE(io->isOpen()); - auto recordingObjects = io->getRecordingObjects(); - REQUIRE(recordingObjects != nullptr); - REQUIRE(recordingObjects->size() == 0); - - // good series (created through the normal workflow) - auto nwbFile = NWB::NWBFile::create(io); - nwbFile->initialize(generateUuid()); - auto mockArrays = getMockChannelArrays(); - nwbFile->createElectrodesTable(mockArrays); - std::vector contIdx; - nwbFile->createElectricalSeries(mockArrays, - getMockChannelArrayNames("esdata"), - BaseDataType::I16, - contIdx); - REQUIRE(recordingObjects->size() == 12); - std::vector expextedCacheSize = { - 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 2, 2}; - // Confirm that the recording data caches are as expected - for (SizeType i = 0; i < recordingObjects->size(); ++i) { - auto obj = recordingObjects->getRecordingObject(i); - REQUIRE(obj != nullptr); - std::cout << "obj[" << i << "] = " << obj->getFullTypeName() << ": " - << obj->getPath() - << " -- cache size: " << obj->getCacheRecordingData().size() - << std::endl; - REQUIRE(obj->getCacheRecordingData().size() <= expextedCacheSize[i]); - } - // clear the BaseRecordingData caches – should succeed - Status clearCachesStatus = recordingObjects->clearRecordingDataCache(); - REQUIRE(clearCachesStatus == Status::Success); - // Confirm that the recording data caches are cleared - for (SizeType i = 0; i < recordingObjects->size(); ++i) { - auto obj = recordingObjects->getRecordingObject(i); - REQUIRE(obj != nullptr); - REQUIRE(obj->getCacheRecordingData().size() == 0); - } - - // add a series that throws when clearing its cache - auto throwing = std::make_shared("/throwing", io); - recordingObjects->addRecordingObject(throwing); - REQUIRE(recordingObjects->size() == 13); - - REQUIRE(recordingObjects->clearRecordingDataCache() == Status::Failure); - - // Clear the recordingObjects and check it's cleared - recordingObjects->clear(); - REQUIRE(recordingObjects->size() == 0); - - // cleanup - io->close(); - } } diff --git a/tests/testRecordingWorkflow.cpp b/tests/testRecordingWorkflow.cpp index e52a8a48d..161b3bad6 100644 --- a/tests/testRecordingWorkflow.cpp +++ b/tests/testRecordingWorkflow.cpp @@ -46,10 +46,21 @@ TEST_CASE("writeContinuousData", "[recording]") // 3. create NWBFile object auto nwbfile = NWB::NWBFile::create(io); - nwbfile->initialize(generateUuid()); + nwbfile->initialize(generateUuid(), + "Test continuous recording", + "Test data collection", + getCurrentTime(), + getCurrentTime(), + getTestSubjectSpec()); // 4. create an electrodes table. - nwbfile->createElectrodesTable(mockRecordingArrays); + nwbfile->createElectrodesTable(mockRecordingArrays, true, 50); + + // Verify chunk size + auto chunking = io->getStorageObjectChunking( + "/general/extracellular_ephys/electrodes/id"); + REQUIRE(chunking.size() == 1); + REQUIRE(chunking[0] == 50); // 5. create datasets (automatically added to recording objects) std::vector containerIndexes = {}; diff --git a/tests/testRegisteredType.cpp b/tests/testRegisteredType.cpp index 0f82bd662..41b9ada71 100644 --- a/tests/testRegisteredType.cpp +++ b/tests/testRegisteredType.cpp @@ -84,11 +84,17 @@ TEST_CASE("RegisterType", "[base]") } else if (subclassFullName == "core::ElectrodesTable") { examplePath = ElectrodesTable::electrodesTablePath; exampleName = "electrodes"; + } else if (subclassFullName == "core::Subject") { + examplePath = "/general/subject"; + exampleName = "subject"; } else { - examplePath = "/example/path"; - exampleName = "path"; + examplePath = "/example/path_" + typeName; + exampleName = "path_" + typeName; } + // Clear the cache to ensure we create a new instance + io->getRecordingObjects()->clear(); + // Create the type auto instance = RegisteredType::create(subclassFullName, examplePath, io); REQUIRE(instance != nullptr); // Check that the object was created @@ -228,18 +234,28 @@ TEST_CASE("RegisterType", "[base]") REQUIRE(fallbackInstance->getTypeName() == fallBackClassName); REQUIRE(fallbackInstance->getNamespace() == fallBackNamespace); + // Clear the cache so the next call doesn't return the cached fallback + // instance + io->getRecordingObjects()->clear(); + // Test with unknown type and fallbackToBase=false // Should return nullptr as before auto noFallbackInstance = RegisteredType::create(fullClassName, examplePath, io, false); REQUIRE(noFallbackInstance == nullptr); + // Clear the cache again + io->getRecordingObjects()->clear(); + // Test with empty type name and fallbackToBase=true auto emptyWithFallback = RegisteredType::create("", examplePath, io, true); REQUIRE(emptyWithFallback != nullptr); REQUIRE(emptyWithFallback->getTypeName() == fallBackClassName); REQUIRE(emptyWithFallback->getNamespace() == fallBackNamespace); + // Clear the cache + io->getRecordingObjects()->clear(); + // Test with unknown type and fallbackToBase=true while reading the type // information from file Should return base Container type auto fallbackInstance2 = RegisteredType::create(examplePath, io, true); @@ -247,7 +263,10 @@ TEST_CASE("RegisterType", "[base]") REQUIRE(fallbackInstance2->getTypeName() == fallBackClassName); REQUIRE(fallbackInstance2->getNamespace() == fallBackNamespace); - // Test with unknown type and fallbackToBase=true while reading the type + // Clear the cache + io->getRecordingObjects()->clear(); + + // Test with unknown type and fallbackToBase=false while reading the type // information from file Should return nullptr as before auto noFallbackInstance2 = RegisteredType::create(examplePath, io, false); REQUIRE(noFallbackInstance2 == nullptr); @@ -283,18 +302,28 @@ TEST_CASE("RegisterType", "[base]") REQUIRE(fallbackInstance->getTypeName() == fallBackClassName); REQUIRE(fallbackInstance->getNamespace() == fallBackNamespace); + // Clear the cache so the next call doesn't return the cached fallback + // instance + io->getRecordingObjects()->clear(); + // Test with unknown type and fallbackToBase=false // Should return nullptr as before auto noFallbackInstance = RegisteredType::create(fullClassName, examplePath, io, false); REQUIRE(noFallbackInstance == nullptr); + // Clear the cache again + io->getRecordingObjects()->clear(); + // Test with empty type name and fallbackToBase=true auto emptyWithFallback = RegisteredType::create("", examplePath, io, true); REQUIRE(emptyWithFallback != nullptr); REQUIRE(emptyWithFallback->getTypeName() == fallBackClassName); REQUIRE(emptyWithFallback->getNamespace() == fallBackNamespace); + // Clear the cache + io->getRecordingObjects()->clear(); + // Test with unknown type and fallbackToBase=true while reading the type // information from file Should return base Container type auto fallbackInstance2 = RegisteredType::create(examplePath, io, true); @@ -302,7 +331,10 @@ TEST_CASE("RegisterType", "[base]") REQUIRE(fallbackInstance2->getTypeName() == fallBackClassName); REQUIRE(fallbackInstance2->getNamespace() == fallBackNamespace); - // Test with unknown type and fallbackToBase=true while reading the type + // Clear the cache + io->getRecordingObjects()->clear(); + + // Test with unknown type and fallbackToBase=false while reading the type // information from file Should return nullptr as before auto noFallbackInstance2 = RegisteredType::create(examplePath, io, false); REQUIRE(noFallbackInstance2 == nullptr); @@ -434,7 +466,7 @@ TEST_CASE("RegisterType", "[base]") SizeArray {3}, SizeArray {0}, BaseDataType::F32, datasetValues.data()); // Check that the cache is not empty - auto cachedData = testInstance->getCacheRecordingData(); + auto cachedData = testInstance->getIO()->getRecordingDataCache(); REQUIRE(cachedData.size() == 1); // Check that the cached data is the same as the last dataset auto checkCachedValue = cachedData.find(datasetPath); @@ -442,10 +474,10 @@ TEST_CASE("RegisterType", "[base]") REQUIRE(checkCachedValue->second.get() == dataset4.get()); // Clear the cache - testInstance->clearRecordingDataCache(); + testInstance->getIO()->clearRecordingDataCache(); // Check that the cache is empty - cachedData = testInstance->getCacheRecordingData(); + cachedData = testInstance->getIO()->getRecordingDataCache(); REQUIRE(cachedData.empty()); // Get a new dataset instance and check that it is not the same as the diff --git a/tests/testSubject.cpp b/tests/testSubject.cpp new file mode 100644 index 000000000..4d804ade6 --- /dev/null +++ b/tests/testSubject.cpp @@ -0,0 +1,187 @@ +#include +#include +#include + +#include + +#include "Utils.hpp" +#include "io/BaseIO.hpp" +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/NWBFile.hpp" +#include "nwb/file/Subject.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("Subject", "[file]") +{ + SECTION("is registered and always uses the canonical subject path") + { + auto registry = NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("core::Subject") != registry.end()); + + auto io = createIO("HDF5", getTestFilePath("subject_constructor.h5")); + auto subject = NWB::Subject::create("/not/the/subject/path", io); + + REQUIRE(subject->getPath() == "/general/subject"); + REQUIRE(subject->getIO() == io); + } + + SECTION("convenience create method creates a Subject at the canonical path") + { + auto io = createIO("HDF5", getTestFilePath("subject_create.h5")); + auto subject = NWB::Subject::create(io); + + REQUIRE(subject->getPath() == "/general/subject"); + REQUIRE(subject->getIO() == io); + } + + SECTION("setting parameters via with function chaining") + { + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.withAge("P90D") + .withAgeReference("gestational") + .withDateOfBirth("2024-01-15T00:00:00.000000+00:00") + .withDescription("Test subject") + .withGenotype("wt/wt") + .withSex("M") + .withSpecies("Mus musculus") + .withStrain("C57BL/6J") + .withSubjectId("subject-001") + .withWeight("25 g"); + + REQUIRE(subjectSpec.age == "P90D"); + REQUIRE(subjectSpec.ageReference == "gestational"); + REQUIRE(subjectSpec.dateOfBirth == "2024-01-15T00:00:00.000000+00:00"); + REQUIRE(subjectSpec.description == "Test subject"); + REQUIRE(subjectSpec.genotype == "wt/wt"); + REQUIRE(subjectSpec.sex == "M"); + REQUIRE(subjectSpec.species == "Mus musculus"); + REQUIRE(subjectSpec.strain == "C57BL/6J"); + REQUIRE(subjectSpec.subjectId == "subject-001"); + REQUIRE(subjectSpec.weight == "25 g"); + } + + SECTION("writes and reads every metadata field") + { + const std::string filename = getTestFilePath("subject_all_fields.h5"); + auto io = std::make_shared(filename); + io->open(); + io->createGroup("/general"); + + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.age = "P90D"; + subjectSpec.ageReference = "gestational"; + subjectSpec.dateOfBirth = "2024-01-15T00:00:00.000000+00:00"; + subjectSpec.description = "Test subject"; + subjectSpec.genotype = "wt/wt"; + subjectSpec.sex = "M"; + subjectSpec.species = "Mus musculus"; + subjectSpec.strain = "C57BL/6J"; + subjectSpec.subjectId = "subject-001"; + subjectSpec.weight = "25 g"; + + auto subject = NWB::Subject::create("/general/subject", io); + REQUIRE(subject->initialize(subjectSpec) == Status::Success); + io->close(); + + auto readIO = std::make_shared(filename); + readIO->open(IO::FileMode::ReadOnly); + auto readSubject = std::dynamic_pointer_cast( + NWB::RegisteredType::create("/general/subject", readIO)); + + REQUIRE(readSubject != nullptr); + REQUIRE(readSubject->readAge()->values().data + == std::vector {"P90D"}); + REQUIRE(readSubject->readAgeReference()->values().data + == std::vector {"gestational"}); + REQUIRE(readSubject->readDateOfBirth()->values().data + == std::vector {"2024-01-15T00:00:00.000000+00:00"}); + REQUIRE(readSubject->readDescription()->values().data + == std::vector {"Test subject"}); + REQUIRE(readSubject->readGenotype()->values().data + == std::vector {"wt/wt"}); + REQUIRE(readSubject->readSex()->values().data + == std::vector {"M"}); + REQUIRE(readSubject->readSpecies()->values().data + == std::vector {"Mus musculus"}); + REQUIRE(readSubject->readStrain()->values().data + == std::vector {"C57BL/6J"}); + REQUIRE(readSubject->readSubjectId()->values().data + == std::vector {"subject-001"}); + REQUIRE(readSubject->readWeight()->values().data + == std::vector {"25 g"}); + + readIO->close(); + } + + SECTION("uses birth as the default age reference and omits unset fields") + { + const std::string filename = getTestFilePath("subject_optional_fields.h5"); + auto io = std::make_shared(filename); + io->open(); + io->createGroup("/general"); + + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.age = "P7D"; + auto subject = NWB::Subject::create("/general/subject", io); + + REQUIRE(subject->initialize(subjectSpec) == Status::Success); + REQUIRE(subject->readAgeReference()->values().data + == std::vector {"birth"}); + REQUIRE_FALSE(subject->readSpecies()->exists()); + REQUIRE_FALSE(subject->readDateOfBirth()->exists()); + + io->close(); + } + + SECTION("rejects invalid date of birth without writing it") + { + const std::string filename = getTestFilePath("subject_invalid_dob.h5"); + auto io = std::make_shared(filename); + io->open(); + io->createGroup("/general"); + + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.dateOfBirth = "January 15, 2024"; + auto subject = NWB::Subject::create("/general/subject", io); + + REQUIRE(subject->initialize(subjectSpec) == Status::Failure); + REQUIRE_FALSE(subject->readDateOfBirth()->exists()); + REQUIRE_FALSE(io->objectExists("/general/subject/date_of_birth")); + + io->close(); + } + + SECTION("is created by NWBFile initialization when metadata is supplied") + { + const std::string filename = getTestFilePath("nwbfile_subject.nwb"); + auto io = std::make_shared(filename); + io->open(); + + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.age = "P90D"; + subjectSpec.ageReference = "gestational"; + subjectSpec.dateOfBirth = "2024-01-15T00:00:00.000000+00:00"; + subjectSpec.description = "Test subject"; + subjectSpec.genotype = "wt/wt"; + subjectSpec.sex = "M"; + subjectSpec.species = "Mus musculus"; + subjectSpec.strain = "C57BL/6J"; + subjectSpec.subjectId = "subject-001"; + subjectSpec.weight = "25 g"; + auto currentTime = getCurrentTime(); + auto nwbFile = NWB::NWBFile::create(io); + REQUIRE(nwbFile->initialize(generateUuid(), + "Subject integration test", + "Subject test data collection", + currentTime, + currentTime, + std::optional(subjectSpec)) + == Status::Success); + REQUIRE(io->objectExists("/general/subject")); + REQUIRE(io->objectExists("/general/subject/subject_id")); + + io->close(); + } +} diff --git a/tests/testTimeIntervals.cpp b/tests/testTimeIntervals.cpp new file mode 100644 index 000000000..0e426e4da --- /dev/null +++ b/tests/testTimeIntervals.cpp @@ -0,0 +1,279 @@ +#include +#include +#include + +#include + +#include "io/hdf5/HDF5IO.hpp" +#include "nwb/epoch/TimeIntervals.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "nwb/hdmf/table/VectorIndex.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; +using namespace AQNWB::NWB; +using namespace AQNWB::IO; +using namespace AQNWB::IO::HDF5; + +TEST_CASE("TimeIntervals - createDefaultDataSpecs", "[TimeIntervals]") +{ + SECTION("Without tags column") + { + auto specs = + TimeIntervals::createDefaultDataSpecs("/intervals/epochs", 100, false); + + // Should have id, start_time, stop_time + REQUIRE(specs.size() == 3); + REQUIRE(specs[0]->name == "id"); + REQUIRE(specs[1]->name == "start_time"); + REQUIRE(specs[2]->name == "stop_time"); + } + + SECTION("With tags column") + { + auto specs = + TimeIntervals::createDefaultDataSpecs("/intervals/epochs", 100, true); + + // Should have id, start_time, stop_time, tags, tags_index + REQUIRE(specs.size() == 5); + REQUIRE(specs[0]->name == "id"); + REQUIRE(specs[1]->name == "start_time"); + REQUIRE(specs[2]->name == "stop_time"); + REQUIRE(specs[3]->name == "tags"); + REQUIRE(specs[4]->name == "tags_index"); + } +} + +TEST_CASE("TimeIntervals - DataSpec validation", "[TimeIntervals]") +{ + auto io = createIO("HDF5", + getTestFilePath("testTimeIntervalsDataSpecValidation.h5")); + io->open(); + + auto timeIntervals = TimeIntervals::create("/intervals/epochs", io); + auto validSpecs = TimeIntervals::createDefaultDataSpecs("/intervals/epochs"); + REQUIRE(timeIntervals->validateDataSpecs(validSpecs) == Status::Success); + + // Each required time column must use VectorData, be one-dimensional, and + // store 32-bit floats. + auto wrongStartTimeSpecType = validSpecs; + wrongStartTimeSpecType[1] = ElementIdentifiers::createDataSpec( + "start_time", ArrayDataSetConfig(BaseDataType::I32, {0}, {10})); + auto wrongStartTimeShape = validSpecs; + wrongStartTimeShape[1] = VectorData::createDataSpec( + "start_time", + ArrayDataSetConfig(BaseDataType::F32, {0, 1}, {1, 1}), + "start time"); + auto wrongStartTimeDataType = validSpecs; + wrongStartTimeDataType[1] = VectorData::createDataSpec( + "start_time", + ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "start time"); + + auto wrongStopTimeSpecType = validSpecs; + wrongStopTimeSpecType[2] = ElementIdentifiers::createDataSpec( + "stop_time", ArrayDataSetConfig(BaseDataType::I32, {0}, {10})); + auto wrongStopTimeShape = validSpecs; + wrongStopTimeShape[2] = VectorData::createDataSpec( + "stop_time", + ArrayDataSetConfig(BaseDataType::F32, {0, 1}, {1, 1}), + "stop time"); + auto wrongStopTimeDataType = validSpecs; + wrongStopTimeDataType[2] = VectorData::createDataSpec( + "stop_time", + ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "stop time"); + + REQUIRE(timeIntervals->validateDataSpecs(wrongStartTimeSpecType) + == Status::Failure); + REQUIRE(timeIntervals->validateDataSpecs(wrongStartTimeShape) + == Status::Failure); + REQUIRE(timeIntervals->validateDataSpecs(wrongStartTimeDataType) + == Status::Failure); + REQUIRE(timeIntervals->validateDataSpecs(wrongStopTimeSpecType) + == Status::Failure); + REQUIRE(timeIntervals->validateDataSpecs(wrongStopTimeShape) + == Status::Failure); + REQUIRE(timeIntervals->validateDataSpecs(wrongStopTimeDataType) + == Status::Failure); + + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStartTimeSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStartTimeShape), + std::invalid_argument); + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStartTimeDataType), + std::invalid_argument); + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStopTimeSpecType), + std::invalid_argument); + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStopTimeShape), + std::invalid_argument); + REQUIRE_THROWS_AS( + timeIntervals->initialize("Test intervals", wrongStopTimeDataType), + std::invalid_argument); + + io->close(); +} + +TEST_CASE("TimeIntervals - Initialize and Add Rows", "[TimeIntervals]") +{ + std::string filename = getTestFilePath("testTimeIntervals.h5"); + + SECTION("Basic initialization and row addition") + { + { + auto io = createIO("HDF5", filename); + io->open(FileMode::Overwrite); + + // Verify intervals group does not exist initially + REQUIRE(io->objectExists("/intervals") == false); + + auto timeIntervals = TimeIntervals::create("/intervals/epochs", io); + + REQUIRE(timeIntervals != nullptr); + + Status status = timeIntervals->initialize("Test epochs"); + REQUIRE(status == Status::Success); + + // Verify intervals group was created + REQUIRE(io->objectExists("/intervals") == true); + + // Add some rows + AQNWB::Types::RowData row1 = {{"start_time", 1.0f}, {"stop_time", 2.0f}}; + + AQNWB::Types::RowData row2 = {{"start_time", 2.5f}, {"stop_time", 3.5f}}; + + REQUIRE(timeIntervals->addRow(row1) == Status::Success); + REQUIRE(timeIntervals->addRow(row2) == Status::Success); + + REQUIRE(timeIntervals->finalize() == Status::Success); + } + + // Read back and verify + { + auto io = createIO("HDF5", filename); + io->open(FileMode::ReadOnly); + auto timeIntervals = TimeIntervals::create("/intervals/epochs", io); + + REQUIRE(timeIntervals != nullptr); + + auto startTimeCol = timeIntervals->readStartTime(); + REQUIRE(startTimeCol != nullptr); + + auto stopTimeCol = timeIntervals->readStopTime(); + REQUIRE(stopTimeCol != nullptr); + + auto startTimeData = startTimeCol->readData(); + auto stopTimeData = stopTimeCol->readData(); + + REQUIRE(startTimeData->getShape()[0] == 2); + REQUIRE(stopTimeData->getShape()[0] == 2); + + auto startTimes = startTimeData->values().data; + auto stopTimes = stopTimeData->values().data; + + REQUIRE(startTimes[0] == Catch::Approx(1.0f)); + REQUIRE(startTimes[1] == Catch::Approx(2.5f)); + + REQUIRE(stopTimes[0] == Catch::Approx(2.0f)); + REQUIRE(stopTimes[1] == Catch::Approx(3.5f)); + } + } + + SECTION("Initialization at non-intervals path") + { + { + auto io = createIO("HDF5", filename); + io->open(FileMode::Overwrite); + + // Verify intervals group does not exist initially + REQUIRE(io->objectExists("/intervals") == false); + REQUIRE(io->createGroup("/intervals2") == Status::Success); + + auto timeIntervals = TimeIntervals::create("/intervals2/epochs", io); + + REQUIRE(timeIntervals != nullptr); + + Status status = timeIntervals->initialize("Test epochs"); + REQUIRE(status == Status::Success); + + // Verify intervals group was NOT created + REQUIRE(io->objectExists("/intervals") == false); + } + } + + SECTION("Initialization with tags column") + { + { + auto io = createIO("HDF5", filename); + io->open(FileMode::Overwrite); + + // Verify intervals group does not exist initially + REQUIRE(io->objectExists("/intervals") == false); + + auto timeIntervals = TimeIntervals::create("/intervals/epochs", io); + + REQUIRE(timeIntervals != nullptr); + + auto specs = + TimeIntervals::createDefaultDataSpecs("/intervals/epochs", 100, true); + Status status = timeIntervals->initialize("Test epochs with tags", specs); + REQUIRE(status == Status::Success); + + // Verify intervals group was created + REQUIRE(io->objectExists("/intervals") == true); + + // Add rows with tags + AQNWB::Types::RowData row1 = { + {"start_time", 1.0f}, + {"stop_time", 2.0f}, + {"tags", std::vector {"tag1", "tag2"}}}; + + AQNWB::Types::RowData row2 = { + {"start_time", 2.5f}, + {"stop_time", 3.5f}, + {"tags", std::vector {"tag3"}}}; + + REQUIRE(timeIntervals->addRow(row1) == Status::Success); + REQUIRE(timeIntervals->addRow(row2) == Status::Success); + + REQUIRE(timeIntervals->finalize() == Status::Success); + } + + // Read back and verify + { + auto io = createIO("HDF5", filename); + io->open(FileMode::ReadOnly); + auto timeIntervals = TimeIntervals::create("/intervals/epochs", io); + + REQUIRE(timeIntervals != nullptr); + + auto tagsCol = timeIntervals->readTags(); + REQUIRE(tagsCol != nullptr); + + auto tagsIndexCol = timeIntervals->readTagsIndex(); + REQUIRE(tagsIndexCol != nullptr); + + auto tagsData = tagsCol->readData(); + auto tagsIndexData = tagsIndexCol->readData(); + + REQUIRE(tagsData->getShape()[0] == 3); // 3 tags total + REQUIRE(tagsIndexData->getShape()[0] == 2); // 2 rows + + auto tags = tagsData->values().data; + + REQUIRE(tags[0] == "tag1"); + REQUIRE(tags[1] == "tag2"); + REQUIRE(tags[2] == "tag3"); + + auto tagsIndex = tagsIndexData->values().data; + + REQUIRE(tagsIndex[0] == 2); + REQUIRE(tagsIndex[1] == 3); + } + } +} \ No newline at end of file diff --git a/tests/testTimestampVectorData.cpp b/tests/testTimestampVectorData.cpp new file mode 100644 index 000000000..dec18e619 --- /dev/null +++ b/tests/testTimestampVectorData.cpp @@ -0,0 +1,179 @@ +#include +#include + +#include "Types.hpp" +#include "io/BaseIO.hpp" +#include "nwb/RegisteredType.hpp" +#include "nwb/event/TimestampVectorData.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("TimestampVectorData", "[event]") +{ + SECTION( + "test TimestampVectorData is registered as a subclass of RegisteredType") + { + auto registry = AQNWB::NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("core::TimestampVectorData") != registry.end()); + } + + SECTION("test TimestampVectorData initialize, write, and read") + { + std::string path = getTestFilePath("testTimestampVectorData.h5"); + std::string dataPath = "/timestamps"; + std::string description = "Test timestamp data"; + float resolution = 1.0f / 30000.0f; + std::vector timestamps = {0.0f, 0.1f, 0.25f, 0.5f, 0.75f}; + SizeArray dataShape = {timestamps.size()}; + SizeArray chunking = {timestamps.size()}; + SizeArray positionOffset = {0}; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto timestampVectorData = + AQNWB::NWB::TimestampVectorData::create(dataPath, io); + REQUIRE(timestampVectorData != nullptr); + + Status initStatus = + timestampVectorData->initialize(config, description, resolution); + REQUIRE(initStatus == Status::Success); + + auto dataRecorder = timestampVectorData->recordData(); + REQUIRE(dataRecorder != nullptr); + + Status writeStatus = dataRecorder->writeDataBlock( + dataShape, positionOffset, BaseDataType::F32, timestamps.data()); + REQUIRE(writeStatus == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + REQUIRE(readDataUntyped != nullptr); + + auto readTimestampVectorData = + std::dynamic_pointer_cast( + readDataUntyped); + REQUIRE(readTimestampVectorData != nullptr); + + REQUIRE(readTimestampVectorData->getNamespace() == "core"); + REQUIRE(readTimestampVectorData->getTypeName() == "TimestampVectorData"); + + auto namespaceData = readTimestampVectorData->readNamespace(); + REQUIRE(namespaceData->values().data[0] == "core"); + + auto neurodataTypeData = readTimestampVectorData->readNeurodataType(); + REQUIRE(neurodataTypeData->values().data[0] == "TimestampVectorData"); + + auto descriptionData = readTimestampVectorData->readDescription(); + REQUIRE(descriptionData->values().data[0] == description); + + auto unitData = readTimestampVectorData->readUnit(); + REQUIRE(unitData->values().data[0] == "seconds"); + + auto resolutionData = readTimestampVectorData->readResolution(); + REQUIRE(resolutionData->values().data[0] == Catch::Approx(resolution)); + + auto timestampData = readTimestampVectorData->readData(); + auto writtenValues = timestampData->values().data; + REQUIRE(writtenValues.size() == timestamps.size()); + for (size_t i = 0; i < writtenValues.size(); ++i) { + REQUIRE(writtenValues[i] == Catch::Approx(timestamps[i])); + } + + io->close(); + } + } + + SECTION("test TimestampVectorData findOwnedTypes") + { + std::string path = getTestFilePath("testTimestampVectorDataOwnedTypes.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + SizeArray dataShape = {4}; + SizeArray chunking = {4}; + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto timestampVectorData = + AQNWB::NWB::TimestampVectorData::create("/timestamps", io); + REQUIRE(timestampVectorData != nullptr); + + Status initStatus = + timestampVectorData->initialize(config, "Owned types test", 0.001f); + REQUIRE(initStatus == Status::Success); + + auto ownedTypes = timestampVectorData->findOwnedTypes(); + REQUIRE(ownedTypes.empty()); + + io->close(); + } + + SECTION("test TimestampVectorData initialize with optional resolution") + { + std::string path = getTestFilePath("testTimestampVectorDataOptional.h5"); + std::string dataPath = "/timestamps_optional"; + std::vector timestamps = {0.0f, 0.1f, 0.25f}; + SizeArray dataShape = {timestamps.size()}; + SizeArray chunking = {timestamps.size()}; + + { + std::string description = "Test timestamp data optional"; + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + auto timestampVectorData = + AQNWB::NWB::TimestampVectorData::create(dataPath, io); + REQUIRE(timestampVectorData != nullptr); + + Status initStatus = + timestampVectorData->initialize(config, description, std::nullopt); + REQUIRE(initStatus == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + REQUIRE(readDataUntyped != nullptr); + + auto readTimestampVectorData = + std::dynamic_pointer_cast( + readDataUntyped); + REQUIRE(readTimestampVectorData != nullptr); + + auto resolutionData = readTimestampVectorData->readResolution(); + REQUIRE(resolutionData->exists() == false); + + io->close(); + } + } + + SECTION("test TimestampVectorData initialize fails after IO deletion") + { + SizeArray dataShape = {3}; + SizeArray chunking = {3}; + IO::ArrayDataSetConfig config(BaseDataType::F32, dataShape, chunking); + + auto io = + createIO("HDF5", getTestFilePath("testTimestampVectorDataNoIO.h5")); + auto timestampVectorData = + AQNWB::NWB::TimestampVectorData::create("/timestamps", io); + REQUIRE(timestampVectorData != nullptr); + + io.reset(); + + Status initStatus = + timestampVectorData->initialize(config, "Missing IO", 0.01f); + REQUIRE(initStatus == Status::Failure); + } +} diff --git a/tests/testTypes.cpp b/tests/testTypes.cpp index 34e565c73..0ee3e0143 100644 --- a/tests/testTypes.cpp +++ b/tests/testTypes.cpp @@ -156,3 +156,90 @@ TEST_CASE("Test SizeTypeNotSet constant", "[Types]") REQUIRE(index != Types::SizeTypeNotSet); } } + +TEST_CASE("Test CellValue", "[Types]") +{ + SECTION("Default constructor") + { + Types::CellValue cell; + REQUIRE(cell.holds_alternative()); + REQUIRE(cell.toString() == "null"); + } + + SECTION("Scalar constructors and get") + { + Types::CellValue intCell(42); + REQUIRE(intCell.holds_alternative()); + REQUIRE(intCell.get() == 42); + REQUIRE(intCell.toString() == "42"); + + Types::CellValue doubleCell(3.14); + REQUIRE(doubleCell.holds_alternative()); + REQUIRE_THAT(doubleCell.get(), + Catch::Matchers::WithinRel(3.14, 1e-5)); + // toString for double might have trailing zeros, so we just check it + // doesn't throw + REQUIRE_NOTHROW(doubleCell.toString()); + + Types::CellValue stringCell("hello"); + REQUIRE(stringCell.holds_alternative()); + REQUIRE(stringCell.get() == "hello"); + REQUIRE(stringCell.toString() == "hello"); + + std::string str = "world"; + Types::CellValue stdStringCell(str); + REQUIRE(stdStringCell.holds_alternative()); + REQUIRE(stdStringCell.get() == "world"); + REQUIRE(stdStringCell.toString() == "world"); + } + + SECTION("Vector constructors and get") + { + std::vector intVec = {1, 2, 3}; + Types::CellValue intVecCell(intVec); + REQUIRE(intVecCell.holds_alternative>()); + REQUIRE(intVecCell.get>() == intVec); + REQUIRE(intVecCell.toString() == "[1, 2, 3]"); + + std::vector strVec = {"a", "b"}; + Types::CellValue strVecCell(strVec); + REQUIRE(strVecCell.holds_alternative>()); + REQUIRE(strVecCell.get>() == strVec); + REQUIRE(strVecCell.toString() == "[a, b]"); + } + + SECTION("Implicit conversion operator") + { + Types::CellValue intCell(42); + int val = intCell; + REQUIRE(val == 42); + + Types::CellValue strCell("test"); + std::string strVal = strCell; + REQUIRE(strVal == "test"); + + std::vector vec = {1, 2}; + Types::CellValue vecCell(vec); + std::vector vecVal = vecCell; + REQUIRE(vecVal == vec); + } + + SECTION("holds_alternative with wrong type") + { + Types::CellValue intCell(42); + REQUIRE_FALSE(intCell.holds_alternative()); + REQUIRE_FALSE(intCell.holds_alternative()); + REQUIRE_FALSE(intCell.holds_alternative>()); + + Types::CellValue vecCell(std::vector {1, 2}); + REQUIRE_FALSE(vecCell.holds_alternative()); + REQUIRE_FALSE(vecCell.holds_alternative>()); + } + + SECTION("get with wrong type throws") + { + Types::CellValue intCell(42); + REQUIRE_THROWS_AS(intCell.get(), std::bad_variant_access); + REQUIRE_THROWS_AS(intCell.get>(), std::bad_variant_access); + } +} diff --git a/tests/testUtils.hpp b/tests/testUtils.hpp index c3fe3f29d..e57f53ba3 100644 --- a/tests/testUtils.hpp +++ b/tests/testUtils.hpp @@ -11,6 +11,7 @@ #include "Channel.hpp" #include "Types.hpp" #include "io/hdf5/HDF5IO.hpp" +#include "nwb/file/Subject.hpp" using namespace AQNWB; using namespace AQNWB::IO; @@ -49,6 +50,22 @@ inline std::string getTestFilePath(const std::string& filename) return filepath.generic_string(); } +inline NWB::Subject::SubjectSpec getTestSubjectSpec() +{ + NWB::Subject::SubjectSpec subjectSpec; + subjectSpec.age = "P90D"; + subjectSpec.ageReference = "birth"; + subjectSpec.dateOfBirth = "2024-01-15T00:00:00.000000+00:00"; + subjectSpec.description = "Test subject"; + subjectSpec.genotype = "wt/wt"; + subjectSpec.sex = "U"; + subjectSpec.species = "Mus musculus"; + subjectSpec.strain = "C57BL/6J"; + subjectSpec.subjectId = "test-subject"; + subjectSpec.weight = "25 g"; + return subjectSpec; +} + inline std::vector getMockChannelArrays( SizeType numChannels = 2, SizeType numArrays = 2, diff --git a/tests/testUtilsFunctions.cpp b/tests/testUtilsFunctions.cpp index c5d66e14e..16ddcf12c 100644 --- a/tests/testUtilsFunctions.cpp +++ b/tests/testUtilsFunctions.cpp @@ -122,6 +122,14 @@ TEST_CASE("Test path merging", "[utils]") } } +TEST_CASE("Test path ancestry", "[utils]") +{ + REQUIRE(AQNWB::isPathOrDescendant("/events", "/events")); + REQUIRE(AQNWB::isPathOrDescendant("/events/table", "/events")); + REQUIRE_FALSE(AQNWB::isPathOrDescendant("/events2/table", "/events")); + REQUIRE_FALSE(AQNWB::isPathOrDescendant("/event", "/events")); +} + TEST_CASE("Test time conversion functions", "[utils]") { std::time_t now = std::time(nullptr); diff --git a/tests/testVectorData.cpp b/tests/testVectorData.cpp index 8bcef9a32..29292eb38 100644 --- a/tests/testVectorData.cpp +++ b/tests/testVectorData.cpp @@ -4,6 +4,7 @@ #include "Types.hpp" #include "Utils.hpp" #include "io/BaseIO.hpp" +#include "io/RecordingObjects.hpp" #include "io/hdf5/HDF5RecordingData.hpp" #include "nwb/RegisteredType.hpp" #include "nwb/hdmf/table/VectorData.hpp" @@ -151,6 +152,16 @@ TEST_CASE("VectorData", "[base]") TEST_CASE("VectorDataTyped", "[base]") { + SECTION("typed vector data facades are not registered") + { + auto io = createIO("HDF5", getTestFilePath("testVectorDataTypedFacade.h5")); + const std::string dataPath = "/typed_vector_data"; + + auto vectorDataTyped = NWB::VectorDataTyped::create(dataPath, io); + REQUIRE(vectorDataTyped != nullptr); + REQUIRE(io->getRecordingObjects()->getRecordingObject(dataPath) == nullptr); + } + SECTION("test VectorDataTyped with int") { std::string path = getTestFilePath("testVectorDataTyped_int.h5"); @@ -213,6 +224,9 @@ TEST_CASE("VectorDataTyped", "[base]") auto convertedVectorDataTyped = NWB::VectorDataTyped::fromVectorData(baseVectorData); REQUIRE(convertedVectorDataTyped != nullptr); + REQUIRE(std::dynamic_pointer_cast>( + convertedVectorDataTyped) + != nullptr); // Read the "namespace" attribute via the readNamespace field namespaceData = convertedVectorDataTyped->readNamespace(); @@ -486,6 +500,115 @@ TEST_CASE("VectorDataTyped", "[base]") recordIo->close(); } + + SECTION("test appendValue with std::vector") + { + std::string path = getTestFilePath("testVectorDataAppendVector.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + std::string dataPath = "/vdata_append_vector"; + IO::ArrayDataSetConfig config(BaseDataType::I32, {0}, {10}); + auto vectorData = NWB::VectorData::create(dataPath, io); + vectorData->initialize(config, "Test append vector"); + + std::vector vec1 = {1, 2, 3}; + std::vector vec2 = {4, 5}; + std::vector vec3 = {6, 7, 8, 9}; + + size_t elementsAppended = 0; + REQUIRE(vectorData->appendData(vec1, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 3); + REQUIRE(vectorData->appendData(vec2, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 2); + REQUIRE(vectorData->appendData(vec3, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 4); + + io->flush(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + auto readVectorData = + std::dynamic_pointer_cast(readDataUntyped); + REQUIRE(readVectorData != nullptr); + + auto dataGeneric = readVectorData->readData()->valuesGeneric(); + auto dataTyped = DataBlock::fromGeneric(dataGeneric); + auto data = dataTyped.data; + + REQUIRE(data.size() == 9); + REQUIRE(data == std::vector({1, 2, 3, 4, 5, 6, 7, 8, 9})); + + io->close(); + } + + SECTION("test appendBuffer") + { + std::string path = getTestFilePath("testVectorDataAppendBuffer.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + std::string dataPath = "/vdata_append_buffer"; + IO::ArrayDataSetConfig config(BaseDataType::I32, {0}, {10}); + auto vectorData = NWB::VectorData::create(dataPath, io); + vectorData->initialize(config, "Test append buffer"); + + std::vector vec1 = {1, 2, 3}; + std::vector vec2 = {4, 5}; + std::vector vec3 = {6, 7, 8, 9}; + + IO::BaseDataType::BaseDataVectorVariant var1 = vec1; + IO::BaseDataType::BaseDataVectorVariant var2 = vec2; + IO::BaseDataType::BaseDataVectorVariant var3 = vec3; + + REQUIRE(vectorData->appendBuffer(var1) == Status::Success); + REQUIRE(vectorData->appendBuffer(var2) == Status::Success); + REQUIRE(vectorData->appendBuffer(var3) == Status::Success); + + io->flush(); + + auto readDataUntyped = NWB::RegisteredType::create(dataPath, io); + auto readVectorData = + std::dynamic_pointer_cast(readDataUntyped); + REQUIRE(readVectorData != nullptr); + + auto dataGeneric = readVectorData->readData()->valuesGeneric(); + auto dataTyped = DataBlock::fromGeneric(dataGeneric); + auto data = dataTyped.data; + + REQUIRE(data.size() == 9); + REQUIRE(data == std::vector({1, 2, 3, 4, 5, 6, 7, 8, 9})); + + io->close(); + } + + SECTION("rejects values with a mismatched dataset type") + { + std::string path = getTestFilePath("testVectorDataTypeMismatch.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto vectorData = NWB::VectorData::create("/type_mismatch", io); + REQUIRE(vectorData->initialize( + IO::ArrayDataSetConfig(BaseDataType::F64, {0}, {10}), + "Type mismatch test") + == Status::Success); + + size_t elementsAppended = 42; + REQUIRE(vectorData->appendData(int32_t {1}, elementsAppended) + == Status::Failure); + REQUIRE(elementsAppended == 0); + REQUIRE( + vectorData->appendData(std::vector {1, 2}, elementsAppended) + == Status::Failure); + REQUIRE(elementsAppended == 0); + + IO::BaseDataType::BaseDataVectorVariant buffer = + std::vector {1, 2}; + REQUIRE(vectorData->appendBuffer(buffer) == Status::Failure); + REQUIRE(vectorData->recordData()->getShape()[0] == 0); + + io->close(); + } } // TEST_CASE("VectorDataTyped", "[base]") TEST_CASE("LinkArrayDataSetConfig for VectorData", "[base][link]") diff --git a/tests/testVectorIndex.cpp b/tests/testVectorIndex.cpp new file mode 100644 index 000000000..cf829942c --- /dev/null +++ b/tests/testVectorIndex.cpp @@ -0,0 +1,534 @@ +#include + +#include "Types.hpp" +#include "io/BaseIO.hpp" +#include "nwb/RegisteredType.hpp" +#include "nwb/hdmf/table/DynamicTable.hpp" +#include "nwb/hdmf/table/VectorData.hpp" +#include "nwb/hdmf/table/VectorIndex.hpp" +#include "testUtils.hpp" + +using namespace AQNWB; + +TEST_CASE("VectorIndex", "[table]") +{ + SECTION("is registered as a subclass of RegisteredType") + { + auto registry = NWB::RegisteredType::getRegistry(); + REQUIRE(registry.find("hdmf-common::VectorIndex") != registry.end()); + } + + SECTION("initializes, writes, and reads its target") + { + const std::string path = getTestFilePath("testVectorIndex.h5"); + const std::string targetPath = "/ragged_values"; + const std::string indexPath = targetPath + "_index"; + const std::string targetDescription = "Ragged array values"; + const std::string indexDescription = "End indices for ragged array rows"; + const std::vector targetValues = {10, 11, 20, 21, 22, 30}; + const std::vector indices = {2, 5, 6}; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create(targetPath, io); + REQUIRE(target != nullptr); + REQUIRE(target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, + {targetValues.size()}, + {targetValues.size()}), + targetDescription) + == Status::Success); + + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + REQUIRE(vectorIndex != nullptr); + REQUIRE(vectorIndex->initialize( + IO::ArrayDataSetConfig( + BaseDataType::U32, {indices.size()}, {indices.size()}), + indexDescription, + targetPath) + == Status::Success); + + auto targetRecorder = target->recordData(); + auto indexRecorder = vectorIndex->recordData(); + REQUIRE(targetRecorder != nullptr); + REQUIRE(indexRecorder != nullptr); + + REQUIRE(io->startRecording() == Status::Success); + REQUIRE(targetRecorder->writeDataBlock({targetValues.size()}, + {0}, + BaseDataType::I32, + targetValues.data()) + == Status::Success); + REQUIRE(indexRecorder->writeDataBlock( + {indices.size()}, {0}, BaseDataType::U32, indices.data()) + == Status::Success); + REQUIRE(io->stopRecording() == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readData = NWB::RegisteredType::create(indexPath, io); + auto vectorIndex = std::dynamic_pointer_cast(readData); + REQUIRE(vectorIndex != nullptr); + REQUIRE(vectorIndex->getNamespace() == "hdmf-common"); + REQUIRE(vectorIndex->getTypeName() == "VectorIndex"); + REQUIRE(vectorIndex->readDescription()->values().data[0] + == indexDescription); + REQUIRE(vectorIndex->readData()->values().data == indices); + + auto target = vectorIndex->readTarget(); + REQUIRE(target != nullptr); + REQUIRE(target->getPath() == targetPath); + REQUIRE(target->readDescription()->values().data[0] == targetDescription); + auto typedTarget = NWB::VectorDataTyped::fromVectorData(target); + REQUIRE(typedTarget != nullptr); + REQUIRE(typedTarget->readData()->values().data == targetValues); + + io->close(); + } + } + + SECTION("rejects non-unsigned and non-one-dimensional configurations") + { + const std::string path = getTestFilePath("testVectorIndexValidation.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto vectorIndex = NWB::VectorIndex::create("/index", io); + REQUIRE(vectorIndex != nullptr); + + REQUIRE_THROWS_AS(vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::I32, {3}, {3}), + "Invalid index type", + "/target"), + std::invalid_argument); + REQUIRE_THROWS_AS( + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {2, 2}, {2, 2}), + "Invalid index dimensions", + "/target"), + std::invalid_argument); + + io->close(); + } + + SECTION("test appendData") + { + const std::string path = getTestFilePath("testVectorIndexAppendData.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + const std::string targetPath = "/ragged_values"; + const std::string indexPath = targetPath + "_index"; + + auto target = NWB::VectorData::create(targetPath, io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + targetPath); + + std::vector vec1 = {1, 2}; + std::vector vec2 = {3, 4, 5}; + std::vector vec3 = {6}; + + size_t elementsAppended = 0; + REQUIRE(vectorIndex->appendData(vec1, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 2); + REQUIRE(vectorIndex->appendData(vec2, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 3); + REQUIRE(vectorIndex->appendData(vec3, elementsAppended) == Status::Success); + REQUIRE(elementsAppended == 1); + + io->flush(); + + auto readData = NWB::RegisteredType::create(indexPath, io); + auto readVectorIndex = + std::dynamic_pointer_cast(readData); + REQUIRE(readVectorIndex != nullptr); + + auto indices = readVectorIndex->readData()->values().data; + REQUIRE(indices.size() == 3); + REQUIRE(indices[0] == 2); + REQUIRE(indices[1] == 5); + REQUIRE(indices[2] == 6); + + io->close(); + } + + SECTION("DataSpec initialization") + { + const std::string path = getTestFilePath("testVectorIndexDataSpec.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create("/target", io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + + auto vectorIndex = NWB::VectorIndex::create("/index", io); + NWB::VectorIndex::DataSpec spec( + "index", + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index desc", + "/target"); + + REQUIRE(spec.initialize(*vectorIndex) == Status::Success); + REQUIRE(vectorIndex->readDescription()->values().data[0] == "Index desc"); + + // Test failure with wrong type + auto wrongData = NWB::VectorData::create("/wrong", io); + REQUIRE(spec.initialize(*wrongData) == Status::Failure); + + io->close(); + } + + SECTION("getTargetColumn lazy loading") + { + const std::string path = getTestFilePath("testVectorIndexLazyLoad.h5"); + const std::string targetPath = "/target"; + const std::string indexPath = "/index"; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + auto target = NWB::VectorData::create(targetPath, io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + Status status = vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + targetPath); + REQUIRE(status == Status::Success); + io->close(); + } + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + auto readData = NWB::RegisteredType::create(indexPath, io); + auto vectorIndex = std::dynamic_pointer_cast(readData); + REQUIRE(vectorIndex != nullptr); + + // m_targetColumn should be null initially, getTargetColumn should read it + auto target = vectorIndex->getTargetColumn(); + REQUIRE(target != nullptr); + REQUIRE(target->getPath() == targetPath); + + // Second call should return cached + auto target2 = vectorIndex->getTargetColumn(); + REQUIRE(target == target2); + + io->close(); + } + } + + SECTION("readIndexedCellValues") + { + const std::string path = getTestFilePath("testVectorIndexReadIndexed.h5"); + const std::string targetPath = "/target"; + const std::string indexPath = "/index"; + + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create(targetPath, io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::F32, {0}, {10}), + "Target"); + + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + targetPath); + + std::vector vec1 = {1.0f, 2.0f}; + std::vector vec2 = {}; // Empty vector + std::vector vec3 = {3.0f, 4.0f, 5.0f}; + std::vector vec4 = {6.0f}; + + size_t elementsAppended = 0; + vectorIndex->appendData(vec1, elementsAppended); + vectorIndex->appendData(vec2, elementsAppended); + vectorIndex->appendData(vec3, elementsAppended); + vectorIndex->appendData(vec4, elementsAppended); + + io->flush(); + + // Read all + auto allCells = vectorIndex->readIndexedCellValues(); + REQUIRE(allCells.size() == 4); + REQUIRE(allCells[0].get>() == vec1); + REQUIRE(allCells[1].get>() == vec2); + REQUIRE(allCells[2].get>() == vec3); + REQUIRE(allCells[3].get>() == vec4); + + // Read slice (start=1, count=2) + auto sliceCells = vectorIndex->readIndexedCellValues(1, 2); + REQUIRE(sliceCells.size() == 2); + REQUIRE(sliceCells[0].get>() == vec2); + REQUIRE(sliceCells[1].get>() == vec3); + + // Read out of bounds + REQUIRE_THROWS(vectorIndex->readIndexedCellValues(10, 2)); + + io->close(); + } + + SECTION("readIndexedCellValues preserves empty string vector type") + { + const std::string path = + getTestFilePath("testVectorIndexReadIndexedStrings.h5"); + const std::string targetPath = "/target"; + const std::string indexPath = "/index"; + + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create(targetPath, io); + REQUIRE( + target->initialize(IO::ArrayDataSetConfig( + BaseDataType(BaseDataType::V_STR), {0}, {10}), + "Target") + == Status::Success); + + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + REQUIRE(vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + targetPath) + == Status::Success); + + size_t elementsAppended = 0; + REQUIRE(vectorIndex->appendData(std::vector {"one"}, + elementsAppended) + == Status::Success); + REQUIRE( + vectorIndex->appendData(std::vector {}, elementsAppended) + == Status::Success); + + const auto cells = vectorIndex->readIndexedCellValues(); + REQUIRE(cells.size() == 2); + REQUIRE(cells[0].get>() + == std::vector {"one"}); + REQUIRE(cells[1].get>().empty()); + + io->close(); + } + + SECTION("appendData and initializeAppendState with different data types") + { + std::vector types = {BaseDataType::Type::T_U8, + BaseDataType::Type::T_U16, + BaseDataType::Type::T_U32, + BaseDataType::Type::T_U64}; + + std::vector vec1 = {1, 2}; + std::vector vec2 = {3, 4, 5}; + + for (auto type : types) { + const std::string path = + getTestFilePath("testVectorIndexTypes_" + + std::to_string(static_cast(type)) + ".h5"); + const std::string targetPath = "/target"; + const std::string indexPath = "/index"; + + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create(targetPath, io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + + auto vectorIndex = NWB::VectorIndex::create(indexPath, io); + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType(type), {0}, {10}), + "Index", + targetPath); + + size_t elementsAppended = 0; + REQUIRE(vectorIndex->appendData(vec1, elementsAppended) + == Status::Success); + REQUIRE(elementsAppended == 2); + + io->close(); + } + + // Re-open and append more to test initializeAppendState + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto readData = NWB::RegisteredType::create(indexPath, io); + auto vectorIndex = + std::dynamic_pointer_cast(readData); + REQUIRE(vectorIndex != nullptr); + + size_t elementsAppended = 0; + // This will trigger initializeAppendState + REQUIRE(vectorIndex->appendData(vec2, elementsAppended) + == Status::Success); + REQUIRE(elementsAppended == 3); + + io->flush(); + + // Verify indices + if (type == BaseDataType::Type::T_U8) { + auto indices = vectorIndex->readData()->values().data; + REQUIRE(indices.size() == 2); + REQUIRE(indices[0] == 2); + REQUIRE(indices[1] == 5); + } else if (type == BaseDataType::Type::T_U16) { + auto indices = vectorIndex->readData()->values().data; + REQUIRE(indices.size() == 2); + REQUIRE(indices[0] == 2); + REQUIRE(indices[1] == 5); + } else if (type == BaseDataType::Type::T_U32) { + auto indices = vectorIndex->readData()->values().data; + REQUIRE(indices.size() == 2); + REQUIRE(indices[0] == 2); + REQUIRE(indices[1] == 5); + } else if (type == BaseDataType::Type::T_U64) { + auto indices = vectorIndex->readData()->values().data; + REQUIRE(indices.size() == 2); + REQUIRE(indices[0] == 2); + REQUIRE(indices[1] == 5); + } + + // Test readIndexedCellValues to cover extractIndexValue for all types + auto cells = vectorIndex->readIndexedCellValues(); + REQUIRE(cells.size() == 2); + REQUIRE(cells[0].get>() == vec1); + REQUIRE(cells[1].get>() == vec2); + + // Test readIndexedCellValues with start/count/stride/block + auto cells2 = vectorIndex->readIndexedCellValues( + 0, 0); // count 0 should return empty + REQUIRE(cells2.empty()); + // Stride of 0 should throw + REQUIRE_THROWS(vectorIndex->readIndexedCellValues(0, 2, 0)); + // Block of 0 should throw + REQUIRE_THROWS(vectorIndex->readIndexedCellValues(0, 2, 1, 0)); + + io->close(); + } + } + } + + SECTION("Uninitialized VectorIndex") + { + const std::string path = getTestFilePath("testVectorIndexUninitialized.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto vectorIndex = NWB::VectorIndex::create("/index", io); + + std::vector vec = {1, 2}; + size_t elementsAppended = 0; + + // appendData on uninitialized index should fail + REQUIRE(vectorIndex->appendData(vec, elementsAppended) == Status::Failure); + + // readIndexedCellValues on uninitialized index should throw runtime_error + REQUIRE_THROWS_AS(vectorIndex->readIndexedCellValues(), std::runtime_error); + + io->close(); + } + + SECTION("Initialize in read-only mode") + { + const std::string path = getTestFilePath("testVectorIndexReadOnly.h5"); + { + std::shared_ptr io = createIO("HDF5", path); + io->open(); + io->close(); + } + + std::shared_ptr io = createIO("HDF5", path); + io->open(AQNWB::IO::FileMode::ReadOnly); + + auto vectorIndex = NWB::VectorIndex::create("/index", io); + REQUIRE(vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + "/target") + == Status::Failure); + + io->close(); + } + + SECTION("Index out of bounds of target") + { + const std::string path = getTestFilePath("testVectorIndexOutOfBounds.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create("/target", io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + auto vectorIndex = NWB::VectorIndex::create("/index", io); + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + "/target"); + + // Append 2 elements to target + std::vector vec = {1, 2}; + size_t elementsAppended = 0; + vectorIndex->appendData(vec, elementsAppended); + // Manually write an index that is out of bounds (e.g., 5) + { + auto indexRecorder = vectorIndex->recordData(); + io->startRecording(); + std::vector badIndices = {5}; + indexRecorder->writeDataBlock( + {1}, {1}, BaseDataType::U32, badIndices.data()); + io->flush(); + } + REQUIRE_THROWS_AS(vectorIndex->readIndexedCellValues(), std::out_of_range); + + io->stopRecording(); + io->close(); + } + + SECTION("Invalid index data (decreasing indices)") + { + const std::string path = getTestFilePath("testVectorIndexInvalidData.h5"); + std::shared_ptr io = createIO("HDF5", path); + io->open(); + + auto target = NWB::VectorData::create("/target", io); + target->initialize(IO::ArrayDataSetConfig(BaseDataType::I32, {0}, {10}), + "Target"); + + auto vectorIndex = NWB::VectorIndex::create("/index", io); + vectorIndex->initialize( + IO::ArrayDataSetConfig(BaseDataType::U32, {0}, {10}), + "Index", + "/target"); + + // Manually write invalid index data (decreasing indices) + { + auto indexRecorder = vectorIndex->recordData(); + io->startRecording(); + std::vector badIndices = {5, 2}; // Decreasing indices + indexRecorder->writeDataBlock( + {2}, {0}, BaseDataType::U32, badIndices.data()); + io->flush(); + } + + REQUIRE_THROWS_AS(vectorIndex->readIndexedCellValues(), + std::invalid_argument); + io->stopRecording(); + io->close(); + } +}